/** * Creates a human-friendly representation of the Collector name. * * InternalBucket Collectors use the aggregation name in their toString() method, * which makes the profiled output a bit nicer. * * @param c The Collector to derive a name from * @return A (hopefully) prettier name */ private String deriveCollectorName(Collector c) { String s = c.getClass().getSimpleName(); // MutiCollector which wraps multiple BucketCollectors is generated // via an anonymous class, so this corrects the lack of a name by // asking the enclosingClass if (s.equals("")) { s = c.getClass().getEnclosingClass().getSimpleName(); } // Aggregation collector toString()'s include the user-defined agg name if (reason.equals(CollectorResult.REASON_AGGREGATION) || reason.equals(CollectorResult.REASON_AGGREGATION_GLOBAL)) { s += ": [" + c.toString() + "]"; } return s; }
private void countTestCase(Query query, IndexReader reader, boolean shouldCollect) throws Exception { TestSearchContext context = new TestSearchContext(null); context.parsedQuery(new ParsedQuery(query)); context.setSize(0); context.setTask(new SearchTask(123L, "", "", "", null)); IndexSearcher searcher = new IndexSearcher(reader); final AtomicBoolean collected = new AtomicBoolean(); IndexSearcher contextSearcher = new IndexSearcher(reader) { protected void search(List<LeafReaderContext> leaves, Weight weight, Collector collector) throws IOException { collected.set(true); super.search(leaves, weight, collector); } }; final boolean rescore = QueryPhase.execute(context, contextSearcher); assertFalse(rescore); assertEquals(searcher.count(query), context.queryResult().topDocs().totalHits); assertEquals(shouldCollect, collected.get()); }
public void testPostFilterDisablesCountOptimization() throws Exception { TestSearchContext context = new TestSearchContext(null); context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery())); context.setSize(0); context.setTask(new SearchTask(123L, "", "", "", null)); final AtomicBoolean collected = new AtomicBoolean(); IndexSearcher contextSearcher = new IndexSearcher(new MultiReader()) { protected void search(List<LeafReaderContext> leaves, Weight weight, Collector collector) throws IOException { collected.set(true); super.search(leaves, weight, collector); } }; QueryPhase.execute(context, contextSearcher); assertEquals(0, context.queryResult().topDocs().totalHits); assertFalse(collected.get()); context.parsedPostFilter(new ParsedQuery(new MatchNoDocsQuery())); QueryPhase.execute(context, contextSearcher); assertEquals(0, context.queryResult().topDocs().totalHits); assertTrue(collected.get()); }
public void testMinScoreDisablesCountOptimization() throws Exception { TestSearchContext context = new TestSearchContext(null); context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery())); context.setSize(0); context.setTask(new SearchTask(123L, "", "", "", null)); final AtomicBoolean collected = new AtomicBoolean(); IndexSearcher contextSearcher = new IndexSearcher(new MultiReader()) { protected void search(List<LeafReaderContext> leaves, Weight weight, Collector collector) throws IOException { collected.set(true); super.search(leaves, weight, collector); } }; QueryPhase.execute(context, contextSearcher); assertEquals(0, context.queryResult().topDocs().totalHits); assertFalse(collected.get()); context.minimumScore(1); QueryPhase.execute(context, contextSearcher); assertEquals(0, context.queryResult().topDocs().totalHits); assertTrue(collected.get()); }
/** * Creates a human-friendly representation of the Collector name. * * Bucket Collectors use the aggregation name in their toString() method, * which makes the profiled output a bit nicer. * * @param c The Collector to derive a name from * @return A (hopefully) prettier name */ private String deriveCollectorName(Collector c) { String s = c.getClass().getSimpleName(); // MutiCollector which wraps multiple BucketCollectors is generated // via an anonymous class, so this corrects the lack of a name by // asking the enclosingClass if (s.equals("")) { s = c.getClass().getEnclosingClass().getSimpleName(); } // Aggregation collector toString()'s include the user-defined agg name if (reason.equals(CollectorResult.REASON_AGGREGATION) || reason.equals(CollectorResult.REASON_AGGREGATION_GLOBAL)) { s += ": [" + c.toString() + "]"; } return s; }
private Collector getInsanityWrapper(final String field, Collector collector) { SchemaField sf = searcher.getSchema().getFieldOrNull(field); if (sf != null && !sf.hasDocValues() && !sf.multiValued() && sf.getType().getNumberType() != null) { // it's a single-valued numeric field: we must currently create insanity :( // there isn't a GroupedFacetCollector that works on numerics right now... return new FilterCollector(collector) { @Override public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { LeafReader insane = Insanity.wrapInsanity(context.reader(), field); return in.getLeafCollector(insane.getContext()); } }; } else { return collector; } }
/** * Search all documents matching the passed query and pass the result on to * the provided {@link Consumer}. * * @param aQuery * Query to execute. May not be <code>null</code>- * @param aCollector * The Lucene collector to be used. May not be <code>null</code>. * @throws IOException * On Lucene error * @see #getAllDocuments(Query) */ public void searchAtomic (@Nonnull final Query aQuery, @Nonnull final Collector aCollector) throws IOException { ValueEnforcer.notNull (aQuery, "Query"); ValueEnforcer.notNull (aCollector, "Collector"); m_aLucene.runAtomic ( () -> { final IndexSearcher aSearcher = m_aLucene.getSearcher (); if (aSearcher != null) { if (s_aLogger.isDebugEnabled ()) s_aLogger.debug ("Searching Lucene: " + aQuery); // Search all documents, collect them _timedSearch ( () -> aSearcher.search (aQuery, aCollector), aQuery); } else s_aLogger.error ("Failed to obtain IndexSearcher"); }); }
@Nonnull @ReturnsMutableCopy public ICommonsSortedSet <IParticipantIdentifier> getAllContainedParticipantIDs () { final ICommonsSortedSet <IParticipantIdentifier> aTargetSet = new CommonsTreeSet <> (); final Query aQuery = PDQueryManager.andNotDeleted (new MatchAllDocsQuery ()); try { final ObjIntConsumer <Document> aConsumer = (aDoc, nDocID) -> aTargetSet.add (PDField.PARTICIPANT_ID.getDocValue (aDoc)); final Collector aCollector = new AllDocumentsCollector (m_aLucene, aConsumer); searchAtomic (aQuery, aCollector); } catch (final IOException ex) { s_aLogger.error ("Error searching for documents with query " + aQuery, ex); } return aTargetSet; }
private void collectHit(Collector collector, Collector[] sidewaysCollectors) throws IOException { //if (DEBUG) { // System.out.println(" hit"); //} collector.collect(collectDocID); if (drillDownCollector != null) { drillDownCollector.collect(collectDocID); } // TODO: we could "fix" faceting of the sideways counts // to do this "union" (of the drill down hits) in the // end instead: // Tally sideways counts: for (int dim=0;dim<sidewaysCollectors.length;dim++) { sidewaysCollectors[dim].collect(collectDocID); } }
private void collectHit(Collector collector, Collector[] sidewaysCollectors, Collector[] sidewaysCollectors2) throws IOException { //if (DEBUG) { // System.out.println(" hit"); //} collector.collect(collectDocID); if (drillDownCollector != null) { drillDownCollector.collect(collectDocID); } // TODO: we could "fix" faceting of the sideways counts // to do this "union" (of the drill down hits) in the // end instead: // Tally sideways counts: for (int i=0;i<sidewaysCollectors.length;i++) { sidewaysCollectors[i].collect(collectDocID); } for (int i=0;i<sidewaysCollectors2.length;i++) { sidewaysCollectors2[i].collect(collectDocID); } }
@Override public boolean score(Collector collector, int max) throws IOException { FakeScorer fakeScorer = new FakeScorer(); collector.setScorer(fakeScorer); if (doc == -1) { doc = nextDocOutOfOrder(); } while(doc < max) { fakeScorer.doc = doc; fakeScorer.score = scores[ords[scoreUpto]]; collector.collect(doc); doc = nextDocOutOfOrder(); } return doc != DocsEnum.NO_MORE_DOCS; }
@Test public void testNullCollectors() throws Exception { // Tests that the collector rejects all null collectors. try { MultiCollector.wrap(null, null); fail("only null collectors should not be supported"); } catch (IllegalArgumentException e) { // expected } // Tests that the collector handles some null collectors well. If it // doesn't, an NPE would be thrown. Collector c = MultiCollector.wrap(new DummyCollector(), null, new DummyCollector()); assertTrue(c instanceof MultiCollector); assertTrue(c.acceptsDocsOutOfOrder()); c.collect(1); c.setNextReader(null); c.setScorer(null); }
@Test public void testCollector() throws Exception { // Tests that the collector delegates calls to input collectors properly. // Tests that the collector handles some null collectors well. If it // doesn't, an NPE would be thrown. DummyCollector[] dcs = new DummyCollector[] { new DummyCollector(), new DummyCollector() }; Collector c = MultiCollector.wrap(dcs); assertTrue(c.acceptsDocsOutOfOrder()); c.collect(1); c.setNextReader(null); c.setScorer(null); for (DummyCollector dc : dcs) { assertTrue(dc.acceptsDocsOutOfOrderCalled); assertTrue(dc.collectCalled); assertTrue(dc.setNextReaderCalled); assertTrue(dc.setScorerCalled); } }
private DocSet getDocSetScore(List<Query> queries) throws IOException { Query main = queries.remove(0); ProcessedFilter pf = getProcessedFilter(null, queries); DocSetCollector setCollector = new DocSetCollector(maxDoc()>>6, maxDoc()); Collector collector = setCollector; if (pf.postFilter != null) { pf.postFilter.setLastDelegate(collector); collector = pf.postFilter; } search(main, pf.filter, collector); if(collector instanceof DelegatingCollector) { ((DelegatingCollector) collector).finish(); } DocSet docSet = setCollector.getDocSet(); return docSet; }
@SuppressWarnings("unchecked") public void execute() throws IOException { final int nrOfCommands = commands.size(); List<Collector> collectors = new ArrayList<>(nrOfCommands); for (Command command : commands) { collectors.addAll(command.create()); } ProcessedFilter filter = searcher.getProcessedFilter (queryCommand.getFilter(), queryCommand.getFilterList()); Query query = QueryUtils.makeQueryable(queryCommand.getQuery()); if (truncateGroups) { docSet = computeGroupedDocSet(query, filter, collectors); } else if (needDocset) { docSet = computeDocSet(query, filter, collectors); } else if (!collectors.isEmpty()) { searchWithTimeLimiter(query, filter, MultiCollector.wrap(collectors.toArray(new Collector[nrOfCommands]))); } else { searchWithTimeLimiter(query, filter, null); } }
public GroupExpandCollector(SortedDocValues docValues, FixedBitSet groupBits, IntOpenHashSet collapsedSet, int limit, Sort sort) throws IOException { int numGroups = collapsedSet.size(); groups = new IntObjectOpenHashMap<>(numGroups * 2); collectors = new ArrayList<>(); DocIdSetIterator iterator = groupBits.iterator(); int group; while ((group = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { Collector collector = (sort == null) ? TopScoreDocCollector.create(limit, true) : TopFieldCollector.create(sort, limit, false, false, false, true); groups.put(group, collector); collectors.add(collector); } this.collapsedSet = collapsedSet; this.groupBits = groupBits; this.docValues = docValues; }
@Override public void preProcess(SearchContext context) { if (context.aggregations() != null) { List<Aggregator> collectors = new ArrayList<>(); Aggregator[] aggregators; try { AggregatorFactories factories = context.aggregations().factories(); aggregators = factories.createTopLevelAggregators(); for (int i = 0; i < aggregators.length; i++) { if (aggregators[i] instanceof GlobalAggregator == false) { collectors.add(aggregators[i]); } } context.aggregations().aggregators(aggregators); if (!collectors.isEmpty()) { Collector collector = BucketCollector.wrap(collectors); ((BucketCollector)collector).preCollection(); if (context.getProfilers() != null) { collector = new InternalProfileCollector(collector, CollectorResult.REASON_AGGREGATION, // TODO: report on child aggs as well Collections.emptyList()); } context.queryCollectors().put(AggregationPhase.class, collector); } } catch (IOException e) { throw new AggregationInitializationException("Could not initialize aggregators", e); } } }
@Override public boolean acceptsDocsOutOfOrder() { for (Collector c : collectors) { if (!c.acceptsDocsOutOfOrder()) { return false; } } return true; }
@Override public void preProcess(SearchContext context) { if (context.aggregations() != null) { AggregationContext aggregationContext = new AggregationContext(context); context.aggregations().aggregationContext(aggregationContext); List<Aggregator> collectors = new ArrayList<>(); Aggregator[] aggregators; try { AggregatorFactories factories = context.aggregations().factories(); aggregators = factories.createTopLevelAggregators(aggregationContext); for (int i = 0; i < aggregators.length; i++) { if (aggregators[i] instanceof GlobalAggregator == false) { collectors.add(aggregators[i]); } } context.aggregations().aggregators(aggregators); if (!collectors.isEmpty()) { Collector collector = BucketCollector.wrap(collectors); ((BucketCollector)collector).preCollection(); if (context.getProfilers() != null) { // TODO: report on child aggs as well List<InternalProfileCollector> emptyList = Collections.emptyList(); collector = new InternalProfileCollector(collector, CollectorResult.REASON_AGGREGATION, emptyList); } context.queryCollectors().put(AggregationPhase.class, collector); } } catch (IOException e) { throw new AggregationInitializationException("Could not initialize aggregators", e); } } }
private String getCommitIndexVersion(final IndexSearcher searcher, AnyObjectId commitId) throws IOException { final AtomicReference<String> indexVersion = new AtomicReference<>(null); searcher.search(COMMIT_HASH.query(commitId.getName()), new Collector() { private int docBase; @Override public void setScorer(Scorer scorer) throws IOException { } @Override public void collect(int doc) throws IOException { indexVersion.set(searcher.doc(docBase+doc).get(COMMIT_INDEX_VERSION.name())); } @Override public void setNextReader(AtomicReaderContext context) throws IOException { docBase = context.docBase; } @Override public boolean acceptsDocsOutOfOrder() { return true; } }); return indexVersion.get(); }
public LindenDocsCollector(Collector collector) { if (!(collector instanceof TopDocsCollector) && !(collector instanceof EarlyTerminationCollector)) { throw new RuntimeException("Unsupported collector class in LindenDocsCollector: " + collector.getClass().getName()); } hitCollector = collector; wrappedCollector = collector; }
@Override protected Collector createCollector() throws Exception { Collector collector = null; if (clnName.equalsIgnoreCase("topScoreDocOrdered") == true) { collector = TopScoreDocCollector.create(numHits(), true); } else if (clnName.equalsIgnoreCase("topScoreDocUnOrdered") == true) { collector = TopScoreDocCollector.create(numHits(), false); } else if (clnName.length() > 0){ collector = Class.forName(clnName).asSubclass(Collector.class).newInstance(); } else { collector = super.createCollector(); } return collector; }
DrillSidewaysScorer(AtomicReaderContext context, Scorer baseScorer, Collector drillDownCollector, DocsAndCost[] dims, boolean scoreSubDocsAtOnce) { this.dims = dims; this.context = context; this.baseScorer = baseScorer; this.drillDownCollector = drillDownCollector; this.scoreSubDocsAtOnce = scoreSubDocsAtOnce; }
/** Utility method, to search and also collect all hits * into the provided {@link Collector}. */ public static TopFieldDocs search(IndexSearcher searcher, Query q, Filter filter, int n, Sort sort, Collector fc) throws IOException { if (sort == null) { throw new IllegalArgumentException("sort must not be null"); } return (TopFieldDocs) doSearch(searcher, null, q, filter, n, sort, false, false, fc); }
/** Utility method, to search and also collect all hits * into the provided {@link Collector}. */ public static TopFieldDocs search(IndexSearcher searcher, Query q, Filter filter, int n, Sort sort, boolean doDocScores, boolean doMaxScore, Collector fc) throws IOException { if (sort == null) { throw new IllegalArgumentException("sort must not be null"); } return (TopFieldDocs) doSearch(searcher, null, q, filter, n, sort, doDocScores, doMaxScore, fc); }
/** Utility method, to search and also collect all hits * into the provided {@link Collector}. */ public static TopDocs searchAfter(IndexSearcher searcher, ScoreDoc after, Query q, Filter filter, int n, Sort sort, Collector fc) throws IOException { if (sort == null) { throw new IllegalArgumentException("sort must not be null"); } return doSearch(searcher, after, q, filter, n, sort, false, false, fc); }
/** Utility method, to search and also collect all hits * into the provided {@link Collector}. */ public static TopDocs searchAfter(IndexSearcher searcher, ScoreDoc after, Query q, Filter filter, int n, Sort sort, boolean doDocScores, boolean doMaxScore, Collector fc) throws IOException { if (sort == null) { throw new IllegalArgumentException("sort must not be null"); } return doSearch(searcher, after, q, filter, n, sort, doDocScores, doMaxScore, fc); }
DrillSidewaysQuery(Query baseQuery, Collector drillDownCollector, Collector[] drillSidewaysCollectors, Query[] drillDownQueries, boolean scoreSubDocsAtOnce) { this.baseQuery = baseQuery; this.drillDownCollector = drillDownCollector; this.drillSidewaysCollectors = drillSidewaysCollectors; this.drillDownQueries = drillDownQueries; this.scoreSubDocsAtOnce = scoreSubDocsAtOnce; }
/** * {@inheritDoc} */ @Override protected Collector createFirstPassCollector() throws IOException { // Ok we don't want groups, but do want a total count if (actualGroupsToFind <= 0) { fallBackCollector = new TotalHitCountCollector(); return fallBackCollector; } sort = sort == null ? Sort.RELEVANCE : sort; firstPass = new TermFirstPassGroupingCollector(groupBy, sort, actualGroupsToFind); return firstPass; }
/** * {@inheritDoc} */ @Override protected Collector createFirstPassCollector() throws IOException { DocSet groupFilt = searcher.getDocSet(query); topCollector = newCollector(groupSort, needScores); collector = new FilterCollector(groupFilt, topCollector); return collector; }
