void assertHits(Query q, float scores[]) throws Exception { ScoreDoc expected[] = new ScoreDoc[scores.length]; int expectedDocs[] = new int[scores.length]; for (int i = 0; i < expected.length; i++) { expectedDocs[i] = i; expected[i] = new ScoreDoc(i, scores[i]); } TopDocs docs = searcher.search(q, 2, new Sort(new SortField("id", SortField.Type.STRING))); /* for (int i=0;i<docs.scoreDocs.length;i++) { System.out.println(searcher.explain(q, docs.scoreDocs[i].doc)); } */ CheckHits.checkHits(random(), q, "", searcher, expectedDocs); CheckHits.checkHitsQuery(q, expected, docs.scoreDocs, expectedDocs); CheckHits.checkExplanations(q, "", searcher); }
/** scores[] are in docId order */ protected void checkValueSource(ValueSource vs, float scores[], float delta) throws IOException { FunctionQuery q = new FunctionQuery(vs); // //TODO is there any point to this check? // int expectedDocs[] = new int[scores.length];//fill with ascending 0....length-1 // for (int i = 0; i < expectedDocs.length; i++) { // expectedDocs[i] = i; // } // CheckHits.checkHits(random(), q, "", indexSearcher, expectedDocs); //TopDocs is sorted but we actually don't care about the order TopDocs docs = indexSearcher.search(q, 1000);//calculates the score for (int i = 0; i < docs.scoreDocs.length; i++) { ScoreDoc gotSD = docs.scoreDocs[i]; float expectedScore = scores[gotSD.doc]; assertEquals("Not equal for doc "+gotSD.doc, expectedScore, gotSD.score, delta); } CheckHits.checkExplanations(q, "", indexSearcher); }
/** tests the returned sort values are correct */ public void testSortValues() throws Exception { Expression expr = JavascriptCompiler.compile("sqrt(_score)"); SimpleBindings bindings = new SimpleBindings(); bindings.add(new SortField("_score", SortField.Type.SCORE)); Sort sort = new Sort(expr.getSortField(bindings, true)); Query query = new TermQuery(new Term("body", "contents")); TopFieldDocs td = searcher.search(query, null, 3, sort, true, true); for (int i = 0; i < 3; i++) { FieldDoc d = (FieldDoc) td.scoreDocs[i]; float expected = (float) Math.sqrt(d.score); float actual = ((Double)d.fields[0]).floatValue(); assertEquals(expected, actual, CheckHits.explainToleranceDelta(expected, actual)); } }
/** tests same binding used more than once in an expression */ public void testTwoOfSameBinding() throws Exception { Expression expr = JavascriptCompiler.compile("_score + _score"); SimpleBindings bindings = new SimpleBindings(); bindings.add(new SortField("_score", SortField.Type.SCORE)); Sort sort = new Sort(expr.getSortField(bindings, true)); Query query = new TermQuery(new Term("body", "contents")); TopFieldDocs td = searcher.search(query, null, 3, sort, true, true); for (int i = 0; i < 3; i++) { FieldDoc d = (FieldDoc) td.scoreDocs[i]; float expected = 2*d.score; float actual = ((Double)d.fields[0]).floatValue(); assertEquals(expected, actual, CheckHits.explainToleranceDelta(expected, actual)); } }
/** Uses variables with $ */ public void testDollarVariable() throws Exception { Expression expr = JavascriptCompiler.compile("$0+$score"); SimpleBindings bindings = new SimpleBindings(); bindings.add(new SortField("$0", SortField.Type.SCORE)); bindings.add(new SortField("$score", SortField.Type.SCORE)); Sort sort = new Sort(expr.getSortField(bindings, true)); Query query = new TermQuery(new Term("body", "contents")); TopFieldDocs td = searcher.search(query, null, 3, sort, true, true); for (int i = 0; i < 3; i++) { FieldDoc d = (FieldDoc) td.scoreDocs[i]; float expected = 2*d.score; float actual = ((Double)d.fields[0]).floatValue(); assertEquals(expected, actual, CheckHits.explainToleranceDelta(expected, actual)); } }
/** tests expression referring to another expression */ public void testExpressionRefersToExpression() throws Exception { Expression expr1 = JavascriptCompiler.compile("_score"); Expression expr2 = JavascriptCompiler.compile("2*expr1"); SimpleBindings bindings = new SimpleBindings(); bindings.add(new SortField("_score", SortField.Type.SCORE)); bindings.add("expr1", expr1); Sort sort = new Sort(expr2.getSortField(bindings, true)); Query query = new TermQuery(new Term("body", "contents")); TopFieldDocs td = searcher.search(query, null, 3, sort, true, true); for (int i = 0; i < 3; i++) { FieldDoc d = (FieldDoc) td.scoreDocs[i]; float expected = 2*d.score; float actual = ((Double)d.fields[0]).floatValue(); assertEquals(expected, actual, CheckHits.explainToleranceDelta(expected, actual)); } }
private void doTestLotsOfBindings(int n) throws Exception { SimpleBindings bindings = new SimpleBindings(); StringBuilder sb = new StringBuilder(); for (int i = 0; i < n; i++) { if (i > 0) { sb.append("+"); } sb.append("x" + i); bindings.add(new SortField("x" + i, SortField.Type.SCORE)); } Expression expr = JavascriptCompiler.compile(sb.toString()); Sort sort = new Sort(expr.getSortField(bindings, true)); Query query = new TermQuery(new Term("body", "contents")); TopFieldDocs td = searcher.search(query, null, 3, sort, true, true); for (int i = 0; i < 3; i++) { FieldDoc d = (FieldDoc) td.scoreDocs[i]; float expected = n*d.score; float actual = ((Double)d.fields[0]).floatValue(); assertEquals(expected, actual, CheckHits.explainToleranceDelta(expected, actual)); } }
public void test() throws IOException { PayloadTermQuery query = new PayloadTermQuery(new Term("field", "seventy"), new MaxPayloadFunction()); TopDocs hits = searcher.search(query, null, 100); assertTrue("hits is null and it shouldn't be", hits != null); assertTrue("hits Size: " + hits.totalHits + " is not: " + 100, hits.totalHits == 100); //they should all have the exact same score, because they all contain seventy once, and we set //all the other similarity factors to be 1 assertTrue(hits.getMaxScore() + " does not equal: " + 1, hits.getMaxScore() == 1); for (int i = 0; i < hits.scoreDocs.length; i++) { ScoreDoc doc = hits.scoreDocs[i]; assertTrue(doc.score + " does not equal: " + 1, doc.score == 1); } CheckHits.checkExplanations(query, PayloadHelper.FIELD, searcher, true); Spans spans = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), query); assertTrue("spans is null and it shouldn't be", spans != null); /*float score = hits.score(0); for (int i =1; i < hits.length(); i++) { assertTrue("scores are not equal and they should be", score == hits.score(i)); }*/ }
public void testNoPayload() throws Exception { PayloadTermQuery q1 = new PayloadTermQuery(new Term(PayloadHelper.NO_PAYLOAD_FIELD, "zero"), new MaxPayloadFunction()); PayloadTermQuery q2 = new PayloadTermQuery(new Term(PayloadHelper.NO_PAYLOAD_FIELD, "foo"), new MaxPayloadFunction()); BooleanClause c1 = new BooleanClause(q1, BooleanClause.Occur.MUST); BooleanClause c2 = new BooleanClause(q2, BooleanClause.Occur.MUST_NOT); BooleanQuery query = new BooleanQuery(); query.add(c1); query.add(c2); TopDocs hits = searcher.search(query, null, 100); assertTrue("hits is null and it shouldn't be", hits != null); assertTrue("hits Size: " + hits.totalHits + " is not: " + 1, hits.totalHits == 1); int[] results = new int[1]; results[0] = 0;//hits.scoreDocs[0].doc; CheckHits.checkHitCollector(random(), query, PayloadHelper.NO_PAYLOAD_FIELD, searcher, results); }
/** tests a pre-intersected automaton against the original */ public void testFiniteVersusInfinite() throws Exception { for (int i = 0; i < numIterations; i++) { String reg = AutomatonTestUtil.randomRegexp(random()); Automaton automaton = Operations.determinize(new RegExp(reg, RegExp.NONE).toAutomaton(), DEFAULT_MAX_DETERMINIZED_STATES); final List<BytesRef> matchedTerms = new ArrayList<>(); for(BytesRef t : terms) { if (Operations.run(automaton, t.utf8ToString())) { matchedTerms.add(t); } } Automaton alternate = Automata.makeStringUnion(matchedTerms); //System.out.println("match " + matchedTerms.size() + " " + alternate.getNumberOfStates() + " states, sigma=" + alternate.getStartPoints().length); //AutomatonTestUtil.minimizeSimple(alternate); //System.out.println("minimize done"); AutomatonQuery a1 = new AutomatonQuery(new Term("field", ""), automaton); AutomatonQuery a2 = new AutomatonQuery(new Term("field", ""), alternate, Integer.MAX_VALUE); ScoreDoc[] origHits = searcher.search(a1, 25).scoreDocs; ScoreDoc[] newHits = searcher.search(a2, 25).scoreDocs; CheckHits.checkEqual(a1, origHits, newHits); } }
/** scores[] are in docId order */ protected void checkValueSource(ValueSource vs, float scores[], float delta) throws IOException { FunctionQuery q = new FunctionQuery(vs); // //TODO is there any point to this check? // int expectedDocs[] = new int[scores.length];//fill with ascending 0....length-1 // for (int i = 0; i < expectedDocs.length; i++) { // expectedDocs[i] = i; // } // CheckHits.checkHits(random(), q, "", indexSearcher, expectedDocs); TopDocs docs = indexSearcher.search(q, 1000);//calculates the score for (int i = 0; i < docs.scoreDocs.length; i++) { ScoreDoc gotSD = docs.scoreDocs[i]; float expectedScore = scores[gotSD.doc]; assertEquals("Not equal for doc "+gotSD.doc, expectedScore, gotSD.score, delta); } CheckHits.checkExplanations(q, "", indexSearcher); }
/** tests a pre-intersected automaton against the original */ public void testFiniteVersusInfinite() throws Exception { for (int i = 0; i < numIterations; i++) { String reg = AutomatonTestUtil.randomRegexp(random()); Automaton automaton = new RegExp(reg, RegExp.NONE).toAutomaton(); final List<BytesRef> matchedTerms = new ArrayList<BytesRef>(); for(BytesRef t : terms) { if (BasicOperations.run(automaton, t.utf8ToString())) { matchedTerms.add(t); } } Automaton alternate = BasicAutomata.makeStringUnion(matchedTerms); //System.out.println("match " + matchedTerms.size() + " " + alternate.getNumberOfStates() + " states, sigma=" + alternate.getStartPoints().length); //AutomatonTestUtil.minimizeSimple(alternate); //System.out.println("minmize done"); AutomatonQuery a1 = new AutomatonQuery(new Term("field", ""), automaton); AutomatonQuery a2 = new AutomatonQuery(new Term("field", ""), alternate); CheckHits.checkEqual(a1, searcher.search(a1, 25).scoreDocs, searcher.search(a2, 25).scoreDocs); } }
void assertHits(Query q, float scores[]) throws Exception { ScoreDoc expected[] = new ScoreDoc[scores.length]; int expectedDocs[] = new int[scores.length]; for (int i = 0; i < expected.length; i++) { expectedDocs[i] = i; expected[i] = new ScoreDoc(i, scores[i]); } TopDocs docs = is.search(q, 10, new Sort(new SortField("id", SortField.Type.STRING))); CheckHits.checkHits(random(), q, "", is, expectedDocs); CheckHits.checkHitsQuery(q, expected, docs.scoreDocs, expectedDocs); CheckHits.checkExplanations(q, "", is); }
void assertHits(Query q, float scores[]) throws Exception { ScoreDoc expected[] = new ScoreDoc[scores.length]; int expectedDocs[] = new int[scores.length]; for (int i = 0; i < expected.length; i++) { expectedDocs[i] = i; expected[i] = new ScoreDoc(i, scores[i]); } TopDocs docs = searcher.search(q, null, documents.size(), new Sort(new SortField("id", SortField.Type.STRING)), true, false); CheckHits.checkHits(random(), q, "", searcher, expectedDocs); CheckHits.checkHitsQuery(q, expected, docs.scoreDocs, expectedDocs); CheckHits.checkExplanations(q, "", searcher); }
void assertQuery(Query query, Filter filter, Sort sort) throws Exception { int size = TestUtil.nextInt(random(), 1, searcher.getIndexReader().maxDoc() / 5); TopDocs expected = searcher.search(query, filter, size, sort, random().nextBoolean(), random().nextBoolean()); // make our actual sort, mutating original by replacing some of the // sortfields with equivalent expressions SortField original[] = sort.getSort(); SortField mutated[] = new SortField[original.length]; for (int i = 0; i < mutated.length; i++) { if (random().nextInt(3) > 0) { SortField s = original[i]; Expression expr = JavascriptCompiler.compile(s.getField()); SimpleBindings simpleBindings = new SimpleBindings(); simpleBindings.add(s); boolean reverse = s.getType() == SortField.Type.SCORE || s.getReverse(); mutated[i] = expr.getSortField(simpleBindings, reverse); } else { mutated[i] = original[i]; } } Sort mutatedSort = new Sort(mutated); TopDocs actual = searcher.search(query, filter, size, mutatedSort, random().nextBoolean(), random().nextBoolean()); CheckHits.checkEqual(query, expected.scoreDocs, actual.scoreDocs); if (size < actual.totalHits) { expected = searcher.searchAfter(expected.scoreDocs[size-1], query, filter, size, sort); actual = searcher.searchAfter(actual.scoreDocs[size-1], query, filter, size, mutatedSort); CheckHits.checkEqual(query, expected.scoreDocs, actual.scoreDocs); } }
public void testMultipleMatchesPerDoc() throws Exception { PayloadTermQuery query = new PayloadTermQuery(new Term(PayloadHelper.MULTI_FIELD, "seventy"), new MaxPayloadFunction()); TopDocs hits = searcher.search(query, null, 100); assertTrue("hits is null and it shouldn't be", hits != null); assertTrue("hits Size: " + hits.totalHits + " is not: " + 100, hits.totalHits == 100); //they should all have the exact same score, because they all contain seventy once, and we set //all the other similarity factors to be 1 //System.out.println("Hash: " + seventyHash + " Twice Hash: " + 2*seventyHash); assertTrue(hits.getMaxScore() + " does not equal: " + 4.0, hits.getMaxScore() == 4.0); //there should be exactly 10 items that score a 4, all the rest should score a 2 //The 10 items are: 70 + i*100 where i in [0-9] int numTens = 0; for (int i = 0; i < hits.scoreDocs.length; i++) { ScoreDoc doc = hits.scoreDocs[i]; if (doc.doc % 10 == 0) { numTens++; assertTrue(doc.score + " does not equal: " + 4.0, doc.score == 4.0); } else { assertTrue(doc.score + " does not equal: " + 2, doc.score == 2); } } assertTrue(numTens + " does not equal: " + 10, numTens == 10); CheckHits.checkExplanations(query, "field", searcher, true); Spans spans = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), query); assertTrue("spans is null and it shouldn't be", spans != null); //should be two matches per document int count = 0; //100 hits times 2 matches per hit, we should have 200 in count while (spans.next()) { count++; } assertTrue(count + " does not equal: " + 200, count == 200); }
public void testIgnoreSpanScorer() throws Exception { PayloadTermQuery query = new PayloadTermQuery(new Term(PayloadHelper.MULTI_FIELD, "seventy"), new MaxPayloadFunction(), false); IndexReader reader = DirectoryReader.open(directory); IndexSearcher theSearcher = newSearcher(reader); theSearcher.setSimilarity(new FullSimilarity()); TopDocs hits = searcher.search(query, null, 100); assertTrue("hits is null and it shouldn't be", hits != null); assertTrue("hits Size: " + hits.totalHits + " is not: " + 100, hits.totalHits == 100); //they should all have the exact same score, because they all contain seventy once, and we set //all the other similarity factors to be 1 //System.out.println("Hash: " + seventyHash + " Twice Hash: " + 2*seventyHash); assertTrue(hits.getMaxScore() + " does not equal: " + 4.0, hits.getMaxScore() == 4.0); //there should be exactly 10 items that score a 4, all the rest should score a 2 //The 10 items are: 70 + i*100 where i in [0-9] int numTens = 0; for (int i = 0; i < hits.scoreDocs.length; i++) { ScoreDoc doc = hits.scoreDocs[i]; if (doc.doc % 10 == 0) { numTens++; assertTrue(doc.score + " does not equal: " + 4.0, doc.score == 4.0); } else { assertTrue(doc.score + " does not equal: " + 2, doc.score == 2); } } assertTrue(numTens + " does not equal: " + 10, numTens == 10); CheckHits.checkExplanations(query, "field", searcher, true); Spans spans = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), query); assertTrue("spans is null and it shouldn't be", spans != null); //should be two matches per document int count = 0; //100 hits times 2 matches per hit, we should have 200 in count while (spans.next()) { count++; } reader.close(); }
void assertHits(Query q, float scores[]) throws Exception { ScoreDoc expected[] = new ScoreDoc[scores.length]; int expectedDocs[] = new int[scores.length]; for (int i = 0; i < expected.length; i++) { expectedDocs[i] = i; expected[i] = new ScoreDoc(i, scores[i]); } TopDocs docs = searcher.search(q, documents.size(), new Sort(new SortField("id", SortField.Type.STRING))); CheckHits.checkHits(random(), q, "", searcher, expectedDocs); CheckHits.checkHitsQuery(q, expected, docs.scoreDocs, expectedDocs); CheckHits.checkExplanations(q, "", searcher); }
public void testIgnoreSpanScorer() throws Exception { PayloadTermQuery query = new PayloadTermQuery(new Term(PayloadHelper.MULTI_FIELD, "seventy"), new MaxPayloadFunction(), false); IndexReader reader = DirectoryReader.open(directory); IndexSearcher theSearcher = new IndexSearcher(reader); theSearcher.setSimilarity(new FullSimilarity()); TopDocs hits = searcher.search(query, null, 100); assertTrue("hits is null and it shouldn't be", hits != null); assertTrue("hits Size: " + hits.totalHits + " is not: " + 100, hits.totalHits == 100); //they should all have the exact same score, because they all contain seventy once, and we set //all the other similarity factors to be 1 //System.out.println("Hash: " + seventyHash + " Twice Hash: " + 2*seventyHash); assertTrue(hits.getMaxScore() + " does not equal: " + 4.0, hits.getMaxScore() == 4.0); //there should be exactly 10 items that score a 4, all the rest should score a 2 //The 10 items are: 70 + i*100 where i in [0-9] int numTens = 0; for (int i = 0; i < hits.scoreDocs.length; i++) { ScoreDoc doc = hits.scoreDocs[i]; if (doc.doc % 10 == 0) { numTens++; assertTrue(doc.score + " does not equal: " + 4.0, doc.score == 4.0); } else { assertTrue(doc.score + " does not equal: " + 2, doc.score == 2); } } assertTrue(numTens + " does not equal: " + 10, numTens == 10); CheckHits.checkExplanations(query, "field", searcher, true); Spans spans = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), query); assertTrue("spans is null and it shouldn't be", spans != null); //should be two matches per document int count = 0; //100 hits times 2 matches per hit, we should have 200 in count while (spans.next()) { count++; } reader.close(); }
void assertQuery(Query query, Filter filter, Sort sort) throws Exception { int size = _TestUtil.nextInt(random(), 1, searcher.getIndexReader().maxDoc()/5); TopDocs expected = searcher.search(query, filter, size, sort, random().nextBoolean(), random().nextBoolean()); // make our actual sort, mutating original by replacing some of the // sortfields with equivalent expressions SortField original[] = sort.getSort(); SortField mutated[] = new SortField[original.length]; for (int i = 0; i < mutated.length; i++) { if (random().nextInt(3) > 0) { SortField s = original[i]; Expression expr = JavascriptCompiler.compile(s.getField()); SimpleBindings simpleBindings = new SimpleBindings(); simpleBindings.add(s); boolean reverse = s.getType() == SortField.Type.SCORE || s.getReverse(); mutated[i] = expr.getSortField(simpleBindings, reverse); } else { mutated[i] = original[i]; } } Sort mutatedSort = new Sort(mutated); TopDocs actual = searcher.search(query, filter, size, mutatedSort, random().nextBoolean(), random().nextBoolean()); CheckHits.checkEqual(query, expected.scoreDocs, actual.scoreDocs); if (size < actual.totalHits) { expected = searcher.searchAfter(expected.scoreDocs[size-1], query, filter, size, sort); actual = searcher.searchAfter(actual.scoreDocs[size-1], query, filter, size, mutatedSort); CheckHits.checkEqual(query, expected.scoreDocs, actual.scoreDocs); } }
private void verifyResults(float boost, IndexSearcher s, Map<Integer,Float> h1, Map<Integer,Float> h2customNeutral, Map<Integer,Float> h3CustomMul, Map<Integer,Float> h4CustomAdd, Map<Integer,Float> h5CustomMulAdd, Query q1, Query q2, Query q3, Query q4, Query q5) throws Exception { // verify numbers of matches log("#hits = "+h1.size()); assertEquals("queries should have same #hits",h1.size(),h2customNeutral.size()); assertEquals("queries should have same #hits",h1.size(),h3CustomMul.size()); assertEquals("queries should have same #hits",h1.size(),h4CustomAdd.size()); assertEquals("queries should have same #hits",h1.size(),h5CustomMulAdd.size()); QueryUtils.check(random(), q1, s, rarely()); QueryUtils.check(random(), q2, s, rarely()); QueryUtils.check(random(), q3, s, rarely()); QueryUtils.check(random(), q4, s, rarely()); QueryUtils.check(random(), q5, s, rarely()); // verify scores ratios for (final Integer doc : h1.keySet()) { log("doc = "+doc); float fieldScore = expectedFieldScore(s.getIndexReader().document(doc).get(ID_FIELD)); log("fieldScore = " + fieldScore); assertTrue("fieldScore should not be 0", fieldScore > 0); float score1 = h1.get(doc); logResult("score1=", s, q1, doc, score1); float score2 = h2customNeutral.get(doc); logResult("score2=", s, q2, doc, score2); assertEquals("same score (just boosted) for neutral", boost * score1, score2, CheckHits.explainToleranceDelta(boost * score1, score2)); float score3 = h3CustomMul.get(doc); logResult("score3=", s, q3, doc, score3); assertEquals("new score for custom mul", boost * fieldScore * score1, score3, CheckHits.explainToleranceDelta(boost * fieldScore * score1, score3)); float score4 = h4CustomAdd.get(doc); logResult("score4=", s, q4, doc, score4); assertEquals("new score for custom add", boost * (fieldScore + score1), score4, CheckHits.explainToleranceDelta(boost * (fieldScore + score1), score4)); float score5 = h5CustomMulAdd.get(doc); logResult("score5=", s, q5, doc, score5); assertEquals("new score for custom mul add", boost * fieldScore * (score1 + fieldScore), score5, CheckHits.explainToleranceDelta(boost * fieldScore * (score1 + fieldScore), score5)); } }
private void checkHits(Query query, int[] results) throws IOException { CheckHits.checkHits(random(), query, field, searcher, results); }
private void checkHits(Query query, int[] results) throws IOException { CheckHits.checkHits(random(), query, "field", searcher, results); }
public void testSpanNearQuery() throws Exception { SpanNearQuery q = makeQuery(); CheckHits.checkHits(random(), q, FIELD, searcher, new int[] {0,1}); }