/** blast some random strings through differently configured tokenizers */ public void testRandomRegexps() throws Exception { int iters = atLeast(30); for (int i = 0; i < iters; i++) { final CharacterRunAutomaton dfa = new CharacterRunAutomaton(AutomatonTestUtil.randomAutomaton(random()), Integer.MAX_VALUE); final boolean lowercase = random().nextBoolean(); final int limit = TestUtil.nextInt(random(), 0, 500); Analyzer a = new Analyzer() { @Override protected TokenStreamComponents createComponents(String fieldName, Reader reader) { Tokenizer t = new MockTokenizer(reader, dfa, lowercase, limit); return new TokenStreamComponents(t, t); } }; checkRandomData(random(), a, 100); a.close(); } }
/** blast some random strings through differently configured tokenizers */ public void testRandomRegexps() throws Exception { int iters = atLeast(30); for (int i = 0; i < iters; i++) { final CharacterRunAutomaton dfa = new CharacterRunAutomaton(AutomatonTestUtil.randomAutomaton(random())); final boolean lowercase = random().nextBoolean(); final int limit = _TestUtil.nextInt(random(), 0, 500); Analyzer a = new Analyzer() { @Override protected TokenStreamComponents createComponents(String fieldName, Reader reader) { Tokenizer t = new MockTokenizer(reader, dfa, lowercase, limit); return new TokenStreamComponents(t, t); } }; checkRandomData(random(), a, 100); a.close(); } }
/** * Terms api equivalency */ public void assertTermsEquals(String info, IndexReader leftReader, Terms leftTerms, Terms rightTerms, boolean deep) throws IOException { if (leftTerms == null || rightTerms == null) { assertNull(info, leftTerms); assertNull(info, rightTerms); return; } assertTermsStatisticsEquals(info, leftTerms, rightTerms); assertEquals(leftTerms.hasOffsets(), rightTerms.hasOffsets()); assertEquals(leftTerms.hasPositions(), rightTerms.hasPositions()); assertEquals(leftTerms.hasPayloads(), rightTerms.hasPayloads()); TermsEnum leftTermsEnum = leftTerms.iterator(null); TermsEnum rightTermsEnum = rightTerms.iterator(null); assertTermsEnumEquals(info, leftReader, leftTermsEnum, rightTermsEnum, true); assertTermsSeekingEquals(info, leftTerms, rightTerms); if (deep) { int numIntersections = atLeast(3); for (int i = 0; i < numIntersections; i++) { String re = AutomatonTestUtil.randomRegexp(random()); CompiledAutomaton automaton = new CompiledAutomaton(new RegExp(re, RegExp.NONE).toAutomaton()); if (automaton.type == CompiledAutomaton.AUTOMATON_TYPE.NORMAL) { // TODO: test start term too TermsEnum leftIntersection = leftTerms.intersect(automaton, null); TermsEnum rightIntersection = rightTerms.intersect(automaton, null); assertTermsEnumEquals(info, leftReader, leftIntersection, rightIntersection, rarely()); } } } }
public void assertTerms(Terms leftTerms, Terms rightTerms, boolean deep) throws Exception { if (leftTerms == null || rightTerms == null) { assertNull(leftTerms); assertNull(rightTerms); return; } assertTermsStatistics(leftTerms, rightTerms); // NOTE: we don't assert hasOffsets/hasPositions/hasPayloads because they are allowed to be different TermsEnum leftTermsEnum = leftTerms.iterator(null); TermsEnum rightTermsEnum = rightTerms.iterator(null); assertTermsEnum(leftTermsEnum, rightTermsEnum, true); assertTermsSeeking(leftTerms, rightTerms); if (deep) { int numIntersections = atLeast(3); for (int i = 0; i < numIntersections; i++) { String re = AutomatonTestUtil.randomRegexp(random()); CompiledAutomaton automaton = new CompiledAutomaton(new RegExp(re, RegExp.NONE).toAutomaton()); if (automaton.type == CompiledAutomaton.AUTOMATON_TYPE.NORMAL) { // TODO: test start term too TermsEnum leftIntersection = leftTerms.intersect(automaton, null); TermsEnum rightIntersection = rightTerms.intersect(automaton, null); assertTermsEnum(leftIntersection, rightIntersection, rarely()); } } } }
/** test a bunch of random regular expressions */ public void testRegexps() throws Exception { int num = atLeast(1000); for (int i = 0; i < num; i++) { String reg = AutomatonTestUtil.randomRegexp(random()); if (VERBOSE) { System.out.println("TEST: regexp=" + reg); } assertSame(reg); } }
/** test a bunch of random regular expressions */ public void testRegexps() throws Exception { // we generate aweful regexps: good for testing. // but for preflex codec, the test can be very slow, so use less iterations. int num = Codec.getDefault().getName().equals("Lucene3x") ? 100 * RANDOM_MULTIPLIER : atLeast(1000); for (int i = 0; i < num; i++) { String reg = AutomatonTestUtil.randomRegexp(random()); if (VERBOSE) { System.out.println("TEST: regexp=" + reg); } assertSame(reg); } }
/** * Terms api equivalency */ public void assertTerms(Terms leftTerms, Terms rightTerms, boolean deep) throws Exception { if (leftTerms == null || rightTerms == null) { assertNull(info, leftTerms); assertNull(info, rightTerms); return; } assertTermsStatistics(leftTerms, rightTerms); assertEquals(leftTerms.hasOffsets(), rightTerms.hasOffsets()); assertEquals(leftTerms.hasPositions(), rightTerms.hasPositions()); assertEquals(leftTerms.hasPayloads(), rightTerms.hasPayloads()); TermsEnum leftTermsEnum = leftTerms.iterator(null); TermsEnum rightTermsEnum = rightTerms.iterator(null); assertTermsEnum(leftTermsEnum, rightTermsEnum, true); assertTermsSeeking(leftTerms, rightTerms); if (deep) { int numIntersections = atLeast(3); for (int i = 0; i < numIntersections; i++) { String re = AutomatonTestUtil.randomRegexp(random()); CompiledAutomaton automaton = new CompiledAutomaton(new RegExp(re, RegExp.NONE).toAutomaton()); if (automaton.type == CompiledAutomaton.AUTOMATON_TYPE.NORMAL) { // TODO: test start term too TermsEnum leftIntersection = leftTerms.intersect(automaton, null); TermsEnum rightIntersection = rightTerms.intersect(automaton, null); assertTermsEnum(leftIntersection, rightIntersection, rarely()); } } } }