public void testDocsWithField() throws Exception { FieldCache cache = FieldCache.DEFAULT; cache.purgeAllCaches(); assertEquals(0, cache.getCacheEntries().length); cache.getDoubles(reader, "theDouble", true); // The double[] takes two slots (one w/ null parser, one // w/ real parser), and docsWithField should also // have been populated: assertEquals(3, cache.getCacheEntries().length); Bits bits = cache.getDocsWithField(reader, "theDouble"); // No new entries should appear: assertEquals(3, cache.getCacheEntries().length); assertTrue(bits instanceof Bits.MatchAllBits); FieldCache.Ints ints = cache.getInts(reader, "sparse", true); assertEquals(6, cache.getCacheEntries().length); Bits docsWithField = cache.getDocsWithField(reader, "sparse"); assertEquals(6, cache.getCacheEntries().length); for (int i = 0; i < docsWithField.length(); i++) { if (i%2 == 0) { assertTrue(docsWithField.get(i)); assertEquals(i, ints.get(i)); } else { assertFalse(docsWithField.get(i)); } } FieldCache.Ints numInts = cache.getInts(reader, "numInt", random().nextBoolean()); docsWithField = cache.getDocsWithField(reader, "numInt"); for (int i = 0; i < docsWithField.length(); i++) { if (i%2 == 0) { assertTrue(docsWithField.get(i)); assertEquals(i, numInts.get(i)); } else { assertFalse(docsWithField.get(i)); } } }
public void testGetDocsWithFieldThreadSafety() throws Exception { final FieldCache cache = FieldCache.DEFAULT; cache.purgeAllCaches(); int NUM_THREADS = 3; Thread[] threads = new Thread[NUM_THREADS]; final AtomicBoolean failed = new AtomicBoolean(); final AtomicInteger iters = new AtomicInteger(); final int NUM_ITER = 200 * RANDOM_MULTIPLIER; final CyclicBarrier restart = new CyclicBarrier(NUM_THREADS, new Runnable() { @Override public void run() { cache.purgeAllCaches(); iters.incrementAndGet(); } }); for(int threadIDX=0;threadIDX<NUM_THREADS;threadIDX++) { threads[threadIDX] = new Thread() { @Override public void run() { try { while(!failed.get()) { final int op = random().nextInt(3); if (op == 0) { // Purge all caches & resume, once all // threads get here: restart.await(); if (iters.get() >= NUM_ITER) { break; } } else if (op == 1) { Bits docsWithField = cache.getDocsWithField(reader, "sparse"); for (int i = 0; i < docsWithField.length(); i++) { assertEquals(i%2 == 0, docsWithField.get(i)); } } else { FieldCache.Ints ints = cache.getInts(reader, "sparse", true); Bits docsWithField = cache.getDocsWithField(reader, "sparse"); for (int i = 0; i < docsWithField.length(); i++) { if (i%2 == 0) { assertTrue(docsWithField.get(i)); assertEquals(i, ints.get(i)); } else { assertFalse(docsWithField.get(i)); } } } } } catch (Throwable t) { failed.set(true); restart.reset(); throw new RuntimeException(t); } } }; threads[threadIDX].start(); } for(int threadIDX=0;threadIDX<NUM_THREADS;threadIDX++) { threads[threadIDX].join(); } assertFalse(failed.get()); }
public void testNonexistantFields() throws Exception { Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir); Document doc = new Document(); iw.addDocument(doc); DirectoryReader ir = iw.getReader(); iw.close(); AtomicReader ar = getOnlySegmentReader(ir); final FieldCache cache = FieldCache.DEFAULT; cache.purgeAllCaches(); assertEquals(0, cache.getCacheEntries().length); Bytes bytes = cache.getBytes(ar, "bogusbytes", true); assertEquals(0, bytes.get(0)); Shorts shorts = cache.getShorts(ar, "bogusshorts", true); assertEquals(0, shorts.get(0)); Ints ints = cache.getInts(ar, "bogusints", true); assertEquals(0, ints.get(0)); Longs longs = cache.getLongs(ar, "boguslongs", true); assertEquals(0, longs.get(0)); Floats floats = cache.getFloats(ar, "bogusfloats", true); assertEquals(0, floats.get(0), 0.0f); Doubles doubles = cache.getDoubles(ar, "bogusdoubles", true); assertEquals(0, doubles.get(0), 0.0D); BinaryDocValues binaries = cache.getTerms(ar, "bogusterms", true); BytesRef scratch = binaries.get(0); assertEquals(0, scratch.length); SortedDocValues sorted = cache.getTermsIndex(ar, "bogustermsindex"); assertEquals(-1, sorted.getOrd(0)); scratch = sorted.get(0); assertEquals(0, scratch.length); SortedSetDocValues sortedSet = cache.getDocTermOrds(ar, "bogusmultivalued"); sortedSet.setDocument(0); assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd()); Bits bits = cache.getDocsWithField(ar, "bogusbits"); assertFalse(bits.get(0)); // check that we cached nothing assertEquals(0, cache.getCacheEntries().length); ir.close(); dir.close(); }
public void testNonIndexedFields() throws Exception { Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir); Document doc = new Document(); doc.add(new StoredField("bogusbytes", "bogus")); doc.add(new StoredField("bogusshorts", "bogus")); doc.add(new StoredField("bogusints", "bogus")); doc.add(new StoredField("boguslongs", "bogus")); doc.add(new StoredField("bogusfloats", "bogus")); doc.add(new StoredField("bogusdoubles", "bogus")); doc.add(new StoredField("bogusterms", "bogus")); doc.add(new StoredField("bogustermsindex", "bogus")); doc.add(new StoredField("bogusmultivalued", "bogus")); doc.add(new StoredField("bogusbits", "bogus")); iw.addDocument(doc); DirectoryReader ir = iw.getReader(); iw.close(); AtomicReader ar = getOnlySegmentReader(ir); final FieldCache cache = FieldCache.DEFAULT; cache.purgeAllCaches(); assertEquals(0, cache.getCacheEntries().length); Bytes bytes = cache.getBytes(ar, "bogusbytes", true); assertEquals(0, bytes.get(0)); Shorts shorts = cache.getShorts(ar, "bogusshorts", true); assertEquals(0, shorts.get(0)); Ints ints = cache.getInts(ar, "bogusints", true); assertEquals(0, ints.get(0)); Longs longs = cache.getLongs(ar, "boguslongs", true); assertEquals(0, longs.get(0)); Floats floats = cache.getFloats(ar, "bogusfloats", true); assertEquals(0, floats.get(0), 0.0f); Doubles doubles = cache.getDoubles(ar, "bogusdoubles", true); assertEquals(0, doubles.get(0), 0.0D); BinaryDocValues binaries = cache.getTerms(ar, "bogusterms", true); BytesRef scratch = binaries.get(0); assertEquals(0, scratch.length); SortedDocValues sorted = cache.getTermsIndex(ar, "bogustermsindex"); assertEquals(-1, sorted.getOrd(0)); scratch = sorted.get(0); assertEquals(0, scratch.length); SortedSetDocValues sortedSet = cache.getDocTermOrds(ar, "bogusmultivalued"); sortedSet.setDocument(0); assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd()); Bits bits = cache.getDocsWithField(ar, "bogusbits"); assertFalse(bits.get(0)); // check that we cached nothing assertEquals(0, cache.getCacheEntries().length); ir.close(); dir.close(); }
public void testIntFieldCache() throws IOException { Directory dir = newDirectory(); IndexWriterConfig cfg = newIndexWriterConfig(new MockAnalyzer(random())); cfg.setMergePolicy(newLogMergePolicy()); RandomIndexWriter iw = new RandomIndexWriter(random(), dir, cfg); Document doc = new Document(); IntField field = new IntField("f", 0, Store.YES); doc.add(field); final int[] values = new int[TestUtil.nextInt(random(), 1, 10)]; for (int i = 0; i < values.length; ++i) { final int v; switch (random().nextInt(10)) { case 0: v = Integer.MIN_VALUE; break; case 1: v = 0; break; case 2: v = Integer.MAX_VALUE; break; default: v = TestUtil.nextInt(random(), -10, 10); break; } values[i] = v; if (v == 0 && random().nextBoolean()) { // missing iw.addDocument(new Document()); } else { field.setIntValue(v); iw.addDocument(doc); } } iw.forceMerge(1); final DirectoryReader reader = iw.getReader(); final FieldCache.Ints ints = FieldCache.DEFAULT.getInts(getOnlySegmentReader(reader), "f", false); for (int i = 0; i < values.length; ++i) { assertEquals(values[i], ints.get(i)); } reader.close(); iw.close(); dir.close(); }
public void testNonexistantFields() throws Exception { Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir); Document doc = new Document(); iw.addDocument(doc); DirectoryReader ir = iw.getReader(); iw.close(); AtomicReader ar = getOnlySegmentReader(ir); final FieldCache cache = FieldCache.DEFAULT; cache.purgeAllCaches(); assertEquals(0, cache.getCacheEntries().length); Bytes bytes = cache.getBytes(ar, "bogusbytes", true); assertEquals(0, bytes.get(0)); Shorts shorts = cache.getShorts(ar, "bogusshorts", true); assertEquals(0, shorts.get(0)); Ints ints = cache.getInts(ar, "bogusints", true); assertEquals(0, ints.get(0)); Longs longs = cache.getLongs(ar, "boguslongs", true); assertEquals(0, longs.get(0)); Floats floats = cache.getFloats(ar, "bogusfloats", true); assertEquals(0, floats.get(0), 0.0f); Doubles doubles = cache.getDoubles(ar, "bogusdoubles", true); assertEquals(0, doubles.get(0), 0.0D); BytesRef scratch = new BytesRef(); BinaryDocValues binaries = cache.getTerms(ar, "bogusterms"); binaries.get(0, scratch); assertTrue(scratch.bytes == BinaryDocValues.MISSING); SortedDocValues sorted = cache.getTermsIndex(ar, "bogustermsindex"); assertEquals(-1, sorted.getOrd(0)); sorted.get(0, scratch); assertTrue(scratch.bytes == BinaryDocValues.MISSING); SortedSetDocValues sortedSet = cache.getDocTermOrds(ar, "bogusmultivalued"); sortedSet.setDocument(0); assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd()); Bits bits = cache.getDocsWithField(ar, "bogusbits"); assertFalse(bits.get(0)); // check that we cached nothing assertEquals(0, cache.getCacheEntries().length); ir.close(); dir.close(); }
public void testNonIndexedFields() throws Exception { Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir); Document doc = new Document(); doc.add(new StoredField("bogusbytes", "bogus")); doc.add(new StoredField("bogusshorts", "bogus")); doc.add(new StoredField("bogusints", "bogus")); doc.add(new StoredField("boguslongs", "bogus")); doc.add(new StoredField("bogusfloats", "bogus")); doc.add(new StoredField("bogusdoubles", "bogus")); doc.add(new StoredField("bogusterms", "bogus")); doc.add(new StoredField("bogustermsindex", "bogus")); doc.add(new StoredField("bogusmultivalued", "bogus")); doc.add(new StoredField("bogusbits", "bogus")); iw.addDocument(doc); DirectoryReader ir = iw.getReader(); iw.close(); AtomicReader ar = getOnlySegmentReader(ir); final FieldCache cache = FieldCache.DEFAULT; cache.purgeAllCaches(); assertEquals(0, cache.getCacheEntries().length); Bytes bytes = cache.getBytes(ar, "bogusbytes", true); assertEquals(0, bytes.get(0)); Shorts shorts = cache.getShorts(ar, "bogusshorts", true); assertEquals(0, shorts.get(0)); Ints ints = cache.getInts(ar, "bogusints", true); assertEquals(0, ints.get(0)); Longs longs = cache.getLongs(ar, "boguslongs", true); assertEquals(0, longs.get(0)); Floats floats = cache.getFloats(ar, "bogusfloats", true); assertEquals(0, floats.get(0), 0.0f); Doubles doubles = cache.getDoubles(ar, "bogusdoubles", true); assertEquals(0, doubles.get(0), 0.0D); BytesRef scratch = new BytesRef(); BinaryDocValues binaries = cache.getTerms(ar, "bogusterms"); binaries.get(0, scratch); assertTrue(scratch.bytes == BinaryDocValues.MISSING); SortedDocValues sorted = cache.getTermsIndex(ar, "bogustermsindex"); assertEquals(-1, sorted.getOrd(0)); sorted.get(0, scratch); assertTrue(scratch.bytes == BinaryDocValues.MISSING); SortedSetDocValues sortedSet = cache.getDocTermOrds(ar, "bogusmultivalued"); sortedSet.setDocument(0); assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd()); Bits bits = cache.getDocsWithField(ar, "bogusbits"); assertFalse(bits.get(0)); // check that we cached nothing assertEquals(0, cache.getCacheEntries().length); ir.close(); dir.close(); }
public void testNonexistantFields() throws Exception { Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir); Document doc = new Document(); iw.addDocument(doc); DirectoryReader ir = iw.getReader(); iw.close(); AtomicReader ar = getOnlySegmentReader(ir); final FieldCache cache = FieldCache.DEFAULT; cache.purgeAllCaches(); assertEquals(0, cache.getCacheEntries().length); Bytes bytes = cache.getBytes(ar, "bogusbytes", true); assertEquals(0, bytes.get(0)); Shorts shorts = cache.getShorts(ar, "bogusshorts", true); assertEquals(0, shorts.get(0)); Ints ints = cache.getInts(ar, "bogusints", true); assertEquals(0, ints.get(0)); Longs longs = cache.getLongs(ar, "boguslongs", true); assertEquals(0, longs.get(0)); Floats floats = cache.getFloats(ar, "bogusfloats", true); assertEquals(0, floats.get(0), 0.0f); Doubles doubles = cache.getDoubles(ar, "bogusdoubles", true); assertEquals(0, doubles.get(0), 0.0D); BytesRef scratch = new BytesRef(); BinaryDocValues binaries = cache.getTerms(ar, "bogusterms", true); binaries.get(0, scratch); assertEquals(0, scratch.length); SortedDocValues sorted = cache.getTermsIndex(ar, "bogustermsindex"); assertEquals(-1, sorted.getOrd(0)); sorted.get(0, scratch); assertEquals(0, scratch.length); SortedSetDocValues sortedSet = cache.getDocTermOrds(ar, "bogusmultivalued"); sortedSet.setDocument(0); assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd()); Bits bits = cache.getDocsWithField(ar, "bogusbits"); assertFalse(bits.get(0)); // check that we cached nothing assertEquals(0, cache.getCacheEntries().length); ir.close(); dir.close(); }
public void testNonIndexedFields() throws Exception { Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir); Document doc = new Document(); doc.add(new StoredField("bogusbytes", "bogus")); doc.add(new StoredField("bogusshorts", "bogus")); doc.add(new StoredField("bogusints", "bogus")); doc.add(new StoredField("boguslongs", "bogus")); doc.add(new StoredField("bogusfloats", "bogus")); doc.add(new StoredField("bogusdoubles", "bogus")); doc.add(new StoredField("bogusterms", "bogus")); doc.add(new StoredField("bogustermsindex", "bogus")); doc.add(new StoredField("bogusmultivalued", "bogus")); doc.add(new StoredField("bogusbits", "bogus")); iw.addDocument(doc); DirectoryReader ir = iw.getReader(); iw.close(); AtomicReader ar = getOnlySegmentReader(ir); final FieldCache cache = FieldCache.DEFAULT; cache.purgeAllCaches(); assertEquals(0, cache.getCacheEntries().length); Bytes bytes = cache.getBytes(ar, "bogusbytes", true); assertEquals(0, bytes.get(0)); Shorts shorts = cache.getShorts(ar, "bogusshorts", true); assertEquals(0, shorts.get(0)); Ints ints = cache.getInts(ar, "bogusints", true); assertEquals(0, ints.get(0)); Longs longs = cache.getLongs(ar, "boguslongs", true); assertEquals(0, longs.get(0)); Floats floats = cache.getFloats(ar, "bogusfloats", true); assertEquals(0, floats.get(0), 0.0f); Doubles doubles = cache.getDoubles(ar, "bogusdoubles", true); assertEquals(0, doubles.get(0), 0.0D); BytesRef scratch = new BytesRef(); BinaryDocValues binaries = cache.getTerms(ar, "bogusterms", true); binaries.get(0, scratch); assertEquals(0, scratch.length); SortedDocValues sorted = cache.getTermsIndex(ar, "bogustermsindex"); assertEquals(-1, sorted.getOrd(0)); sorted.get(0, scratch); assertEquals(0, scratch.length); SortedSetDocValues sortedSet = cache.getDocTermOrds(ar, "bogusmultivalued"); sortedSet.setDocument(0); assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd()); Bits bits = cache.getDocsWithField(ar, "bogusbits"); assertFalse(bits.get(0)); // check that we cached nothing assertEquals(0, cache.getCacheEntries().length); ir.close(); dir.close(); }
public void testIntFieldCache() throws IOException { Directory dir = newDirectory(); IndexWriterConfig cfg = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); cfg.setMergePolicy(newLogMergePolicy()); RandomIndexWriter iw = new RandomIndexWriter(random(), dir, cfg); Document doc = new Document(); IntField field = new IntField("f", 0, Store.YES); doc.add(field); final int[] values = new int[_TestUtil.nextInt(random(), 1, 10)]; for (int i = 0; i < values.length; ++i) { final int v; switch (random().nextInt(10)) { case 0: v = Integer.MIN_VALUE; break; case 1: v = 0; break; case 2: v = Integer.MAX_VALUE; break; default: v = _TestUtil.nextInt(random(), -10, 10); break; } values[i] = v; if (v == 0 && random().nextBoolean()) { // missing iw.addDocument(new Document()); } else { field.setIntValue(v); iw.addDocument(doc); } } iw.forceMerge(1); final DirectoryReader reader = iw.getReader(); final FieldCache.Ints ints = FieldCache.DEFAULT.getInts(getOnlySegmentReader(reader), "f", false); for (int i = 0; i < values.length; ++i) { assertEquals(values[i], ints.get(i)); } reader.close(); iw.close(); dir.close(); }