/** * Marks the processing of the provided sequence number as completed as updates the checkpoint if possible. * * @param seqNo the sequence number to mark as completed */ public synchronized void markSeqNoAsCompleted(final long seqNo) { // make sure we track highest seen sequence number if (seqNo >= nextSeqNo) { nextSeqNo = seqNo + 1; } if (seqNo <= checkpoint) { // this is possible during recovery where we might replay an operation that was also replicated return; } final FixedBitSet bitSet = getBitSetForSeqNo(seqNo); final int offset = seqNoToBitSetOffset(seqNo); bitSet.set(offset); if (seqNo == checkpoint + 1) { updateCheckpoint(); } }
public void testSingleValuedLongs() throws Exception { final int numDocs = scaledRandomIntBetween(1, 100); final long[] array = new long[numDocs]; final FixedBitSet docsWithValue = randomBoolean() ? null : new FixedBitSet(numDocs); for (int i = 0; i < array.length; ++i) { if (randomBoolean()) { array[i] = randomLong(); if (docsWithValue != null) { docsWithValue.set(i); } } else if (docsWithValue != null && randomBoolean()) { docsWithValue.set(i); } } final NumericDocValues singleValues = new NumericDocValues() { @Override public long get(int docID) { return array[docID]; } }; final SortedNumericDocValues multiValues = DocValues.singleton(singleValues, docsWithValue); verify(multiValues, numDocs); final FixedBitSet rootDocs = randomRootDocs(numDocs); final FixedBitSet innerDocs = randomInnerDocs(rootDocs); verify(multiValues, numDocs, rootDocs, innerDocs); }
public void testSingleValuedDoubles() throws Exception { final int numDocs = scaledRandomIntBetween(1, 100); final double[] array = new double[numDocs]; final FixedBitSet docsWithValue = randomBoolean() ? null : new FixedBitSet(numDocs); for (int i = 0; i < array.length; ++i) { if (randomBoolean()) { array[i] = randomDouble(); if (docsWithValue != null) { docsWithValue.set(i); } } else if (docsWithValue != null && randomBoolean()) { docsWithValue.set(i); } } final NumericDoubleValues singleValues = new NumericDoubleValues() { @Override public double get(int docID) { return array[docID]; } }; final SortedNumericDoubleValues multiValues = FieldData.singleton(singleValues, docsWithValue); verify(multiValues, numDocs); final FixedBitSet rootDocs = randomRootDocs(numDocs); final FixedBitSet innerDocs = randomInnerDocs(rootDocs); verify(multiValues, numDocs, rootDocs, innerDocs); }
private static FixedBitSet getSeqNosSet(final IndexReader reader, final long highestSeqNo) throws IOException { // _seq_no are stored as doc values for the time being, so this is how we get them // (as opposed to using an IndexSearcher or IndexReader) final FixedBitSet bitSet = new FixedBitSet((int) highestSeqNo + 1); final List<LeafReaderContext> leaves = reader.leaves(); if (leaves.isEmpty()) { return bitSet; } for (int i = 0; i < leaves.size(); i++) { final LeafReader leaf = leaves.get(i).reader(); final NumericDocValues values = leaf.getNumericDocValues(SeqNoFieldMapper.NAME); if (values == null) { continue; } final Bits bits = leaf.getLiveDocs(); for (int docID = 0; docID < leaf.maxDoc(); docID++) { if (bits == null || bits.get(docID)) { final long seqNo = values.get(docID); assertFalse("should not have more than one document with the same seq_no[" + seqNo + "]", bitSet.get((int) seqNo)); bitSet.set((int) seqNo); } } } return bitSet; }
/** union (term group) bit-sets until they are disjoint (O(n^^2)), and each group have different terms */ private void unionTermGroups(ArrayList<FixedBitSet> bb) { int incr; for (int i=0; i<bb.size()-1; i+=incr) { incr = 1; int j = i+1; while (j<bb.size()) { if (bb.get(i).intersects(bb.get(j))) { bb.get(i).or(bb.get(j)); bb.remove(j); incr = 0; } else { ++j; } } } }
public void addValue(int docID, long value) { if (docID < pending.size()) { throw new IllegalArgumentException("DocValuesField \"" + fieldInfo.name + "\" appears more than once in this document (only one value is allowed per field)"); } // Fill in any holes: for (int i = (int)pending.size(); i < docID; ++i) { pending.add(MISSING); } pending.add(value); if (docsWithField != null) { docsWithField = FixedBitSet.ensureCapacity(docsWithField, docID); docsWithField.set(docID); } updateBytesUsed(); }
/** * Create a bit vector representing the live documents of the * virtual collection to be used in searches. * This will respect deleted documents. * * @param The * {@link LeafReaderContext} to search in. * @return A bit vector representing the live documents of the * virtual collection. * @throws IOException */ public FixedBitSet bits (LeafReaderContext atomic) throws IOException { LeafReader r = atomic.reader(); FixedBitSet bitset = new FixedBitSet(r.maxDoc()); DocIdSet docids = this.getDocIdSet(atomic, (Bits) r.getLiveDocs()); if (docids == null) { if (this.cbi != null) { bitset.clear(0, bitset.length()); } else { bitset.set(0, bitset.length()); }; } else bitset.or(docids.iterator()); return bitset; }
@Override public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { current = new FixedBitSet(context.reader().maxDoc()); fixedBitSets.add(context.ord, current); return new LeafCollector() { @Override public void setScorer(Scorer scorer) throws IOException {} @Override public void collect(int doc) throws IOException { current.set(doc); totalHits++; } }; }
public static FuzzySet deserialize(DataInput in) throws IOException { int version=in.readInt(); if (version == VERSION_SPI) { in.readString(); } final HashFunction hashFunction = hashFunctionForVersion(version); int bloomSize=in.readInt(); int numLongs=in.readInt(); long[]longs=new long[numLongs]; for (int i = 0; i < numLongs; i++) { longs[i]=in.readLong(); } FixedBitSet bits = new FixedBitSet(longs,bloomSize+1); return new FuzzySet(bits,bloomSize,hashFunction); }
private boolean openBitSetContains(int[] expectedDocs, FixedBitSet actual, int maxDoc) throws IOException { if (expectedDocs.length != actual.cardinality()) { return false; } FixedBitSet expected = new FixedBitSet(maxDoc); for (int expectedDoc : expectedDocs) { expected.set(expectedDoc); } int docId; DocIdSetIterator iterator = expected.iterator(); while ((docId = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { if (!actual.get(docId)) { return false; } } return true; }
Bits randomLiveDocs(int maxDoc) { if (rarely()) { if (random().nextBoolean()) { return null; } else { return new Bits.MatchNoBits(maxDoc); } } final FixedBitSet bits = new FixedBitSet(maxDoc); final int bitsSet = TestUtil.nextInt(random(), 1, maxDoc - 1); for (int i = 0; i < bitsSet; ++i) { while (true) { final int index = random().nextInt(maxDoc); if (!bits.get(index)) { bits.set(index); break; } } } return bits; }
/** * Creates a {@link Docs} to record hits. The default uses {@link FixedBitSet} * to record hits and you can override to e.g. record the docs in your own * {@link DocIdSet}. */ protected Docs createDocs(final int maxDoc) { return new Docs() { private final FixedBitSet bits = new FixedBitSet(maxDoc); @Override public void addDoc(int docId) throws IOException { bits.set(docId); } @Override public DocIdSet getDocIdSet() { return bits; } }; }
public static Vector weightedSuperposition( BinaryVector v1, double weight1, BinaryVector v2, double weight2) { BinaryVector conclusion = (BinaryVector) VectorFactory.createZeroVector(VectorType.BINARY, v1.getDimension()); FixedBitSet cVote = conclusion.bitSet; FixedBitSet v1vote = v1.bitSet; FixedBitSet v2vote = v2.bitSet; Random random = new Random(); random.setSeed(Bobcat.asLong(v1.writeLongToString())); for (int x = 0; x < v1.getDimension(); x++) { double probability = 0; if (v1vote.get(x)) probability += weight1 / (weight1 + weight2); if (v2vote.get(x)) probability += weight2 / (weight1 + weight2); if (random.nextDouble() <= probability) cVote.set(x); } return conclusion; }
protected void fillDocsAndScores(FixedBitSet matchingDocs, Bits acceptDocs, TermsEnum termsEnum) throws IOException { BytesRef spare = new BytesRef(); DocsEnum docsEnum = null; for (int i = 0; i < terms.size(); i++) { if (termsEnum.seekExact(terms.get(ords[i], spare))) { docsEnum = termsEnum.docs(acceptDocs, docsEnum, DocsEnum.FLAG_NONE); float score = TermsIncludingScoreQuery.this.scores[ords[i]]; for (int doc = docsEnum.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = docsEnum.nextDoc()) { matchingDocs.set(doc); // In the case the same doc is also related to a another doc, a score might be overwritten. I think this // can only happen in a many-to-many relation scores[doc] = score; } } } }
@Override protected void fillDocsAndScores(FixedBitSet matchingDocs, Bits acceptDocs, TermsEnum termsEnum) throws IOException { BytesRef spare = new BytesRef(); DocsEnum docsEnum = null; for (int i = 0; i < terms.size(); i++) { if (termsEnum.seekExact(terms.get(ords[i], spare))) { docsEnum = termsEnum.docs(acceptDocs, docsEnum, DocsEnum.FLAG_NONE); float score = TermsIncludingScoreQuery.this.scores[ords[i]]; for (int doc = docsEnum.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = docsEnum.nextDoc()) { // I prefer this: /*if (scores[doc] < score) { scores[doc] = score; matchingDocs.set(doc); }*/ // But this behaves the same as MVInnerScorer and only then the tests will pass: if (!matchingDocs.get(doc)) { scores[doc] = score; matchingDocs.set(doc); } } } } }
/** * This method is the first of two required to facilitate superposition. The underlying representation * (i.e. the voting record) is an ArrayList of FixedBitSet, each with dimension "dimension", which can * be thought of as an expanding 2D array of bits. Each column keeps count (in binary) for the respective * dimension, and columns are incremented in parallel by sweeping a bitset across the rows. In any dimension * in which the BitSet to be added contains a "1", the effect will be that 1's are changed to 0's until a * new 1 is added (e.g. the column '110' would become '001' and so forth). * * The first method deals with floating point issues, and accelerates superposition by decomposing * the task into segments. * * @param incomingBitSet * @param weight */ protected synchronized void superposeBitSet(FixedBitSet incomingBitSet, double weight) { // If fractional weights are used, encode all weights as integers (1000 x double value). weight = (int) Math.round(weight * Math.pow(10, BINARY_VECTOR_DECIMAL_PLACES)); if (weight == 0) return; // Keep track of number (or cumulative weight) of votes. totalNumberOfVotes.set(totalNumberOfVotes.get() + (int) weight); // Decompose superposition task such that addition of some power of 2 (e.g. 64) is accomplished // by beginning the process at the relevant row (e.g. 7) instead of starting multiple (e.g. 64) // superposition processes at the first row. int logFloorOfWeight = (int) (Math.floor(Math.log(weight)/Math.log(2))); if (logFloorOfWeight < votingRecord.size() - 1) { while (logFloorOfWeight > 0) { superposeBitSetFromRowFloor(incomingBitSet, logFloorOfWeight); weight = weight - (int) Math.pow(2,logFloorOfWeight); logFloorOfWeight = (int) (Math.floor(Math.log(weight)/Math.log(2))); } } // Add remaining component of weight incrementally. for (int x = 0; x < weight; x++) superposeBitSetFromRowFloor(incomingBitSet, 0); }
public void testIsCacheAble() throws Exception { Directory dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); writer.addDocument(new Document()); writer.close(); IndexReader reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir)); // not cacheable: assertDocIdSetCacheable(reader, new QueryWrapperFilter(new TermQuery(new Term("test","value"))), false); // returns default empty docidset, always cacheable: assertDocIdSetCacheable(reader, NumericRangeFilter.newIntRange("test", Integer.valueOf(10000), Integer.valueOf(-10000), true, true), true); // is cacheable: assertDocIdSetCacheable(reader, FieldCacheRangeFilter.newIntRange("test", Integer.valueOf(10), Integer.valueOf(20), true, true), false); // a fixedbitset filter is always cacheable assertDocIdSetCacheable(reader, new Filter() { @Override public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) { return new FixedBitSet(context.reader().maxDoc()); } }, true); reader.close(); dir.close(); }
@Override public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException { final int maxDoc = context.reader().maxDoc(); final FieldCache.Ints idSource = FieldCache.DEFAULT.getInts(context.reader(), "id", false); assertNotNull(idSource); final FixedBitSet bits = new FixedBitSet(maxDoc); for(int docID=0;docID<maxDoc;docID++) { if (random.nextFloat() <= density && (acceptDocs == null || acceptDocs.get(docID))) { bits.set(docID); //System.out.println(" acc id=" + idSource.get(docID) + " docID=" + docID + " id=" + idSource.get(docID) + " v=" + docValues.get(idSource.get(docID)).utf8ToString()); matchValues.add(docValues.get(idSource.get(docID))); } } return bits; }
private void verifyCount(IndexReader ir) throws Exception { Fields fields = MultiFields.getFields(ir); if (fields == null) { return; } for (String field : fields) { Terms terms = fields.terms(field); if (terms == null) { continue; } int docCount = terms.getDocCount(); FixedBitSet visited = new FixedBitSet(ir.maxDoc()); TermsEnum te = terms.iterator(null); while (te.next() != null) { DocsEnum de = TestUtil.docs(random(), te, null, null, DocsEnum.FLAG_NONE); while (de.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { visited.set(de.docID()); } } assertEquals(visited.cardinality(), docCount); } }
protected void writeDoc(SortDoc sortDoc, List<AtomicReaderContext> leaves, FieldWriter[] fieldWriters, FixedBitSet[] sets, Writer out) throws IOException{ int ord = sortDoc.ord; FixedBitSet set = sets[ord]; set.clear(sortDoc.docId); AtomicReaderContext context = leaves.get(ord); boolean needsComma = false; for(FieldWriter fieldWriter : fieldWriters) { if(needsComma) { out.write(','); } fieldWriter.write(sortDoc.docId, context.reader(), out); needsComma = true; } }
public FixedBitSet getCollapsedSet() { if(nullDoc > -1) { this.collapsedSet.set(nullDoc); } if(this.boostOrds != null) { for(int i=0; i<this.boostOrds.length; i++) { ords[boostOrds[i]] = -1; } } for(int i=0; i<ords.length; i++) { int doc = ords[i]; if(doc > -1) { collapsedSet.set(doc); } } return collapsedSet; }
@Override public DocSet andNot(DocSet other) { FixedBitSet newbits = bits.clone(); if (other instanceof BitDocSet) { newbits.andNot(((BitDocSet) other).bits); } else { DocIterator iter = other.iterator(); while (iter.hasNext()) { int doc = iter.nextDoc(); if (doc < newbits.length()) { newbits.clear(doc); } } } return new BitDocSet(newbits); }
@Override public DocSet union(DocSet other) { FixedBitSet newbits = bits.clone(); if (other instanceof BitDocSet) { BitDocSet otherDocSet = (BitDocSet) other; newbits = FixedBitSet.ensureCapacity(newbits, otherDocSet.bits.length()); newbits.or(otherDocSet.bits); } else { DocIterator iter = other.iterator(); while (iter.hasNext()) { int doc = iter.nextDoc(); newbits = FixedBitSet.ensureCapacity(newbits, doc); newbits.set(doc); } } return new BitDocSet(newbits); }
@Override /** * Reads a (dense) version of a vector from a Lucene input stream. */ public void readFromLuceneStream(IndexInput inputStream) { long bitArray[] = new long[(dimension / 64)]; for (int i = 0; i < dimension / 64; ++i) { try { bitArray[i] = inputStream.readLong(); } catch (IOException e) { logger.severe("Couldn't read binary vector from lucene output stream."); e.printStackTrace(); } } this.bitSet = new FixedBitSet(bitArray, dimension); this.isSparse = true; }
public GroupExpandCollector(SortedDocValues docValues, FixedBitSet groupBits, IntOpenHashSet collapsedSet, int limit, Sort sort) throws IOException { int numGroups = collapsedSet.size(); groups = new IntObjectOpenHashMap<>(numGroups * 2); collectors = new ArrayList<>(); DocIdSetIterator iterator = groupBits.iterator(); int group; while ((group = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { Collector collector = (sort == null) ? TopScoreDocCollector.create(limit, true) : TopFieldCollector.create(sort, limit, false, false, false, true); groups.put(group, collector); collectors.add(collector); } this.collapsedSet = collapsedSet; this.groupBits = groupBits; this.docValues = docValues; }
public DocSet getDocSet(FixedBitSet bs) { switch(rand.nextInt(10)) { case 0: return getHashDocSet(bs); case 1: return getBitDocSet(bs); case 2: return getBitDocSet(bs); case 3: return getBitDocSet(bs); case 4: return getIntDocSet(bs); case 5: return getIntDocSet(bs); case 6: return getIntDocSet(bs); case 7: return getIntDocSet(bs); case 8: return getIntDocSet(bs); case 9: return getDocSlice(bs); } return null; }
static void generate(int maxSize, int bitsToSet) { bs = new FixedBitSet(maxSize); ids = new int[bitsToSet]; int count=0; if (maxSize>0) { for (int i=0; i<bitsToSet; i++) { int id=rand.nextInt(maxSize); if (!bs.get(id)) { bs.set(id); ids[count++]=id; } } } bds = new BitDocSet(bs,bitsToSet); hds = new HashDocSet(ids,0,count); }
/** * Return the bit array for the provided sequence number, possibly allocating a new array if needed. * * @param seqNo the sequence number to obtain the bit array for * @return the bit array corresponding to the provided sequence number */ private FixedBitSet getBitSetForSeqNo(final long seqNo) { assert Thread.holdsLock(this); assert seqNo >= firstProcessedSeqNo : "seqNo: " + seqNo + " firstProcessedSeqNo: " + firstProcessedSeqNo; final long bitSetOffset = (seqNo - firstProcessedSeqNo) / bitArraysSize; if (bitSetOffset > Integer.MAX_VALUE) { throw new IndexOutOfBoundsException( "sequence number too high; got [" + seqNo + "], firstProcessedSeqNo [" + firstProcessedSeqNo + "]"); } while (bitSetOffset >= processedSeqNo.size()) { processedSeqNo.add(new FixedBitSet(bitArraysSize)); } return processedSeqNo.get((int) bitSetOffset); }
/** * Builds a {@link BitSet} where each documents bit is that that has one or more ordinals associated with it. * if every document has an ordinal associated with it this method returns <code>null</code> */ public BitSet buildDocsWithValuesSet() { if (numDocsWithValue == maxDoc) { return null; } final FixedBitSet bitSet = new FixedBitSet(maxDoc); for (int docID = 0; docID < maxDoc; ++docID) { if (ordinals.firstOrdinals.get(docID) != 0) { bitSet.set(docID); } } return bitSet; }
private static FixedBitSet randomRootDocs(int maxDoc) { FixedBitSet set = new FixedBitSet(maxDoc); for (int i = 0; i < maxDoc; ++i) { if (randomBoolean()) { set.set(i); } } // the last doc must be a root doc set.set(maxDoc - 1); return set; }
private static FixedBitSet randomInnerDocs(FixedBitSet rootDocs) { FixedBitSet innerDocs = new FixedBitSet(rootDocs.length()); for (int i = 0; i < innerDocs.length(); ++i) { if (!rootDocs.get(i) && randomBoolean()) { innerDocs.set(i); } } return innerDocs; }
private void verify(SortedNumericDocValues values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) throws IOException { for (long missingValue : new long[] { 0, randomLong() }) { for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX, MultiValueMode.SUM, MultiValueMode.AVG}) { final NumericDocValues selected = mode.select(values, missingValue, rootDocs, new BitSetIterator(innerDocs, 0L), maxDoc); int prevRoot = -1; for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) { final long actual = selected.get(root); long expected = 0; if (mode == MultiValueMode.MAX) { expected = Long.MIN_VALUE; } else if (mode == MultiValueMode.MIN) { expected = Long.MAX_VALUE; } int numValues = 0; for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit(child + 1)) { values.setDocument(child); for (int j = 0; j < values.count(); ++j) { if (mode == MultiValueMode.SUM || mode == MultiValueMode.AVG) { expected += values.valueAt(j); } else if (mode == MultiValueMode.MIN) { expected = Math.min(expected, values.valueAt(j)); } else if (mode == MultiValueMode.MAX) { expected = Math.max(expected, values.valueAt(j)); } ++numValues; } } if (numValues == 0) { expected = missingValue; } else if (mode == MultiValueMode.AVG) { expected = numValues > 1 ? Math.round((double) expected / (double) numValues) : expected; } assertEquals(mode.toString() + " docId=" + root, expected, actual); prevRoot = root; } } } }
private void verify(SortedNumericDoubleValues values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) throws IOException { for (long missingValue : new long[] { 0, randomLong() }) { for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX, MultiValueMode.SUM, MultiValueMode.AVG}) { final NumericDoubleValues selected = mode.select(values, missingValue, rootDocs, new BitSetIterator(innerDocs, 0L), maxDoc); int prevRoot = -1; for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) { final double actual = selected.get(root); double expected = 0.0; if (mode == MultiValueMode.MAX) { expected = Long.MIN_VALUE; } else if (mode == MultiValueMode.MIN) { expected = Long.MAX_VALUE; } int numValues = 0; for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit(child + 1)) { values.setDocument(child); for (int j = 0; j < values.count(); ++j) { if (mode == MultiValueMode.SUM || mode == MultiValueMode.AVG) { expected += values.valueAt(j); } else if (mode == MultiValueMode.MIN) { expected = Math.min(expected, values.valueAt(j)); } else if (mode == MultiValueMode.MAX) { expected = Math.max(expected, values.valueAt(j)); } ++numValues; } } if (numValues == 0) { expected = missingValue; } else if (mode == MultiValueMode.AVG) { expected = expected/numValues; } assertEquals(mode.toString() + " docId=" + root, expected, actual, 0.1); prevRoot = root; } } } }
public void testSingleValuedStrings() throws Exception { final int numDocs = scaledRandomIntBetween(1, 100); final BytesRef[] array = new BytesRef[numDocs]; final FixedBitSet docsWithValue = randomBoolean() ? null : new FixedBitSet(numDocs); for (int i = 0; i < array.length; ++i) { if (randomBoolean()) { array[i] = new BytesRef(RandomStrings.randomAsciiOfLength(random(), 8)); if (docsWithValue != null) { docsWithValue.set(i); } } else { array[i] = new BytesRef(); if (docsWithValue != null && randomBoolean()) { docsWithValue.set(i); } } } final BinaryDocValues singleValues = new BinaryDocValues() { @Override public BytesRef get(int docID) { return BytesRef.deepCopyOf(array[docID]); } }; final SortedBinaryDocValues multiValues = FieldData.singleton(singleValues, docsWithValue); verify(multiValues, numDocs); final FixedBitSet rootDocs = randomRootDocs(numDocs); final FixedBitSet innerDocs = randomInnerDocs(rootDocs); verify(multiValues, numDocs, rootDocs, innerDocs); }
private void verify(SortedBinaryDocValues values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) throws IOException { for (BytesRef missingValue : new BytesRef[] { new BytesRef(), new BytesRef(RandomStrings.randomAsciiOfLength(random(), 8)) }) { for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX}) { final BinaryDocValues selected = mode.select(values, missingValue, rootDocs, new BitSetIterator(innerDocs, 0L), maxDoc); int prevRoot = -1; for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) { final BytesRef actual = selected.get(root); BytesRef expected = null; for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit(child + 1)) { values.setDocument(child); for (int j = 0; j < values.count(); ++j) { if (expected == null) { expected = BytesRef.deepCopyOf(values.valueAt(j)); } else { if (mode == MultiValueMode.MIN) { expected = expected.compareTo(values.valueAt(j)) <= 0 ? expected : BytesRef.deepCopyOf(values.valueAt(j)); } else if (mode == MultiValueMode.MAX) { expected = expected.compareTo(values.valueAt(j)) > 0 ? expected : BytesRef.deepCopyOf(values.valueAt(j)); } } } } if (expected == null) { expected = missingValue; } assertEquals(mode.toString() + " docId=" + root, expected, actual); prevRoot = root; } } } }
public void testSingleValuedOrds() throws Exception { final int numDocs = scaledRandomIntBetween(1, 100); final int[] array = new int[numDocs]; for (int i = 0; i < array.length; ++i) { if (randomBoolean()) { array[i] = randomInt(1000); } else { array[i] = -1; } } final SortedDocValues singleValues = new SortedDocValues() { @Override public int getOrd(int docID) { return array[docID]; } @Override public BytesRef lookupOrd(int ord) { throw new UnsupportedOperationException(); } @Override public int getValueCount() { return 1 << 20; } }; final RandomAccessOrds multiValues = (RandomAccessOrds) DocValues.singleton(singleValues); verify(multiValues, numDocs); final FixedBitSet rootDocs = randomRootDocs(numDocs); final FixedBitSet innerDocs = randomInnerDocs(rootDocs); verify(multiValues, numDocs, rootDocs, innerDocs); }
private void verify(RandomAccessOrds values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) throws IOException { for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX}) { final SortedDocValues selected = mode.select(values, rootDocs, new BitSetIterator(innerDocs, 0L)); int prevRoot = -1; for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) { final int actual = selected.getOrd(root); int expected = -1; for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit(child + 1)) { values.setDocument(child); for (int j = 0; j < values.cardinality(); ++j) { if (expected == -1) { expected = (int) values.ordAt(j); } else { if (mode == MultiValueMode.MIN) { expected = Math.min(expected, (int)values.ordAt(j)); } else if (mode == MultiValueMode.MAX) { expected = Math.max(expected, (int)values.ordAt(j)); } } } } assertEquals(mode.toString() + " docId=" + root, expected, actual); prevRoot = root; } } }
public void testUnsortedSingleValuedDoubles() throws Exception { final int numDocs = scaledRandomIntBetween(1, 100); final double[] array = new double[numDocs]; final FixedBitSet docsWithValue = randomBoolean() ? null : new FixedBitSet(numDocs); for (int i = 0; i < array.length; ++i) { if (randomBoolean()) { array[i] = randomDouble(); if (docsWithValue != null) { docsWithValue.set(i); } } else if (docsWithValue != null && randomBoolean()) { docsWithValue.set(i); } } final NumericDoubleValues singleValues = new NumericDoubleValues() { @Override public double get(int docID) { return array[docID]; } }; final SortedNumericDoubleValues singletonValues = FieldData.singleton(singleValues, docsWithValue); final MultiValueMode.UnsortedNumericDoubleValues multiValues = new MultiValueMode.UnsortedNumericDoubleValues() { @Override public int count() { return singletonValues.count(); } @Override public void setDocument(int doc) { singletonValues.setDocument(doc); } @Override public double valueAt(int index) { return Math.cos(singletonValues.valueAt(index)); } }; verify(multiValues, numDocs); }
/** * test version lookup with two documents matching the ID */ public void testTwoDocuments() throws Exception { Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); Document doc = new Document(); doc.add(new Field(UidFieldMapper.NAME, "6", UidFieldMapper.Defaults.FIELD_TYPE)); doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 87)); writer.addDocument(doc); writer.addDocument(doc); DirectoryReader reader = DirectoryReader.open(writer); LeafReaderContext segment = reader.leaves().get(0); PerThreadIDAndVersionLookup lookup = new PerThreadIDAndVersionLookup(segment.reader()); // return the last doc when there are duplicates DocIdAndVersion result = lookup.lookup(new BytesRef("6"), null, segment); assertNotNull(result); assertEquals(87, result.version); assertEquals(1, result.docId); // delete the first doc only FixedBitSet live = new FixedBitSet(2); live.set(1); result = lookup.lookup(new BytesRef("6"), live, segment); assertNotNull(result); assertEquals(87, result.version); assertEquals(1, result.docId); // delete the second doc only live.clear(1); live.set(0); result = lookup.lookup(new BytesRef("6"), live, segment); assertNotNull(result); assertEquals(87, result.version); assertEquals(0, result.docId); // delete both docs assertNull(lookup.lookup(new BytesRef("6"), new Bits.MatchNoBits(2), segment)); reader.close(); writer.close(); dir.close(); }
