private BinaryDocValues getVariableBinary(FieldInfo field, final BinaryEntry bytes) throws IOException { final IndexInput data = this.data.clone(); final MonotonicBlockPackedReader addresses = getAddressInstance(data, field, bytes); return new LongBinaryDocValues() { final BytesRef term = new BytesRef(Math.max(0, bytes.maxLength)); @Override public BytesRef get(long id) { long startAddress = bytes.offset + (id == 0 ? 0 : addresses.get(id-1)); long endAddress = bytes.offset + addresses.get(id); int length = (int) (endAddress - startAddress); try { data.seek(startAddress); data.readBytes(term.bytes, 0, length); term.length = length; return term; } catch (IOException e) { throw new RuntimeException(e); } } }; }
/** returns an address instance for prefix-compressed binary values. * @lucene.internal */ protected MonotonicBlockPackedReader getIntervalInstance(IndexInput data, FieldInfo field, BinaryEntry bytes) throws IOException { final MonotonicBlockPackedReader addresses; final long interval = bytes.addressInterval; synchronized (addressInstances) { MonotonicBlockPackedReader addrInstance = addressInstances.get(field.number); if (addrInstance == null) { data.seek(bytes.addressesOffset); final long size; if (bytes.count % interval == 0) { size = bytes.count / interval; } else { size = 1L + bytes.count / interval; } addrInstance = MonotonicBlockPackedReader.of(data, bytes.packedIntsVersion, bytes.blockSize, size, false); addressInstances.put(field.number, addrInstance); ramBytesUsed.addAndGet(addrInstance.ramBytesUsed() + RamUsageEstimator.NUM_BYTES_INT); } addresses = addrInstance; } return addresses; }
private BinaryDocValues getVariableBinary(FieldInfo field, final BinaryEntry bytes) throws IOException { final IndexInput data = this.data.clone(); final MonotonicBlockPackedReader addresses = getAddressInstance(data, field, bytes); return new LongBinaryDocValues() { final BytesRef term = new BytesRef(Math.max(0, bytes.maxLength)); @Override public BytesRef get(long id) { long startAddress = bytes.offset + addresses.get(id); long endAddress = bytes.offset + addresses.get(id+1); int length = (int) (endAddress - startAddress); try { data.seek(startAddress); data.readBytes(term.bytes, 0, length); term.length = length; return term; } catch (IOException e) { throw new RuntimeException(e); } } }; }
/** returns an address instance for prefix-compressed binary values. */ private MonotonicBlockPackedReader getIntervalInstance(IndexInput data, FieldInfo field, BinaryEntry bytes) throws IOException { final MonotonicBlockPackedReader addresses; final long interval = bytes.addressInterval; synchronized (addressInstances) { MonotonicBlockPackedReader addrInstance = addressInstances.get(field.number); if (addrInstance == null) { data.seek(bytes.addressesOffset); final long size; if (bytes.count % interval == 0) { size = bytes.count / interval; } else { size = 1L + bytes.count / interval; } addrInstance = MonotonicBlockPackedReader.of(data, bytes.packedIntsVersion, bytes.blockSize, size, false); addressInstances.put(field.number, addrInstance); ramBytesUsed.addAndGet(addrInstance.ramBytesUsed() + RamUsageEstimator.NUM_BYTES_INT); } addresses = addrInstance; } return addresses; }
private BinaryDocValues getVariableBinary(FieldInfo field, final BinaryEntry bytes) throws IOException { final MonotonicBlockPackedReader addresses = getAddressInstance(field, bytes); final IndexInput data = this.data.slice("var-binary", bytes.offset, bytes.addressesOffset - bytes.offset); final BytesRef term = new BytesRef(Math.max(0, bytes.maxLength)); final byte buffer[] = term.bytes; return new LongBinaryDocValues() { @Override public BytesRef get(long id) { long startAddress = addresses.get(id); long endAddress = addresses.get(id+1); int length = (int) (endAddress - startAddress); try { data.seek(startAddress); data.readBytes(buffer, 0, length); term.length = length; return term; } catch (IOException e) { throw new RuntimeException(e); } } }; }
/** returns a reverse lookup instance for prefix-compressed binary values. */ private synchronized ReverseTermsIndex getReverseIndexInstance(FieldInfo field, BinaryEntry bytes) throws IOException { ReverseTermsIndex index = reverseIndexInstances.get(field.number); if (index == null) { index = new ReverseTermsIndex(); data.seek(bytes.reverseIndexOffset); long size = (bytes.count + REVERSE_INTERVAL_MASK) >>> REVERSE_INTERVAL_SHIFT; index.termAddresses = MonotonicBlockPackedReader.of(data, bytes.packedIntsVersion, bytes.blockSize, size, false); long dataSize = data.readVLong(); PagedBytes pagedBytes = new PagedBytes(15); pagedBytes.copy(data, dataSize); index.terms = pagedBytes.freeze(true); reverseIndexInstances.put(field.number, index); ramBytesUsed.addAndGet(index.termAddresses.ramBytesUsed() + index.terms.ramBytesUsed()); } return index; }
private BinaryDocValues getVariableBinary(FieldInfo field, final BinaryEntry bytes) throws IOException { final IndexInput data = this.data.clone(); data.seek(bytes.addressesOffset); final MonotonicBlockPackedReader addresses = new MonotonicBlockPackedReader(data, bytes.packedIntsVersion, bytes.blockSize, bytes.count, true); return new LongBinaryDocValues() { @Override public void get(long id, BytesRef result) { long startAddress = bytes.offset + (id == 0 ? 0 : + addresses.get(id-1)); long endAddress = bytes.offset + addresses.get(id); int length = (int) (endAddress - startAddress); try { data.seek(startAddress); // NOTE: we could have one buffer, but various consumers (e.g. FieldComparatorSource) // assume "they" own the bytes after calling this! final byte[] buffer = new byte[length]; data.readBytes(buffer, 0, buffer.length); result.bytes = buffer; result.offset = 0; result.length = length; } catch (IOException e) { throw new RuntimeException(e); } } }; }
private BinaryDocValues getVariableBinary(FieldInfo field, final BinaryEntry bytes) throws IOException { final IndexInput data = this.data.clone(); final MonotonicBlockPackedReader addresses = getAddressInstance(data, field, bytes); return new LongBinaryDocValues() { @Override public void get(long id, BytesRef result) { long startAddress = bytes.offset + (id == 0 ? 0 : addresses.get(id-1)); long endAddress = bytes.offset + addresses.get(id); int length = (int) (endAddress - startAddress); try { data.seek(startAddress); // NOTE: we could have one buffer, but various consumers (e.g. FieldComparatorSource) // assume "they" own the bytes after calling this! final byte[] buffer = new byte[length]; data.readBytes(buffer, 0, buffer.length); result.bytes = buffer; result.offset = 0; result.length = length; } catch (IOException e) { throw new RuntimeException(e); } } }; }
/** returns an address instance for variable-length binary values. * @lucene.internal */ protected MonotonicBlockPackedReader getAddressInstance(IndexInput data, FieldInfo field, BinaryEntry bytes) throws IOException { final MonotonicBlockPackedReader addresses; synchronized (addressInstances) { MonotonicBlockPackedReader addrInstance = addressInstances.get(field.number); if (addrInstance == null) { data.seek(bytes.addressesOffset); addrInstance = MonotonicBlockPackedReader.of(data, bytes.packedIntsVersion, bytes.blockSize, bytes.count, false); addressInstances.put(field.number, addrInstance); ramBytesUsed.addAndGet(addrInstance.ramBytesUsed() + RamUsageEstimator.NUM_BYTES_INT); } addresses = addrInstance; } return addresses; }
/** returns an address instance for sortedset ordinal lists * @lucene.internal */ protected MonotonicBlockPackedReader getOrdIndexInstance(IndexInput data, FieldInfo field, NumericEntry entry) throws IOException { final MonotonicBlockPackedReader ordIndex; synchronized (ordIndexInstances) { MonotonicBlockPackedReader ordIndexInstance = ordIndexInstances.get(field.number); if (ordIndexInstance == null) { data.seek(entry.offset); ordIndexInstance = MonotonicBlockPackedReader.of(data, entry.packedIntsVersion, entry.blockSize, entry.count, false); ordIndexInstances.put(field.number, ordIndexInstance); ramBytesUsed.addAndGet(ordIndexInstance.ramBytesUsed() + RamUsageEstimator.NUM_BYTES_INT); } ordIndex = ordIndexInstance; } return ordIndex; }
public CompressedBinaryDocValues(BinaryEntry bytes, MonotonicBlockPackedReader addresses, IndexInput data) throws IOException { this.bytes = bytes; this.interval = bytes.addressInterval; this.addresses = addresses; this.data = data; this.numValues = bytes.count; this.numIndexValues = addresses.size(); this.termsEnum = getTermsEnum(data); }
/** returns an address instance for variable-length binary values. */ private MonotonicBlockPackedReader getAddressInstance(IndexInput data, FieldInfo field, BinaryEntry bytes) throws IOException { final MonotonicBlockPackedReader addresses; synchronized (addressInstances) { MonotonicBlockPackedReader addrInstance = addressInstances.get(field.number); if (addrInstance == null) { data.seek(bytes.addressesOffset); addrInstance = MonotonicBlockPackedReader.of(data, bytes.packedIntsVersion, bytes.blockSize, bytes.count+1, false); addressInstances.put(field.number, addrInstance); ramBytesUsed.addAndGet(addrInstance.ramBytesUsed() + RamUsageEstimator.NUM_BYTES_INT); } addresses = addrInstance; } return addresses; }
/** returns an address instance for sortedset ordinal lists */ private MonotonicBlockPackedReader getOrdIndexInstance(IndexInput data, FieldInfo field, NumericEntry entry) throws IOException { final MonotonicBlockPackedReader ordIndex; synchronized (ordIndexInstances) { MonotonicBlockPackedReader ordIndexInstance = ordIndexInstances.get(field.number); if (ordIndexInstance == null) { data.seek(entry.offset); ordIndexInstance = MonotonicBlockPackedReader.of(data, entry.packedIntsVersion, entry.blockSize, entry.count+1, false); ordIndexInstances.put(field.number, ordIndexInstance); ramBytesUsed.addAndGet(ordIndexInstance.ramBytesUsed() + RamUsageEstimator.NUM_BYTES_INT); } ordIndex = ordIndexInstance; } return ordIndex; }
@Override public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { SortedSetEntry ss = sortedNumerics.get(field.number); NumericEntry numericEntry = numerics.get(field.number); final LongValues values = getNumeric(numericEntry); if (ss.format == SORTED_SINGLE_VALUED) { final Bits docsWithField = getMissingBits(numericEntry.missingOffset); return DocValues.singleton(values, docsWithField); } else if (ss.format == SORTED_WITH_ADDRESSES) { final IndexInput data = this.data.clone(); final MonotonicBlockPackedReader ordIndex = getOrdIndexInstance(data, field, ordIndexes.get(field.number)); return new SortedNumericDocValues() { long startOffset; long endOffset; @Override public void setDocument(int doc) { startOffset = ordIndex.get(doc); endOffset = ordIndex.get(doc+1L); } @Override public long valueAt(int index) { return values.get(startOffset + index); } @Override public int count() { return (int) (endOffset - startOffset); } }; } else { throw new AssertionError(); } }
/** returns an address instance for variable-length binary values. */ private synchronized MonotonicBlockPackedReader getAddressInstance(FieldInfo field, BinaryEntry bytes) throws IOException { MonotonicBlockPackedReader addresses = addressInstances.get(field.number); if (addresses == null) { data.seek(bytes.addressesOffset); addresses = MonotonicBlockPackedReader.of(data, bytes.packedIntsVersion, bytes.blockSize, bytes.count+1, false); addressInstances.put(field.number, addresses); ramBytesUsed.addAndGet(addresses.ramBytesUsed() + RamUsageEstimator.NUM_BYTES_INT); } return addresses; }
/** returns an address instance for prefix-compressed binary values. */ private synchronized MonotonicBlockPackedReader getIntervalInstance(FieldInfo field, BinaryEntry bytes) throws IOException { MonotonicBlockPackedReader addresses = addressInstances.get(field.number); if (addresses == null) { data.seek(bytes.addressesOffset); final long size = (bytes.count + INTERVAL_MASK) >>> INTERVAL_SHIFT; addresses = MonotonicBlockPackedReader.of(data, bytes.packedIntsVersion, bytes.blockSize, size, false); addressInstances.put(field.number, addresses); ramBytesUsed.addAndGet(addresses.ramBytesUsed() + RamUsageEstimator.NUM_BYTES_INT); } return addresses; }
/** returns an address instance for sortedset ordinal lists */ private synchronized MonotonicBlockPackedReader getOrdIndexInstance(FieldInfo field, NumericEntry entry) throws IOException { MonotonicBlockPackedReader instance = ordIndexInstances.get(field.number); if (instance == null) { data.seek(entry.offset); instance = MonotonicBlockPackedReader.of(data, entry.packedIntsVersion, entry.blockSize, entry.count+1, false); ordIndexInstances.put(field.number, instance); ramBytesUsed.addAndGet(instance.ramBytesUsed() + RamUsageEstimator.NUM_BYTES_INT); } return instance; }
@Override public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { SortedSetEntry ss = sortedNumerics.get(field.number); NumericEntry numericEntry = numerics.get(field.number); final LongValues values = getNumeric(numericEntry); if (ss.format == SORTED_SINGLE_VALUED) { final Bits docsWithField = getMissingBits(numericEntry.missingOffset); return DocValues.singleton(values, docsWithField); } else if (ss.format == SORTED_WITH_ADDRESSES) { final MonotonicBlockPackedReader ordIndex = getOrdIndexInstance(field, ordIndexes.get(field.number)); return new SortedNumericDocValues() { long startOffset; long endOffset; @Override public void setDocument(int doc) { startOffset = ordIndex.get(doc); endOffset = ordIndex.get(doc+1L); } @Override public long valueAt(int index) { return values.get(startOffset + index); } @Override public int count() { return (int) (endOffset - startOffset); } }; } else { throw new AssertionError(); } }
public CompressedBinaryDocValues(BinaryEntry bytes, MonotonicBlockPackedReader addresses, ReverseTermsIndex index, IndexInput data) throws IOException { this.maxTermLength = bytes.maxLength; this.numValues = bytes.count; this.addresses = addresses; this.numIndexValues = addresses.size(); this.data = data; this.reverseTerms = index.terms; this.reverseAddresses = index.termAddresses; this.numReverseIndexValues = reverseAddresses.size(); this.termsEnum = getTermsEnum(data); }
private BytesAndAddresses loadBinary(FieldInfo field) throws IOException { BytesAndAddresses bytesAndAddresses = new BytesAndAddresses(); BinaryEntry entry = binaries.get(field.number); data.seek(entry.offset); PagedBytes bytes = new PagedBytes(16); bytes.copy(data, entry.numBytes); bytesAndAddresses.reader = bytes.freeze(true); ramBytesUsed.addAndGet(bytesAndAddresses.reader.ramBytesUsed()); if (entry.minLength != entry.maxLength) { data.seek(data.getFilePointer() + entry.missingBytes); bytesAndAddresses.addresses = MonotonicBlockPackedReader.of(data, entry.packedIntsVersion, entry.blockSize, maxDoc, false); ramBytesUsed.addAndGet(bytesAndAddresses.addresses.ramBytesUsed()); } return bytesAndAddresses; }