private static void assertListEquality( String message, List<? extends PackedInts.Reader> packedInts) { if (packedInts.size() == 0) { return; } PackedInts.Reader base = packedInts.get(0); int valueCount = base.size(); for (PackedInts.Reader packedInt: packedInts) { assertEquals(message + ". The number of values should be the same ", valueCount, packedInt.size()); } for (int i = 0 ; i < valueCount ; i++) { for (int j = 1 ; j < packedInts.size() ; j++) { assertEquals(String.format(Locale.ROOT, "%s. The value at index %d should be the same for %s and %s", message, i, base.getClass().getSimpleName(), packedInts.get(j).getClass().getSimpleName()), base.get(i), packedInts.get(j).get(i)); } } }
public void testSingleValue() throws Exception { for (int bitsPerValue = 1; bitsPerValue <= 64; ++bitsPerValue) { Directory dir = newDirectory(); IndexOutput out = dir.createOutput("out", newIOContext(random())); PackedInts.Writer w = PackedInts.getWriter(out, 1, bitsPerValue, PackedInts.DEFAULT); long value = 17L & PackedInts.maxValue(bitsPerValue); w.add(value); w.finish(); final long end = out.getFilePointer(); out.close(); IndexInput in = dir.openInput("out", newIOContext(random())); Reader reader = PackedInts.getReader(in); String msg = "Impl=" + w.getClass().getSimpleName() + ", bitsPerValue=" + bitsPerValue; assertEquals(msg, 1, reader.size()); assertEquals(msg, value, reader.get(0)); assertEquals(msg, end, in.getFilePointer()); in.close(); dir.close(); } }
public void testPackedIntsNull() { // must be > 10 for the bulk reads below int size = TestUtil.nextInt(random(), 11, 256); Reader packedInts = new PackedInts.NullReader(size); assertEquals(0, packedInts.get(TestUtil.nextInt(random(), 0, size - 1))); long[] arr = new long[size + 10]; int r; Arrays.fill(arr, 1); r = packedInts.get(0, arr, 0, size - 1); assertEquals(size - 1, r); for (r--; r >= 0; r--) { assertEquals(0, arr[r]); } Arrays.fill(arr, 1); r = packedInts.get(10, arr, 0, size + 10); assertEquals(size - 10, r); for (int i = 0; i < size - 10; i++) { assertEquals(0, arr[i]); } }
public void testPackedIntsNull() { // must be > 10 for the bulk reads below int size = _TestUtil.nextInt(random(), 11, 256); Reader packedInts = new PackedInts.NullReader(size); assertEquals(0, packedInts.get(_TestUtil.nextInt(random(), 0, size - 1))); long[] arr = new long[size + 10]; int r; Arrays.fill(arr, 1); r = packedInts.get(0, arr, 0, size - 1); assertEquals(size - 1, r); for (r--; r >= 0; r--) { assertEquals(0, arr[r]); } Arrays.fill(arr, 1); r = packedInts.get(10, arr, 0, size + 10); assertEquals(size - 10, r); for (int i = 0; i < size - 10; i++) { assertEquals(0, arr[i]); } }
@Override public MonotonicLongValues build() { finish(); pending = null; final PackedInts.Reader[] values = Arrays.copyOf(this.values, valuesOff); final long[] mins = Arrays.copyOf(this.mins, valuesOff); final float[] averages = Arrays.copyOf(this.averages, valuesOff); final long ramBytesUsed = MonotonicLongValues.BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values) + RamUsageEstimator.sizeOf(mins) + RamUsageEstimator.sizeOf(averages); return new MonotonicLongValues(pageShift, pageMask, values, mins, averages, size, ramBytesUsed); }
@Override public DeltaPackedLongValues build() { finish(); pending = null; final PackedInts.Reader[] values = Arrays.copyOf(this.values, valuesOff); final long[] mins = Arrays.copyOf(this.mins, valuesOff); final long ramBytesUsed = DeltaPackedLongValues.BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values) + RamUsageEstimator.sizeOf(mins); return new DeltaPackedLongValues(pageShift, pageMask, values, mins, size, ramBytesUsed); }
public void testSave() throws IOException { final int valueCount = TestUtil.nextInt(random(), 1, 2048); for (int bpv = 1; bpv <= 64; ++bpv) { final int maxValue = (int) Math.min(PackedInts.maxValue(31), PackedInts.maxValue(bpv)); final RAMDirectory directory = new RAMDirectory(); List<PackedInts.Mutable> packedInts = createPackedInts(valueCount, bpv); for (PackedInts.Mutable mutable : packedInts) { for (int i = 0; i < mutable.size(); ++i) { mutable.set(i, random().nextInt(maxValue)); } IndexOutput out = directory.createOutput("packed-ints.bin", IOContext.DEFAULT); mutable.save(out); out.close(); IndexInput in = directory.openInput("packed-ints.bin", IOContext.DEFAULT); PackedInts.Reader reader = PackedInts.getReader(in); assertEquals(valueCount, reader.size()); if (mutable instanceof Packed64SingleBlock) { // make sure that we used the right format so that the reader has // the same performance characteristics as the mutable that has been // serialized assertTrue(reader instanceof Packed64SingleBlock); } else { assertFalse(reader instanceof Packed64SingleBlock); } for (int i = 0; i < valueCount; ++i) { assertEquals(mutable.get(i), reader.get(i)); } in.close(); directory.deleteFile("packed-ints.bin"); } directory.close(); } }
public void testSave() throws IOException { final int valueCount = _TestUtil.nextInt(random(), 1, 2048); for (int bpv = 1; bpv <= 64; ++bpv) { final int maxValue = (int) Math.min(PackedInts.maxValue(31), PackedInts.maxValue(bpv)); final RAMDirectory directory = new RAMDirectory(); List<PackedInts.Mutable> packedInts = createPackedInts(valueCount, bpv); for (PackedInts.Mutable mutable : packedInts) { for (int i = 0; i < mutable.size(); ++i) { mutable.set(i, random().nextInt(maxValue)); } IndexOutput out = directory.createOutput("packed-ints.bin", IOContext.DEFAULT); mutable.save(out); out.close(); IndexInput in = directory.openInput("packed-ints.bin", IOContext.DEFAULT); PackedInts.Reader reader = PackedInts.getReader(in); assertEquals(mutable.getBitsPerValue(), reader.getBitsPerValue()); assertEquals(valueCount, reader.size()); if (mutable instanceof Packed64SingleBlock) { // make sure that we used the right format so that the reader has // the same performance characteristics as the mutable that has been // serialized assertTrue(reader instanceof Packed64SingleBlock); } else { assertFalse(reader instanceof Packed64SingleBlock); } for (int i = 0; i < valueCount; ++i) { assertEquals(mutable.get(i), reader.get(i)); } in.close(); directory.deleteFile("packed-ints.bin"); } directory.close(); } }
MonotonicLongValues(int pageShift, int pageMask, Reader[] values, long[] mins, float[] averages, long size, long ramBytesUsed) { super(pageShift, pageMask, values, mins, size, ramBytesUsed); assert values.length == averages.length; this.averages = averages; }
DeltaPackedLongValues(int pageShift, int pageMask, Reader[] values, long[] mins, long size, long ramBytesUsed) { super(pageShift, pageMask, values, size, ramBytesUsed); assert values.length == mins.length; this.mins = mins; }
public void testEndPointer() throws IOException { final Directory dir = newDirectory(); final int valueCount = RandomInts.randomIntBetween(random(), 1, 1000); final IndexOutput out = dir.createOutput("tests.bin", newIOContext(random())); for (int i = 0; i < valueCount; ++i) { out.writeLong(0); } out.close(); final IndexInput in = dir.openInput("tests.bin", newIOContext(random())); for (int version = PackedInts.VERSION_START; version <= PackedInts.VERSION_CURRENT; ++version) { for (int bpv = 1; bpv <= 64; ++bpv) { for (PackedInts.Format format : PackedInts.Format.values()) { if (!format.isSupported(bpv)) { continue; } final long byteCount = format.byteCount(version, valueCount, bpv); String msg = "format=" + format + ",version=" + version + ",valueCount=" + valueCount + ",bpv=" + bpv; // test iterator in.seek(0L); final PackedInts.ReaderIterator it = PackedInts.getReaderIteratorNoHeader(in, format, version, valueCount, bpv, RandomInts.randomIntBetween(random(), 1, 1<<16)); for (int i = 0; i < valueCount; ++i) { it.next(); } assertEquals(msg, byteCount, in.getFilePointer()); // test direct reader in.seek(0L); final PackedInts.Reader directReader = PackedInts.getDirectReaderNoHeader(in, format, version, valueCount, bpv); directReader.get(valueCount - 1); assertEquals(msg, byteCount, in.getFilePointer()); // test reader in.seek(0L); PackedInts.getReaderNoHeader(in, format, version, valueCount, bpv); assertEquals(msg, byteCount, in.getFilePointer()); } } } in.close(); dir.close(); }
private static void assertListEquality( List<? extends PackedInts.Reader> packedInts) { assertListEquality("", packedInts); }