@Test public void testSeekWithRandomData() throws Exception { PrefixTreeCodec encoder = new PrefixTreeCodec(); ByteArrayOutputStream baosInMemory = new ByteArrayOutputStream(); DataOutputStream userDataStream = new DataOutputStream(baosInMemory); int batchId = numBatchesWritten++; HFileContext meta = new HFileContextBuilder() .withHBaseCheckSum(false) .withIncludesMvcc(false) .withIncludesTags(includesTag) .withCompression(Algorithm.NONE) .build(); HFileBlockEncodingContext blkEncodingCtx = new HFileBlockDefaultEncodingContext( DataBlockEncoding.PREFIX_TREE, new byte[0], meta); generateRandomTestData(kvset, batchId, includesTag, encoder, blkEncodingCtx, userDataStream); EncodedSeeker seeker = encoder.createSeeker(KeyValue.COMPARATOR, encoder.newDataBlockDecodingContext(meta)); byte[] onDiskBytes = baosInMemory.toByteArray(); ByteBuffer readBuffer = ByteBuffer.wrap(onDiskBytes, DataBlockEncoding.ID_SIZE, onDiskBytes.length - DataBlockEncoding.ID_SIZE); verifySeeking(seeker, readBuffer, batchId); }
@Test public void testSeekWithFixedData() throws Exception { PrefixTreeCodec encoder = new PrefixTreeCodec(); int batchId = numBatchesWritten++; HFileContext meta = new HFileContextBuilder() .withHBaseCheckSum(false) .withIncludesMvcc(false) .withIncludesTags(includesTag) .withCompression(Algorithm.NONE) .build(); HFileBlockEncodingContext blkEncodingCtx = new HFileBlockDefaultEncodingContext( DataBlockEncoding.PREFIX_TREE, new byte[0], meta); ByteArrayOutputStream baosInMemory = new ByteArrayOutputStream(); DataOutputStream userDataStream = new DataOutputStream(baosInMemory); generateFixedTestData(kvset, batchId, includesTag, encoder, blkEncodingCtx, userDataStream); EncodedSeeker seeker = encoder.createSeeker(KeyValue.COMPARATOR, encoder.newDataBlockDecodingContext(meta)); byte[] onDiskBytes = baosInMemory.toByteArray(); ByteBuffer readBuffer = ByteBuffer.wrap(onDiskBytes, DataBlockEncoding.ID_SIZE, onDiskBytes.length - DataBlockEncoding.ID_SIZE); verifySeeking(seeker, readBuffer, batchId); }
@Test public void testSeekWithRandomData() throws Exception { PrefixTreeCodec encoder = new PrefixTreeCodec(); int batchId = numBatchesWritten++; ByteBuffer dataBuffer = generateRandomTestData(kvset, batchId, includesTag); HFileContext meta = new HFileContextBuilder() .withHBaseCheckSum(false) .withIncludesMvcc(false) .withIncludesTags(includesTag) .withCompression(Algorithm.NONE) .build(); HFileBlockEncodingContext blkEncodingCtx = new HFileBlockDefaultEncodingContext( DataBlockEncoding.PREFIX_TREE, new byte[0], meta); encoder.encodeKeyValues(dataBuffer, blkEncodingCtx); EncodedSeeker seeker = encoder.createSeeker(KeyValue.COMPARATOR, encoder.newDataBlockDecodingContext(meta)); byte[] onDiskBytes = blkEncodingCtx.getOnDiskBytesWithHeader(); ByteBuffer readBuffer = ByteBuffer.wrap(onDiskBytes, DataBlockEncoding.ID_SIZE, onDiskBytes.length - DataBlockEncoding.ID_SIZE); verifySeeking(seeker, readBuffer, batchId); }
@Test public void testSeekWithFixedData() throws Exception { PrefixTreeCodec encoder = new PrefixTreeCodec(); int batchId = numBatchesWritten++; ByteBuffer dataBuffer = generateFixedTestData(kvset, batchId, includesTag); HFileContext meta = new HFileContextBuilder() .withHBaseCheckSum(false) .withIncludesMvcc(false) .withIncludesTags(includesTag) .withCompression(Algorithm.NONE) .build(); HFileBlockEncodingContext blkEncodingCtx = new HFileBlockDefaultEncodingContext( DataBlockEncoding.PREFIX_TREE, new byte[0], meta); encoder.encodeKeyValues(dataBuffer, blkEncodingCtx); EncodedSeeker seeker = encoder.createSeeker(KeyValue.COMPARATOR, encoder.newDataBlockDecodingContext(meta)); byte[] onDiskBytes = blkEncodingCtx.getOnDiskBytesWithHeader(); ByteBuffer readBuffer = ByteBuffer.wrap(onDiskBytes, DataBlockEncoding.ID_SIZE, onDiskBytes.length - DataBlockEncoding.ID_SIZE); verifySeeking(seeker, readBuffer, batchId); }
@Test public void testScanWithRandomData() throws Exception { PrefixTreeCodec encoder = new PrefixTreeCodec(); ByteBuffer dataBuffer = generateRandomTestData(kvset, numBatchesWritten++); HFileBlockEncodingContext blkEncodingCtx = new HFileBlockDefaultEncodingContext( Algorithm.NONE, DataBlockEncoding.PREFIX_TREE, new byte[0]); encoder.encodeKeyValues(dataBuffer, false, blkEncodingCtx); EncodedSeeker seeker = encoder.createSeeker(KeyValue.COMPARATOR, false); byte[] onDiskBytes=blkEncodingCtx.getOnDiskBytesWithHeader(); ByteBuffer readBuffer = ByteBuffer.wrap(onDiskBytes, DataBlockEncoding.ID_SIZE, onDiskBytes.length - DataBlockEncoding.ID_SIZE); seeker.setCurrentBuffer(readBuffer); KeyValue previousKV = null; do{ KeyValue currentKV = seeker.getKeyValue(); if (previousKV != null && KeyValue.COMPARATOR.compare(currentKV, previousKV) < 0) { dumpInputKVSet(); fail("Current kv " + currentKV + " is smaller than previous keyvalue " + previousKV); } previousKV = currentKV; } while (seeker.next()); }
@Test public void testSeekWithFixedData() throws Exception { PrefixTreeCodec encoder = new PrefixTreeCodec(); int batchId = numBatchesWritten++; ByteBuffer dataBuffer = generateFixedTestData(kvset, batchId); HFileBlockEncodingContext blkEncodingCtx = new HFileBlockDefaultEncodingContext( Algorithm.NONE, DataBlockEncoding.PREFIX_TREE, new byte[0]); encoder.encodeKeyValues(dataBuffer, false, blkEncodingCtx); EncodedSeeker seeker = encoder.createSeeker(KeyValue.COMPARATOR, false); byte[] onDiskBytes = blkEncodingCtx.getOnDiskBytesWithHeader(); ByteBuffer readBuffer = ByteBuffer.wrap(onDiskBytes, DataBlockEncoding.ID_SIZE, onDiskBytes.length - DataBlockEncoding.ID_SIZE); verifySeeking(seeker, readBuffer, batchId); }
@Test public void testScanWithRandomData() throws Exception { PrefixTreeCodec encoder = new PrefixTreeCodec(); ByteArrayOutputStream baosInMemory = new ByteArrayOutputStream(); DataOutputStream userDataStream = new DataOutputStream(baosInMemory); HFileContext meta = new HFileContextBuilder() .withHBaseCheckSum(false) .withIncludesMvcc(false) .withIncludesTags(includesTag) .withCompression(Algorithm.NONE) .build(); HFileBlockEncodingContext blkEncodingCtx = new HFileBlockDefaultEncodingContext( DataBlockEncoding.PREFIX_TREE, new byte[0], meta); generateRandomTestData(kvset, numBatchesWritten++, includesTag, encoder, blkEncodingCtx, userDataStream); EncodedSeeker seeker = encoder.createSeeker(KeyValue.COMPARATOR, encoder.newDataBlockDecodingContext(meta)); byte[] onDiskBytes = baosInMemory.toByteArray(); ByteBuffer readBuffer = ByteBuffer.wrap(onDiskBytes, DataBlockEncoding.ID_SIZE, onDiskBytes.length - DataBlockEncoding.ID_SIZE); seeker.setCurrentBuffer(readBuffer); Cell previousKV = null; do { Cell currentKV = seeker.getKeyValue(); System.out.println(currentKV); if (previousKV != null && KeyValue.COMPARATOR.compare(currentKV, previousKV) < 0) { dumpInputKVSet(); fail("Current kv " + currentKV + " is smaller than previous keyvalue " + previousKV); } if (!includesTag) { assertFalse(currentKV.getTagsLength() > 0); } else { Assert.assertTrue(currentKV.getTagsLength() > 0); } previousKV = currentKV; } while (seeker.next()); }
private void verifySeeking(EncodedSeeker encodeSeeker, ByteBuffer encodedData, int batchId) { List<KeyValue> kvList = new ArrayList<KeyValue>(); for (int i = 0; i < NUM_ROWS_PER_BATCH; ++i) { kvList.clear(); encodeSeeker.setCurrentBuffer(encodedData); KeyValue firstOnRow = KeyValueUtil.createFirstOnRow(getRowKey(batchId, i)); encodeSeeker.seekToKeyInBlock( new KeyValue.KeyOnlyKeyValue(firstOnRow.getBuffer(), firstOnRow.getKeyOffset(), firstOnRow.getKeyLength()), false); boolean hasMoreOfEncodeScanner = encodeSeeker.next(); CollectionBackedScanner collectionScanner = new CollectionBackedScanner( this.kvset); boolean hasMoreOfCollectionScanner = collectionScanner.seek(firstOnRow); if (hasMoreOfEncodeScanner != hasMoreOfCollectionScanner) { dumpInputKVSet(); fail("Get error result after seeking " + firstOnRow); } if (hasMoreOfEncodeScanner) { if (KeyValue.COMPARATOR.compare(encodeSeeker.getKeyValue(), collectionScanner.peek()) != 0) { dumpInputKVSet(); fail("Expected " + collectionScanner.peek() + " actual " + encodeSeeker.getKeyValue() + ", after seeking " + firstOnRow); } } } }
@Test public void testScanWithRandomData() throws Exception { PrefixTreeCodec encoder = new PrefixTreeCodec(); ByteBuffer dataBuffer = generateRandomTestData(kvset, numBatchesWritten++, includesTag); HFileContext meta = new HFileContextBuilder() .withHBaseCheckSum(false) .withIncludesMvcc(false) .withIncludesTags(includesTag) .withCompression(Algorithm.NONE) .build(); HFileBlockEncodingContext blkEncodingCtx = new HFileBlockDefaultEncodingContext( DataBlockEncoding.PREFIX_TREE, new byte[0], meta); encoder.encodeKeyValues(dataBuffer, blkEncodingCtx); EncodedSeeker seeker = encoder.createSeeker(KeyValue.COMPARATOR, encoder.newDataBlockDecodingContext(meta)); byte[] onDiskBytes = blkEncodingCtx.getOnDiskBytesWithHeader(); ByteBuffer readBuffer = ByteBuffer.wrap(onDiskBytes, DataBlockEncoding.ID_SIZE, onDiskBytes.length - DataBlockEncoding.ID_SIZE); seeker.setCurrentBuffer(readBuffer); KeyValue previousKV = null; do { KeyValue currentKV = seeker.getKeyValue(); System.out.println(currentKV); if (previousKV != null && KeyValue.COMPARATOR.compare(currentKV, previousKV) < 0) { dumpInputKVSet(); fail("Current kv " + currentKV + " is smaller than previous keyvalue " + previousKV); } if (!includesTag) { assertFalse(currentKV.getTagsLength() > 0); } else { Assert.assertTrue(currentKV.getTagsLength() > 0); } previousKV = currentKV; } while (seeker.next()); }
private void verifySeeking(EncodedSeeker encodeSeeker, ByteBuffer encodedData, int batchId) { List<KeyValue> kvList = new ArrayList<KeyValue>(); for (int i = 0; i < NUM_ROWS_PER_BATCH; ++i) { kvList.clear(); encodeSeeker.setCurrentBuffer(encodedData); KeyValue firstOnRow = KeyValue.createFirstOnRow(getRowKey(batchId, i)); encodeSeeker.seekToKeyInBlock(firstOnRow.getBuffer(), firstOnRow.getKeyOffset(), firstOnRow.getKeyLength(), false); boolean hasMoreOfEncodeScanner = encodeSeeker.next(); CollectionBackedScanner collectionScanner = new CollectionBackedScanner( this.kvset); boolean hasMoreOfCollectionScanner = collectionScanner.seek(firstOnRow); if (hasMoreOfEncodeScanner != hasMoreOfCollectionScanner) { dumpInputKVSet(); fail("Get error result after seeking " + firstOnRow); } if (hasMoreOfEncodeScanner) { if (KeyValue.COMPARATOR.compare(encodeSeeker.getKeyValue(), collectionScanner.peek()) != 0) { dumpInputKVSet(); fail("Expected " + collectionScanner.peek() + " actual " + encodeSeeker.getKeyValue() + ", after seeking " + firstOnRow); } } } }
@Test public void testSeekWithRandomData() throws Exception { PrefixTreeCodec encoder = new PrefixTreeCodec(); int batchId = numBatchesWritten++; ByteBuffer dataBuffer = generateRandomTestData(kvset, batchId); HFileBlockEncodingContext blkEncodingCtx = new HFileBlockDefaultEncodingContext( Algorithm.NONE, DataBlockEncoding.PREFIX_TREE, new byte[0]); encoder.encodeKeyValues(dataBuffer, false, blkEncodingCtx); EncodedSeeker seeker = encoder.createSeeker(KeyValue.COMPARATOR, false); byte[] onDiskBytes = blkEncodingCtx.getOnDiskBytesWithHeader(); ByteBuffer readBuffer = ByteBuffer.wrap(onDiskBytes, DataBlockEncoding.ID_SIZE, onDiskBytes.length - DataBlockEncoding.ID_SIZE); verifySeeking(seeker, readBuffer, batchId); }
@Test public void testSeekBeforeWithFixedData() throws Exception { formatRowNum = true; PrefixTreeCodec encoder = new PrefixTreeCodec(); int batchId = numBatchesWritten++; HFileContext meta = new HFileContextBuilder() .withHBaseCheckSum(false) .withIncludesMvcc(false) .withIncludesTags(includesTag) .withCompression(Algorithm.NONE).build(); HFileBlockEncodingContext blkEncodingCtx = new HFileBlockDefaultEncodingContext( DataBlockEncoding.PREFIX_TREE, new byte[0], meta); ByteArrayOutputStream baosInMemory = new ByteArrayOutputStream(); DataOutputStream userDataStream = new DataOutputStream(baosInMemory); generateFixedTestData(kvset, batchId, false, includesTag, encoder, blkEncodingCtx, userDataStream); EncodedSeeker seeker = encoder.createSeeker(KeyValue.COMPARATOR, encoder.newDataBlockDecodingContext(meta)); byte[] onDiskBytes = baosInMemory.toByteArray(); ByteBuffer readBuffer = ByteBuffer.wrap(onDiskBytes, DataBlockEncoding.ID_SIZE, onDiskBytes.length - DataBlockEncoding.ID_SIZE); seeker.setCurrentBuffer(readBuffer); // Seek before the first keyvalue; KeyValue seekKey = KeyValueUtil.createFirstDeleteFamilyOnRow(getRowKey(batchId, 0), CF_BYTES); seeker.seekToKeyInBlock( new KeyValue.KeyOnlyKeyValue(seekKey.getBuffer(), seekKey.getKeyOffset(), seekKey .getKeyLength()), true); assertEquals(null, seeker.getKeyValue()); // Seek before the middle keyvalue; seekKey = KeyValueUtil.createFirstDeleteFamilyOnRow(getRowKey(batchId, NUM_ROWS_PER_BATCH / 3), CF_BYTES); seeker.seekToKeyInBlock( new KeyValue.KeyOnlyKeyValue(seekKey.getBuffer(), seekKey.getKeyOffset(), seekKey .getKeyLength()), true); assertNotNull(seeker.getKeyValue()); assertArrayEquals(getRowKey(batchId, NUM_ROWS_PER_BATCH / 3 - 1), seeker.getKeyValue().getRow()); // Seek before the last keyvalue; seekKey = KeyValueUtil.createFirstDeleteFamilyOnRow(Bytes.toBytes("zzzz"), CF_BYTES); seeker.seekToKeyInBlock( new KeyValue.KeyOnlyKeyValue(seekKey.getBuffer(), seekKey.getKeyOffset(), seekKey .getKeyLength()), true); assertNotNull(seeker.getKeyValue()); assertArrayEquals(getRowKey(batchId, NUM_ROWS_PER_BATCH - 1), seeker.getKeyValue().getRow()); }
@Test public void testSeekBeforeWithFixedData() throws Exception { formatRowNum = true; PrefixTreeCodec encoder = new PrefixTreeCodec(); int batchId = numBatchesWritten++; ByteBuffer dataBuffer = generateFixedTestData(kvset, batchId, false, includesTag); HFileContext meta = new HFileContextBuilder() .withHBaseCheckSum(false) .withIncludesMvcc(false) .withIncludesTags(includesTag) .withCompression(Algorithm.NONE).build(); HFileBlockEncodingContext blkEncodingCtx = new HFileBlockDefaultEncodingContext( DataBlockEncoding.PREFIX_TREE, new byte[0], meta); encoder.encodeKeyValues(dataBuffer, blkEncodingCtx); EncodedSeeker seeker = encoder.createSeeker(KeyValue.COMPARATOR, encoder.newDataBlockDecodingContext(meta)); byte[] onDiskBytes = blkEncodingCtx.getOnDiskBytesWithHeader(); ByteBuffer readBuffer = ByteBuffer.wrap(onDiskBytes, DataBlockEncoding.ID_SIZE, onDiskBytes.length - DataBlockEncoding.ID_SIZE); seeker.setCurrentBuffer(readBuffer); // Seek before the first keyvalue; KeyValue seekKey = KeyValue.createFirstDeleteFamilyOnRow(getRowKey(batchId, 0), CF_BYTES); seeker.seekToKeyInBlock(seekKey.getBuffer(), seekKey.getKeyOffset(), seekKey.getKeyLength(), true); assertEquals(null, seeker.getKeyValue()); // Seek before the middle keyvalue; seekKey = KeyValue.createFirstDeleteFamilyOnRow(getRowKey(batchId, NUM_ROWS_PER_BATCH / 3), CF_BYTES); seeker.seekToKeyInBlock(seekKey.getBuffer(), seekKey.getKeyOffset(), seekKey.getKeyLength(), true); assertNotNull(seeker.getKeyValue()); assertArrayEquals(getRowKey(batchId, NUM_ROWS_PER_BATCH / 3 - 1), seeker.getKeyValue().getRow()); // Seek before the last keyvalue; seekKey = KeyValue.createFirstDeleteFamilyOnRow(Bytes.toBytes("zzzz"), CF_BYTES); seeker.seekToKeyInBlock(seekKey.getBuffer(), seekKey.getKeyOffset(), seekKey.getKeyLength(), true); assertNotNull(seeker.getKeyValue()); assertArrayEquals(getRowKey(batchId, NUM_ROWS_PER_BATCH - 1), seeker.getKeyValue().getRow()); }
@Test public void testSeekBeforeWithFixedData() throws Exception { formatRowNum = true; PrefixTreeCodec encoder = new PrefixTreeCodec(); int batchId = numBatchesWritten++; ByteBuffer dataBuffer = generateFixedTestData(kvset, batchId, false); HFileBlockEncodingContext blkEncodingCtx = new HFileBlockDefaultEncodingContext( Algorithm.NONE, DataBlockEncoding.PREFIX_TREE, new byte[0]); encoder.encodeKeyValues(dataBuffer, false, blkEncodingCtx); EncodedSeeker seeker = encoder.createSeeker(KeyValue.COMPARATOR, false); byte[] onDiskBytes = blkEncodingCtx.getOnDiskBytesWithHeader(); ByteBuffer readBuffer = ByteBuffer.wrap(onDiskBytes, DataBlockEncoding.ID_SIZE, onDiskBytes.length - DataBlockEncoding.ID_SIZE); seeker.setCurrentBuffer(readBuffer); // Seek before the first keyvalue; KeyValue seekKey = KeyValue.createFirstDeleteFamilyOnRow( getRowKey(batchId, 0), CF_BYTES); seeker.seekToKeyInBlock(seekKey.getBuffer(), seekKey.getKeyOffset(), seekKey.getKeyLength(), true); assertEquals(null, seeker.getKeyValue()); // Seek before the middle keyvalue; seekKey = KeyValue.createFirstDeleteFamilyOnRow( getRowKey(batchId, NUM_ROWS_PER_BATCH / 3), CF_BYTES); seeker.seekToKeyInBlock(seekKey.getBuffer(), seekKey.getKeyOffset(), seekKey.getKeyLength(), true); assertNotNull(seeker.getKeyValue()); assertArrayEquals(getRowKey(batchId, NUM_ROWS_PER_BATCH / 3 - 1), seeker .getKeyValue().getRow()); // Seek before the last keyvalue; seekKey = KeyValue.createFirstDeleteFamilyOnRow(Bytes.toBytes("zzzz"), CF_BYTES); seeker.seekToKeyInBlock(seekKey.getBuffer(), seekKey.getKeyOffset(), seekKey.getKeyLength(), true); assertNotNull(seeker.getKeyValue()); assertArrayEquals(getRowKey(batchId, NUM_ROWS_PER_BATCH - 1), seeker .getKeyValue().getRow()); }