private void doTest(boolean compressTags) throws Exception { Configuration conf = new Configuration(false); conf.setBoolean(CompressionContext.ENABLE_WAL_TAGS_COMPRESSION, compressTags); WALCellCodec codec = new WALCellCodec(conf, new CompressionContext(LRUDictionary.class, false, compressTags)); ByteArrayOutputStream bos = new ByteArrayOutputStream(1024); Encoder encoder = codec.getEncoder(bos); encoder.write(createKV(1)); encoder.write(createKV(0)); encoder.write(createKV(2)); InputStream is = new ByteArrayInputStream(bos.toByteArray()); Decoder decoder = codec.getDecoder(is); decoder.advance(); KeyValue kv = (KeyValue) decoder.current(); List<Tag> tags = kv.getTags(); assertEquals(1, tags.size()); assertEquals("tagValue1", Bytes.toString(tags.get(0).getValue())); decoder.advance(); kv = (KeyValue) decoder.current(); tags = kv.getTags(); assertEquals(0, tags.size()); decoder.advance(); kv = (KeyValue) decoder.current(); tags = kv.getTags(); assertEquals(2, tags.size()); assertEquals("tagValue1", Bytes.toString(tags.get(0).getValue())); assertEquals("tagValue2", Bytes.toString(tags.get(1).getValue())); }
private void doTest(boolean compressTags, boolean offheapKV) throws Exception { Configuration conf = new Configuration(false); conf.setBoolean(CompressionContext.ENABLE_WAL_TAGS_COMPRESSION, compressTags); WALCellCodec codec = new WALCellCodec(conf, new CompressionContext(LRUDictionary.class, false, compressTags)); ByteArrayOutputStream bos = new ByteArrayOutputStream(1024); Encoder encoder = codec.getEncoder(bos); if (offheapKV) { encoder.write(createOffheapKV(1)); encoder.write(createOffheapKV(0)); encoder.write(createOffheapKV(2)); } else { encoder.write(createKV(1)); encoder.write(createKV(0)); encoder.write(createKV(2)); } InputStream is = new ByteArrayInputStream(bos.toByteArray()); Decoder decoder = codec.getDecoder(is); decoder.advance(); KeyValue kv = (KeyValue) decoder.current(); List<Tag> tags = PrivateCellUtil.getTags(kv); assertEquals(1, tags.size()); assertEquals("tagValue1", Bytes.toString(Tag.cloneValue(tags.get(0)))); decoder.advance(); kv = (KeyValue) decoder.current(); tags = PrivateCellUtil.getTags(kv); assertEquals(0, tags.size()); decoder.advance(); kv = (KeyValue) decoder.current(); tags = PrivateCellUtil.getTags(kv); assertEquals(2, tags.size()); assertEquals("tagValue1", Bytes.toString(Tag.cloneValue(tags.get(0)))); assertEquals("tagValue2", Bytes.toString(Tag.cloneValue(tags.get(1)))); }
@Test public void testKVCodecWithTagsForDecodedCellsWithNoTags() throws Exception { KeyValue kv1 = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("1"), HConstants.LATEST_TIMESTAMP, Bytes.toBytes("1")); // kv1.getKey() return a copy of the Key bytes which starts from RK_length. Means from offsets, // we need to reduce the KL and VL parts. OnheapDecodedCell c1 = new OnheapDecodedCell(kv1.getKey(), kv1.getRowLength(), kv1.getFamilyOffset() - KeyValue.ROW_OFFSET, kv1.getFamilyLength(), kv1.getQualifierOffset() - KeyValue.ROW_OFFSET, kv1.getQualifierLength(), kv1.getTimestamp(), kv1.getTypeByte(), kv1.getValueArray(), kv1.getValueOffset(), kv1.getValueLength(), kv1.getSequenceId(), kv1.getTagsArray(), kv1.getTagsOffset(), kv1.getTagsLength()); KeyValue kv2 = new KeyValue(Bytes.toBytes("r2"), Bytes.toBytes("f"), Bytes.toBytes("2"), HConstants.LATEST_TIMESTAMP, Bytes.toBytes("2")); OnheapDecodedCell c2 = new OnheapDecodedCell(kv2.getKey(), kv2.getRowLength(), kv2.getFamilyOffset() - KeyValue.ROW_OFFSET, kv2.getFamilyLength(), kv2.getQualifierOffset() - KeyValue.ROW_OFFSET, kv2.getQualifierLength(), kv2.getTimestamp(), kv2.getTypeByte(), kv2.getValueArray(), kv2.getValueOffset(), kv2.getValueLength(), kv2.getSequenceId(), kv2.getTagsArray(), kv2.getTagsOffset(), kv2.getTagsLength()); KeyValue kv3 = new KeyValue(Bytes.toBytes("r3"), Bytes.toBytes("cf"), Bytes.toBytes("qual"), HConstants.LATEST_TIMESTAMP, Bytes.toBytes("3")); BufferedDataBlockEncoder.OffheapDecodedExtendedCell c3 = new BufferedDataBlockEncoder.OffheapDecodedExtendedCell(ByteBuffer.wrap(kv2.getKey()), kv2.getRowLength(), kv2.getFamilyOffset() - KeyValue.ROW_OFFSET, kv2.getFamilyLength(), kv2.getQualifierOffset() - KeyValue.ROW_OFFSET, kv2.getQualifierLength(), kv2.getTimestamp(), kv2.getTypeByte(), ByteBuffer.wrap(kv2.getValueArray()), kv2.getValueOffset(), kv2.getValueLength(), kv2.getSequenceId(), ByteBuffer.wrap(kv2.getTagsArray()), kv2.getTagsOffset(), kv2.getTagsLength()); ByteArrayOutputStream os = new ByteArrayOutputStream(); KeyValueCodecWithTags codec = new KeyValueCodecWithTags(); Encoder encoder = codec.getEncoder(os); encoder.write(c1); encoder.write(c2); encoder.write(c3); ByteArrayInputStream is = new ByteArrayInputStream(os.toByteArray()); Decoder decoder = codec.getDecoder(is); assertTrue(decoder.advance()); assertTrue(CellUtil.equals(c1, decoder.current())); assertTrue(decoder.advance()); assertTrue(CellUtil.equals(c2, decoder.current())); assertTrue(decoder.advance()); assertTrue(CellUtil.equals(c3, decoder.current())); assertFalse(decoder.advance()); }