private static byte[] uncompressByteString(ByteString bs, Dictionary dict) throws IOException { InputStream in = bs.newInput(); byte status = (byte)in.read(); if (status == Dictionary.NOT_IN_DICTIONARY) { byte[] arr = new byte[StreamUtils.readRawVarint32(in)]; int bytesRead = in.read(arr); if (bytesRead != arr.length) { throw new IOException("Cannot read; wanted " + arr.length + ", but got " + bytesRead); } if (dict != null) dict.addEntry(arr, 0, arr.length); return arr; } else { // Status here is the higher-order byte of index of the dictionary entry. short dictIdx = StreamUtils.toShort(status, (byte)in.read()); byte[] entry = dict.getEntry(dictIdx); if (entry == null) { throw new IOException("Missing dictionary entry for index " + dictIdx); } return entry; } }
private int readIntoArray(byte[] to, int offset, Dictionary dict) throws IOException { byte status = (byte)in.read(); if (status == Dictionary.NOT_IN_DICTIONARY) { // status byte indicating that data to be read is not in dictionary. // if this isn't in the dictionary, we need to add to the dictionary. int length = StreamUtils.readRawVarint32(in); IOUtils.readFully(in, to, offset, length); dict.addEntry(to, offset, length); return length; } else { // the status byte also acts as the higher order byte of the dictionary entry. short dictIdx = StreamUtils.toShort(status, (byte)in.read()); byte[] entry = dict.getEntry(dictIdx); if (entry == null) { throw new IOException("Missing dictionary entry for index " + dictIdx); } // now we write the uncompressed value. Bytes.putBytes(to, offset, entry, 0, entry.length); return entry.length; } }
public static void writeTrailer(FSDataOutputStream stream, ProcedureStoreTracker tracker) throws IOException { long offset = stream.getPos(); // Write EOF Entry ProcedureWALEntry.newBuilder() .setType(ProcedureWALEntry.Type.EOF) .build().writeDelimitedTo(stream); // Write Tracker tracker.writeTo(stream); stream.write(TRAILER_VERSION); StreamUtils.writeLong(stream, TRAILER_MAGIC); StreamUtils.writeLong(stream, offset); }
/** * Uncompress tags from the InputStream and writes to the destination array. * @param src Stream where the compressed tags are available * @param dest Destination array where to write the uncompressed tags * @param offset Offset in destination where tags to be written * @param length Length of all tag bytes * @throws IOException */ public void uncompressTags(InputStream src, byte[] dest, int offset, int length) throws IOException { int endOffset = offset + length; while (offset < endOffset) { byte status = (byte) src.read(); if (status == Dictionary.NOT_IN_DICTIONARY) { int tagLen = StreamUtils.readRawVarint32(src); offset = Bytes.putAsShort(dest, offset, tagLen); IOUtils.readFully(src, dest, offset, tagLen); tagDict.addEntry(dest, offset, tagLen); offset += tagLen; } else { short dictIdx = StreamUtils.toShort(status, (byte) src.read()); byte[] entry = tagDict.getEntry(dictIdx); if (entry == null) { throw new IOException("Missing dictionary entry for index " + dictIdx); } offset = Bytes.putAsShort(dest, offset, entry.length); System.arraycopy(entry, 0, dest, offset, entry.length); offset += entry.length; } } }
/** * Uncompress tags from the InputStream and writes to the destination array. * @param src Stream where the compressed tags are available * @param dest Destination array where to write the uncompressed tags * @param offset Offset in destination where tags to be written * @param length Length of all tag bytes * @throws IOException */ public void uncompressTags(InputStream src, byte[] dest, int offset, short length) throws IOException { int endOffset = offset + length; while (offset < endOffset) { byte status = (byte) src.read(); if (status == Dictionary.NOT_IN_DICTIONARY) { // We are writing short as tagLen. So can downcast this without any risk. short tagLen = (short) StreamUtils.readRawVarint32(src); offset = Bytes.putShort(dest, offset, tagLen); IOUtils.readFully(src, dest, offset, tagLen); tagDict.addEntry(dest, offset, tagLen); offset += tagLen; } else { short dictIdx = StreamUtils.toShort(status, (byte) src.read()); byte[] entry = tagDict.getEntry(dictIdx); if (entry == null) { throw new IOException("Missing dictionary entry for index " + dictIdx); } offset = Bytes.putShort(dest, offset, (short) entry.length); System.arraycopy(entry, 0, dest, offset, entry.length); offset += entry.length; } } }
@Override public void write(Cell cell) throws IOException { // We first write the KeyValue infrastructure as VInts. StreamUtils.writeRawVInt32(out, KeyValueUtil.keyLength(cell)); StreamUtils.writeRawVInt32(out, cell.getValueLength()); // To support tags int tagsLength = cell.getTagsLength(); StreamUtils.writeRawVInt32(out, tagsLength); PrivateCellUtil.compressRow(out, cell, compression.rowDict); PrivateCellUtil.compressFamily(out, cell, compression.familyDict); PrivateCellUtil.compressQualifier(out, cell, compression.qualifierDict); // Write timestamp, type and value as uncompressed. StreamUtils.writeLong(out, cell.getTimestamp()); out.write(cell.getTypeByte()); PrivateCellUtil.writeValue(out, cell, cell.getValueLength()); if (tagsLength > 0) { if (compression.tagCompressionContext != null) { // Write tags using Dictionary compression PrivateCellUtil.compressTags(out, cell, compression.tagCompressionContext); } else { // Tag compression is disabled within the WAL compression. Just write the tags bytes as // it is. PrivateCellUtil.writeTags(out, cell, tagsLength); } } }
public static long writeTrailer(FSDataOutputStream stream, ProcedureStoreTracker tracker) throws IOException { long offset = stream.getPos(); // Write EOF Entry ProcedureWALEntry.newBuilder() .setType(ProcedureWALEntry.Type.PROCEDURE_WAL_EOF) .build().writeDelimitedTo(stream); // Write Tracker tracker.toProto().writeDelimitedTo(stream); stream.write(TRAILER_VERSION); StreamUtils.writeLong(stream, TRAILER_MAGIC); StreamUtils.writeLong(stream, offset); return stream.getPos() - offset; }
private static void getSortedTagOrdinals(List<List<Integer>> fullTagsList, Tag tag) throws IOException { List<Integer> tagsOrdinalInSortedOrder = new ArrayList<Integer>(); int offset = tag.getTagOffset(); int endOffset = offset + tag.getTagLength(); while (offset < endOffset) { Pair<Integer, Integer> result = StreamUtils.readRawVarint32(tag.getBuffer(), offset); tagsOrdinalInSortedOrder.add(result.getFirst()); offset += result.getSecond(); } Collections.sort(tagsOrdinalInSortedOrder); fullTagsList.add(tagsOrdinalInSortedOrder); }
@Override public void write(Cell cell) throws IOException { // We first write the KeyValue infrastructure as VInts. StreamUtils.writeRawVInt32(out, KeyValueUtil.keyLength(cell)); StreamUtils.writeRawVInt32(out, cell.getValueLength()); // To support tags int tagsLength = cell.getTagsLength(); StreamUtils.writeRawVInt32(out, tagsLength); // Write row, qualifier, and family; use dictionary // compression as they're likely to have duplicates. write(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), compression.rowDict); write(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(), compression.familyDict); write(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(), compression.qualifierDict); // Write timestamp, type and value as uncompressed. StreamUtils.writeLong(out, cell.getTimestamp()); out.write(cell.getTypeByte()); out.write(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()); if (tagsLength > 0) { if (compression.tagCompressionContext != null) { // Write tags using Dictionary compression compression.tagCompressionContext.compressTags(out, cell.getTagsArray(), cell.getTagsOffset(), tagsLength); } else { // Tag compression is disabled within the WAL compression. Just write the tags bytes as // it is. out.write(cell.getTagsArray(), cell.getTagsOffset(), tagsLength); } } }
private void write(byte[] data, int offset, int length, Dictionary dict) throws IOException { short dictIdx = Dictionary.NOT_IN_DICTIONARY; if (dict != null) { dictIdx = dict.findEntry(data, offset, length); } if (dictIdx == Dictionary.NOT_IN_DICTIONARY) { out.write(Dictionary.NOT_IN_DICTIONARY); StreamUtils.writeRawVInt32(out, length); out.write(data, offset, length); } else { StreamUtils.writeShort(out, dictIdx); } }
public static ProcedureWALTrailer readTrailer(FSDataInputStream stream, long startPos, long size) throws IOException { long trailerPos = size - 17; // Beginning of the Trailer Jump if (trailerPos < startPos) { throw new InvalidWALDataException("Missing trailer: size=" + size + " startPos=" + startPos); } stream.seek(trailerPos); int version = stream.read(); if (version != TRAILER_VERSION) { throw new InvalidWALDataException("Invalid Trailer version. got " + version + " expected " + TRAILER_VERSION); } long magic = StreamUtils.readLong(stream); if (magic != TRAILER_MAGIC) { throw new InvalidWALDataException("Invalid Trailer magic. got " + magic + " expected " + TRAILER_MAGIC); } long trailerOffset = StreamUtils.readLong(stream); stream.seek(trailerOffset); ProcedureWALEntry entry = readEntry(stream); if (entry.getType() != ProcedureWALEntry.Type.EOF) { throw new InvalidWALDataException("Invalid Trailer begin"); } ProcedureWALTrailer trailer = ProcedureWALTrailer.newBuilder() .setVersion(version) .setTrackerPos(stream.getPos()) .build(); return trailer; }
/** * Write out a KeyValue in the manner in which we used to when KeyValue was a Writable but do * not require a {@link DataOutput}, just take plain {@link OutputStream} * Named <code>oswrite</code> so does not clash with {@link #write(KeyValue, DataOutput)} * @param kv * @param out * @param withTags * @return Length written on stream * @throws IOException * @see #create(DataInput) for the inverse function * @see #write(KeyValue, DataOutput) * @see KeyValueUtil#oswrite(Cell, OutputStream, boolean) */ public static long oswrite(final KeyValue kv, final OutputStream out, final boolean withTags) throws IOException { // In KeyValueUtil#oswrite we do a Cell serialization as KeyValue. Any changes doing here, pls // check KeyValueUtil#oswrite also and do necessary changes. int length = kv.getLength(); if (!withTags) { length = kv.getKeyLength() + kv.getValueLength() + KEYVALUE_INFRASTRUCTURE_SIZE; } // This does same as DataOuput#writeInt (big-endian, etc.) StreamUtils.writeInt(out, length); out.write(kv.getBuffer(), kv.getOffset(), length); return length + Bytes.SIZEOF_INT; }
/** * Uncompress tags from the input ByteBuffer and writes to the destination array. * @param src Buffer where the compressed tags are available * @param dest Destination array where to write the uncompressed tags * @param offset Offset in destination where tags to be written * @param length Length of all tag bytes * @return bytes count read from source to uncompress all tags. * @throws IOException */ public int uncompressTags(ByteBuffer src, byte[] dest, int offset, int length) throws IOException { int srcBeginPos = src.position(); int endOffset = offset + length; while (offset < endOffset) { byte status = src.get(); int tagLen; if (status == Dictionary.NOT_IN_DICTIONARY) { tagLen = StreamUtils.readRawVarint32(src); offset = Bytes.putAsShort(dest, offset, tagLen); src.get(dest, offset, tagLen); tagDict.addEntry(dest, offset, tagLen); offset += tagLen; } else { short dictIdx = StreamUtils.toShort(status, src.get()); byte[] entry = tagDict.getEntry(dictIdx); if (entry == null) { throw new IOException("Missing dictionary entry for index " + dictIdx); } tagLen = entry.length; offset = Bytes.putAsShort(dest, offset, tagLen); System.arraycopy(entry, 0, dest, offset, tagLen); offset += tagLen; } } return src.position() - srcBeginPos; }
private void write(byte[] data, int offset, int length, OutputStream out) throws IOException { short dictIdx = Dictionary.NOT_IN_DICTIONARY; if (tagDict != null) { dictIdx = tagDict.findEntry(data, offset, length); } if (dictIdx == Dictionary.NOT_IN_DICTIONARY) { out.write(Dictionary.NOT_IN_DICTIONARY); StreamUtils.writeRawVInt32(out, length); out.write(data, offset, length); } else { StreamUtils.writeShort(out, dictIdx); } }
private void writeLabelOrdinalsToStream(ExpressionNode node, DataOutputStream dos) throws IOException, InvalidLabelException { if (node.isSingleNode()) { String identifier = null; int labelOrdinal = 0; if (node instanceof LeafExpressionNode) { identifier = ((LeafExpressionNode) node) .getIdentifier(); if (LOG.isTraceEnabled()) { LOG.trace("The identifier is "+identifier); } labelOrdinal = this.visibilityManager.getLabelOrdinal(identifier); } else { // This is a NOT node. LeafExpressionNode lNode = (LeafExpressionNode) ((NonLeafExpressionNode) node) .getChildExps().get(0); identifier = lNode.getIdentifier(); labelOrdinal = this.visibilityManager.getLabelOrdinal(identifier); labelOrdinal = -1 * labelOrdinal; // Store NOT node as -ve ordinal. } if (labelOrdinal == 0) { throw new InvalidLabelException("Invalid visibility label " + identifier); } StreamUtils.writeRawVInt32(dos, labelOrdinal); } else { List<ExpressionNode> childExps = ((NonLeafExpressionNode) node).getChildExps(); for (ExpressionNode child : childExps) { writeLabelOrdinalsToStream(child, dos); } } }
/** * Uncompress tags from the input ByteBuffer and writes to the destination array. * @param src Buffer where the compressed tags are available * @param dest Destination array where to write the uncompressed tags * @param offset Offset in destination where tags to be written * @param length Length of all tag bytes * @return bytes count read from source to uncompress all tags. * @throws IOException */ public int uncompressTags(ByteBuffer src, byte[] dest, int offset, int length) throws IOException { int srcBeginPos = src.position(); int endOffset = offset + length; while (offset < endOffset) { byte status = src.get(); short tagLen; if (status == Dictionary.NOT_IN_DICTIONARY) { // We are writing short as tagLen. So can downcast this without any risk. tagLen = (short) StreamUtils.readRawVarint32(src); offset = Bytes.putShort(dest, offset, tagLen); src.get(dest, offset, tagLen); tagDict.addEntry(dest, offset, tagLen); offset += tagLen; } else { short dictIdx = StreamUtils.toShort(status, src.get()); byte[] entry = tagDict.getEntry(dictIdx); if (entry == null) { throw new IOException("Missing dictionary entry for index " + dictIdx); } tagLen = (short) entry.length; offset = Bytes.putShort(dest, offset, tagLen); System.arraycopy(entry, 0, dest, offset, tagLen); offset += tagLen; } } return src.position() - srcBeginPos; }
private void write(byte[] data, int offset, short length, OutputStream out) throws IOException { short dictIdx = Dictionary.NOT_IN_DICTIONARY; if (tagDict != null) { dictIdx = tagDict.findEntry(data, offset, length); } if (dictIdx == Dictionary.NOT_IN_DICTIONARY) { out.write(Dictionary.NOT_IN_DICTIONARY); StreamUtils.writeRawVInt32(out, length); out.write(data, offset, length); } else { StreamUtils.writeShort(out, dictIdx); } }
public static ProcedureWALTrailer readTrailer(FSDataInputStream stream, long startPos, long size) throws IOException { // Beginning of the Trailer Jump. 17 = 1 byte version + 8 byte magic + 8 byte offset long trailerPos = size - 17; if (trailerPos < startPos) { throw new InvalidWALDataException("Missing trailer: size=" + size + " startPos=" + startPos); } stream.seek(trailerPos); int version = stream.read(); if (version != TRAILER_VERSION) { throw new InvalidWALDataException("Invalid Trailer version. got " + version + " expected " + TRAILER_VERSION); } long magic = StreamUtils.readLong(stream); if (magic != TRAILER_MAGIC) { throw new InvalidWALDataException("Invalid Trailer magic. got " + magic + " expected " + TRAILER_MAGIC); } long trailerOffset = StreamUtils.readLong(stream); stream.seek(trailerOffset); ProcedureWALEntry entry = readEntry(stream); if (entry.getType() != ProcedureWALEntry.Type.PROCEDURE_WAL_EOF) { throw new InvalidWALDataException("Invalid Trailer begin"); } ProcedureWALTrailer trailer = ProcedureWALTrailer.newBuilder() .setVersion(version) .setTrackerPos(stream.getPos()) .build(); return trailer; }
public static int writeFlatKey(Cell cell, OutputStream out) throws IOException { short rowLen = cell.getRowLength(); byte fLen = cell.getFamilyLength(); int qLen = cell.getQualifierLength(); // Using just one if/else loop instead of every time checking before writing every // component of cell if (cell instanceof ByteBufferExtendedCell) { StreamUtils.writeShort(out, rowLen); ByteBufferUtils.copyBufferToStream(out, ((ByteBufferExtendedCell) cell).getRowByteBuffer(), ((ByteBufferExtendedCell) cell).getRowPosition(), rowLen); out.write(fLen); ByteBufferUtils.copyBufferToStream(out, ((ByteBufferExtendedCell) cell).getFamilyByteBuffer(), ((ByteBufferExtendedCell) cell).getFamilyPosition(), fLen); ByteBufferUtils .copyBufferToStream(out, ((ByteBufferExtendedCell) cell).getQualifierByteBuffer(), ((ByteBufferExtendedCell) cell).getQualifierPosition(), qLen); } else { StreamUtils.writeShort(out, rowLen); out.write(cell.getRowArray(), cell.getRowOffset(), rowLen); out.write(fLen); out.write(cell.getFamilyArray(), cell.getFamilyOffset(), fLen); out.write(cell.getQualifierArray(), cell.getQualifierOffset(), qLen); } StreamUtils.writeLong(out, cell.getTimestamp()); out.write(cell.getTypeByte()); return Bytes.SIZEOF_SHORT + rowLen + Bytes.SIZEOF_BYTE + fLen + qLen + Bytes.SIZEOF_LONG + Bytes.SIZEOF_BYTE; }
/** * Put in output stream 32 bit integer (Big Endian byte order). * @param out Where to put integer. * @param value Value of integer. * @throws IOException On stream error. */ public static void putInt(OutputStream out, final int value) throws IOException { // We have writeInt in ByteBufferOutputStream so that it can directly write // int to underlying // ByteBuffer in one step. if (out instanceof ByteBufferWriter) { ((ByteBufferWriter) out).writeInt(value); } else { StreamUtils.writeInt(out, value); } }
/** * Uncompress tags from the input ByteBuffer and writes to the destination array. * @param src Buffer where the compressed tags are available * @param dest Destination array where to write the uncompressed tags * @param offset Offset in destination where tags to be written * @param length Length of all tag bytes * @return bytes count read from source to uncompress all tags. * @throws IOException */ public int uncompressTags(ByteBuff src, byte[] dest, int offset, int length) throws IOException { int srcBeginPos = src.position(); int endOffset = offset + length; while (offset < endOffset) { byte status = src.get(); int tagLen; if (status == Dictionary.NOT_IN_DICTIONARY) { tagLen = StreamUtils.readRawVarint32(src); offset = Bytes.putAsShort(dest, offset, tagLen); src.get(dest, offset, tagLen); tagDict.addEntry(dest, offset, tagLen); offset += tagLen; } else { short dictIdx = StreamUtils.toShort(status, src.get()); byte[] entry = tagDict.getEntry(dictIdx); if (entry == null) { throw new IOException("Missing dictionary entry for index " + dictIdx); } tagLen = entry.length; offset = Bytes.putAsShort(dest, offset, tagLen); System.arraycopy(entry, 0, dest, offset, tagLen); offset += tagLen; } } return src.position() - srcBeginPos; }
@Override public void startBlockEncoding(HFileBlockEncodingContext blkEncodingCtx, DataOutputStream out) throws IOException { if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) { throw new IOException (this.getClass().getName() + " only accepts " + HFileBlockDefaultEncodingContext.class.getName() + " as the " + "encoding context."); } HFileBlockDefaultEncodingContext encodingCtx = (HFileBlockDefaultEncodingContext) blkEncodingCtx; encodingCtx.prepareEncoding(out); if (encodingCtx.getHFileContext().isIncludesTags() && encodingCtx.getHFileContext().isCompressTags()) { if (encodingCtx.getTagCompressionContext() != null) { // It will be overhead to create the TagCompressionContext again and again for every block // encoding. encodingCtx.getTagCompressionContext().clear(); } else { try { TagCompressionContext tagCompressionContext = new TagCompressionContext( LRUDictionary.class, Byte.MAX_VALUE); encodingCtx.setTagCompressionContext(tagCompressionContext); } catch (Exception e) { throw new IOException("Failed to initialize TagCompressionContext", e); } } } StreamUtils.writeInt(out, 0); // DUMMY length. This will be updated in endBlockEncoding() blkEncodingCtx.setEncodingState(new BufferedDataBlockEncodingState()); }