/** Constructor that takes a path, creates and closes the output stream. */ public HFileWriterV2(Configuration conf, CacheConfig cacheConf, FileSystem fs, Path path, FSDataOutputStream ostream, int blockSize, Compression.Algorithm compressAlgo, HFileDataBlockEncoder blockEncoder, final KeyComparator comparator, final ChecksumType checksumType, final int bytesPerChecksum, boolean includeMVCCReadpoint) throws IOException { super(cacheConf, ostream == null ? createOutputStream(conf, fs, path) : ostream, path, blockSize, compressAlgo, blockEncoder, comparator); SchemaMetrics.configureGlobally(conf); this.checksumType = checksumType; this.bytesPerChecksum = bytesPerChecksum; this.includeMemstoreTS = includeMVCCReadpoint; if (!conf.getBoolean(HConstants.HBASE_CHECKSUM_VERIFICATION, false)) { this.minorVersion = 0; } finishInit(conf); }
public void testComparator() throws IOException { if (cacheConf == null) cacheConf = new CacheConfig(conf); Path mFile = new Path(ROOT_DIR, "meta.tfile"); FSDataOutputStream fout = createFSOutput(mFile); KeyComparator comparator = new KeyComparator() { @Override public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { return -Bytes.compareTo(b1, s1, l1, b2, s2, l2); } @Override public int compare(byte[] o1, byte[] o2) { return compare(o1, 0, o1.length, o2, 0, o2.length); } }; Writer writer = HFile.getWriterFactory(conf, cacheConf) .withOutputStream(fout) .withBlockSize(minBlockSize) .withComparator(comparator) .create(); writer.append("3".getBytes(), "0".getBytes()); writer.append("2".getBytes(), "0".getBytes()); writer.append("1".getBytes(), "0".getBytes()); writer.close(); }
public AbstractHFileWriter(CacheConfig cacheConf, FSDataOutputStream outputStream, Path path, int blockSize, Compression.Algorithm compressAlgo, HFileDataBlockEncoder dataBlockEncoder, KeyComparator comparator) { super(null, path); this.outputStream = outputStream; this.path = path; this.name = path != null ? path.getName() : outputStream.toString(); this.blockSize = blockSize; this.compressAlgo = compressAlgo == null ? HFile.DEFAULT_COMPRESSION_ALGORITHM : compressAlgo; this.blockEncoder = dataBlockEncoder != null ? dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE; this.comparator = comparator != null ? comparator : Bytes.BYTES_RAWCOMPARATOR; closeOutputStream = path != null; this.cacheConf = cacheConf; }
public AbstractHFileWriter(CacheConfig cacheConf, FSDataOutputStream outputStream, Path path, int blockSize, Compression.Algorithm compressAlgo, HFileDataBlockEncoder dataBlockEncoder, KeyComparator comparator) { this.outputStream = outputStream; this.path = path; this.name = path != null ? path.getName() : outputStream.toString(); this.blockSize = blockSize; this.compressAlgo = compressAlgo == null ? HFile.DEFAULT_COMPRESSION_ALGORITHM : compressAlgo; this.blockEncoder = dataBlockEncoder != null ? dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE; this.comparator = comparator != null ? comparator : Bytes.BYTES_RAWCOMPARATOR; closeOutputStream = path != null; this.cacheConf = cacheConf; }
@Override public Writer createWriter(FileSystem fs, Path path, FSDataOutputStream ostream, int blockSize, Compression.Algorithm compress, HFileDataBlockEncoder blockEncoder, final KeyComparator comparator, final ChecksumType checksumType, final int bytesPerChecksum, boolean includeMVCCReadpoint) throws IOException { return new HFileWriterV2(conf, cacheConf, fs, path, ostream, blockSize, compress, blockEncoder, comparator, checksumType, bytesPerChecksum, includeMVCCReadpoint); }
@Override public Writer createWriter(FileSystem fs, Path path, FSDataOutputStream ostream, int blockSize, Algorithm compressAlgo, HFileDataBlockEncoder dataBlockEncoder, KeyComparator comparator, final ChecksumType checksumType, final int bytesPerChecksum, boolean includeMVCCReadpoint) throws IOException { // version 1 does not implement checksums return new HFileWriterV1(conf, cacheConf, fs, path, ostream, blockSize, compressAlgo, dataBlockEncoder, comparator); }
/** Constructor that takes a path, creates and closes the output stream. */ public HFileWriterV1(Configuration conf, CacheConfig cacheConf, FileSystem fs, Path path, FSDataOutputStream ostream, int blockSize, Compression.Algorithm compress, HFileDataBlockEncoder blockEncoder, final KeyComparator comparator) throws IOException { super(cacheConf, ostream == null ? createOutputStream(conf, fs, path) : ostream, path, blockSize, compress, blockEncoder, comparator); SchemaMetrics.configureGlobally(conf); }
@Override public Writer createWriter(FileSystem fs, Path path, int blockSize, Compression.Algorithm compress, final KeyComparator comparator) throws IOException { return new HFileWriterV2(conf, cacheConf, fs, path, blockSize, compress, comparator); }
@Override public Writer createWriter(FileSystem fs, Path path, int blockSize, String compress, final KeyComparator comparator) throws IOException { return new HFileWriterV2(conf, cacheConf, fs, path, blockSize, compress, comparator); }
@Override public Writer createWriter(final FSDataOutputStream ostream, final int blockSize, final String compress, final KeyComparator comparator) throws IOException { return new HFileWriterV2(conf, cacheConf, ostream, blockSize, compress, comparator); }
@Override public Writer createWriter(final FSDataOutputStream ostream, final int blockSize, final Compression.Algorithm compress, final KeyComparator c) throws IOException { return new HFileWriterV2(conf, cacheConf, ostream, blockSize, compress, c); }
/** * Constructor that takes a path, creates and closes the output stream. Takes * compression algorithm name as string. */ public HFileWriterV2(Configuration conf, CacheConfig cacheConf, FileSystem fs, Path path, int blockSize, String compressAlgoName, final KeyComparator comparator) throws IOException { this(conf, cacheConf, fs, path, blockSize, compressionByName(compressAlgoName), comparator); }
/** Constructor that takes a path, creates and closes the output stream. */ public HFileWriterV2(Configuration conf, CacheConfig cacheConf, FileSystem fs, Path path, int blockSize, Compression.Algorithm compressAlgo, final KeyComparator comparator) throws IOException { super(cacheConf, createOutputStream(conf, fs, path), path, blockSize, compressAlgo, comparator); finishInit(conf); }
/** Constructor that takes a stream. */ public HFileWriterV2(final Configuration conf, final CacheConfig cacheConf, final FSDataOutputStream outputStream, final int blockSize, final String compressAlgoName, final KeyComparator comparator) throws IOException { this(conf, cacheConf, outputStream, blockSize, Compression.getCompressionAlgorithmByName(compressAlgoName), comparator); }
/** Constructor that takes a stream. */ public HFileWriterV2(final Configuration conf, final CacheConfig cacheConf, final FSDataOutputStream outputStream, final int blockSize, final Compression.Algorithm compress, final KeyComparator comparator) throws IOException { super(cacheConf, outputStream, null, blockSize, compress, comparator); finishInit(conf); }
@Override public Writer createWriter(FileSystem fs, Path path, int blockSize, Compression.Algorithm compressAlgo, final KeyComparator comparator) throws IOException { return new HFileWriterV1(conf, cacheConf, fs, path, blockSize, compressAlgo, comparator); }
@Override public Writer createWriter(FileSystem fs, Path path, int blockSize, String compressAlgoName, final KeyComparator comparator) throws IOException { return new HFileWriterV1(conf, cacheConf, fs, path, blockSize, compressAlgoName, comparator); }
@Override public Writer createWriter(final FSDataOutputStream ostream, final int blockSize, final String compress, final KeyComparator comparator) throws IOException { return new HFileWriterV1(cacheConf, ostream, blockSize, compress, comparator); }
/** * Constructor that takes a path, creates and closes the output stream. Takes * compression algorithm name as string. */ public HFileWriterV1(Configuration conf, CacheConfig cacheConf, FileSystem fs, Path path, int blockSize, String compressAlgoName, final KeyComparator comparator) throws IOException { this(conf, cacheConf, fs, path, blockSize, compressionByName(compressAlgoName), comparator); }
/** Constructor that takes a path, creates and closes the output stream. */ public HFileWriterV1(Configuration conf, CacheConfig cacheConf, FileSystem fs, Path path, int blockSize, Compression.Algorithm compress, final KeyComparator comparator) throws IOException { super(cacheConf, createOutputStream(conf, fs, path), path, blockSize, compress, comparator); }
/** Constructor that takes a stream. */ public HFileWriterV1(CacheConfig cacheConf, final FSDataOutputStream outputStream, final int blockSize, final String compressAlgoName, final KeyComparator comparator) throws IOException { this(cacheConf, outputStream, blockSize, Compression.getCompressionAlgorithmByName(compressAlgoName), comparator); }