@Test(timeout=20000) public void testBZip2NativeCodec() throws IOException { Configuration conf = new Configuration(); conf.set("io.compression.codec.bzip2.library", "system-native"); if (NativeCodeLoader.isNativeCodeLoaded()) { if (Bzip2Factory.isNativeBzip2Loaded(conf)) { codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.BZip2Codec"); codecTest(conf, seed, count, "org.apache.hadoop.io.compress.BZip2Codec"); conf.set("io.compression.codec.bzip2.library", "java-builtin"); codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.BZip2Codec"); codecTest(conf, seed, count, "org.apache.hadoop.io.compress.BZip2Codec"); } else { LOG.warn("Native hadoop library available but native bzip2 is not"); } } }
@Test(timeout=20000) public void testSequenceFileBZip2NativeCodec() throws IOException, ClassNotFoundException, InstantiationException, IllegalAccessException { Configuration conf = new Configuration(); conf.set("io.compression.codec.bzip2.library", "system-native"); if (NativeCodeLoader.isNativeCodeLoaded()) { if (Bzip2Factory.isNativeBzip2Loaded(conf)) { sequenceFileCodecTest(conf, 0, "org.apache.hadoop.io.compress.BZip2Codec", 100); sequenceFileCodecTest(conf, 100, "org.apache.hadoop.io.compress.BZip2Codec", 100); sequenceFileCodecTest(conf, 200000, "org.apache.hadoop.io.compress.BZip2Codec", 1000000); } else { LOG.warn("Native hadoop library available but native bzip2 is not"); } } }
@Test public void testNative() { HadoopNative.requireHadoopNative(); assertTrue(NativeCodeLoader.isNativeCodeLoaded()); assertTrue(NativeCodeLoader.buildSupportsSnappy()); assertTrue(ZlibFactory.isNativeZlibLoaded(new Configuration())); assertTrue(Bzip2Factory.isNativeBzip2Loaded(new Configuration())); }
/** * A tool to test native library availability, */ public static void main(String[] args) { String usage = "NativeLibraryChecker [-a|-h]\n" + " -a use -a to check all libraries are available\n" + " by default just check hadoop library is available\n" + " exit with error code 1 if check failed\n" + " -h print this message\n"; if (args.length > 1 || (args.length == 1 && !(args[0].equals("-a") || args[0].equals("-h")))) { System.err.println(usage); ExitUtil.terminate(1); } boolean checkAll = false; if (args.length == 1) { if (args[0].equals("-h")) { System.out.println(usage); return; } checkAll = true; } Configuration conf = new Configuration(); boolean nativeHadoopLoaded = NativeCodeLoader.isNativeCodeLoaded(); boolean zlibLoaded = false; boolean snappyLoaded = false; // lz4 is linked within libhadoop boolean lz4Loaded = nativeHadoopLoaded; boolean bzip2Loaded = Bzip2Factory.isNativeBzip2Loaded(conf); String hadoopLibraryName = ""; String zlibLibraryName = ""; String snappyLibraryName = ""; String lz4LibraryName = ""; String bzip2LibraryName = ""; if (nativeHadoopLoaded) { hadoopLibraryName = NativeCodeLoader.getLibraryName(); zlibLoaded = ZlibFactory.isNativeZlibLoaded(conf); if (zlibLoaded) { zlibLibraryName = ZlibFactory.getLibraryName(); } snappyLoaded = NativeCodeLoader.buildSupportsSnappy() && SnappyCodec.isNativeCodeLoaded(); if (snappyLoaded && NativeCodeLoader.buildSupportsSnappy()) { snappyLibraryName = SnappyCodec.getLibraryName(); } if (lz4Loaded) { lz4LibraryName = Lz4Codec.getLibraryName(); } if (bzip2Loaded) { bzip2LibraryName = Bzip2Factory.getLibraryName(conf); } } System.out.println("Native library checking:"); System.out.printf("hadoop: %b %s\n", nativeHadoopLoaded, hadoopLibraryName); System.out.printf("zlib: %b %s\n", zlibLoaded, zlibLibraryName); System.out.printf("snappy: %b %s\n", snappyLoaded, snappyLibraryName); System.out.printf("lz4: %b %s\n", lz4Loaded, lz4LibraryName); System.out.printf("bzip2: %b %s\n", bzip2Loaded, bzip2LibraryName); if ((!nativeHadoopLoaded) || (checkAll && !(zlibLoaded && snappyLoaded && lz4Loaded && bzip2Loaded))) { // return 1 to indicated check failed ExitUtil.terminate(1); } }
/** * Create a {@link CompressionOutputStream} that will write to the given * {@link OutputStream} with the given {@link Compressor}. * * @param out the location for the final output stream * @param compressor compressor to use * @return a stream the user can write uncompressed data to, to have it * compressed * @throws IOException */ @Override public CompressionOutputStream createOutputStream(OutputStream out, Compressor compressor) throws IOException { return Bzip2Factory.isNativeBzip2Loaded(conf) ? new CompressorStream(out, compressor, conf.getInt("io.file.buffer.size", 4*1024)) : new BZip2CompressionOutputStream(out); }
/** * Create a {@link CompressionInputStream} that will read from the given * {@link InputStream} with the given {@link Decompressor}, and return a * stream for uncompressed data. * * @param in the stream to read compressed bytes from * @param decompressor decompressor to use * @return a stream to read uncompressed bytes from * @throws IOException */ @Override public CompressionInputStream createInputStream(InputStream in, Decompressor decompressor) throws IOException { return Bzip2Factory.isNativeBzip2Loaded(conf) ? new DecompressorStream(in, decompressor, conf.getInt("io.file.buffer.size", 4*1024)) : new BZip2CompressionInputStream(in); }
/** * Create a {@link CompressionOutputStream} that will write to the given * {@link OutputStream} with the given {@link Compressor}. * * @param out the location for the final output stream * @param compressor compressor to use * @return a stream the user can write uncompressed data to, to have it * compressed * @throws IOException */ @Override public CompressionOutputStream createOutputStream(OutputStream out, Compressor compressor) throws IOException { return Bzip2Factory.isNativeBzip2Loaded(conf) ? new CompressorStream(out, compressor, conf.getInt(IO_FILE_BUFFER_SIZE_KEY, IO_FILE_BUFFER_SIZE_DEFAULT)) : new BZip2CompressionOutputStream(out); }
/** * Create a {@link CompressionInputStream} that will read from the given * {@link InputStream} with the given {@link Decompressor}, and return a * stream for uncompressed data. * * @param in the stream to read compressed bytes from * @param decompressor decompressor to use * @return a stream to read uncompressed bytes from * @throws IOException */ @Override public CompressionInputStream createInputStream(InputStream in, Decompressor decompressor) throws IOException { return Bzip2Factory.isNativeBzip2Loaded(conf) ? new DecompressorStream(in, decompressor, conf.getInt(IO_FILE_BUFFER_SIZE_KEY, IO_FILE_BUFFER_SIZE_DEFAULT)) : new BZip2CompressionInputStream(in); }
