protected Configuration setupCommonConfig() { tmpOivImgDir = Files.createTempDir(); Configuration conf = new Configuration(); conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_KEY, 1); conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 5); conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); conf.set(DFSConfigKeys.DFS_NAMENODE_LEGACY_OIV_IMAGE_DIR_KEY, tmpOivImgDir.getAbsolutePath()); conf.setBoolean(DFSConfigKeys.DFS_IMAGE_COMPRESS_KEY, true); conf.set(DFSConfigKeys.DFS_IMAGE_COMPRESSION_CODEC_KEY, SlowCodec.class.getCanonicalName()); CompressionCodecFactory.setCodecClasses(conf, ImmutableList.<Class>of(SlowCodec.class)); return conf; }
@Before public void setUp() throws Exception { Configuration conf = new Configuration(); conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_KEY, 1); conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 5); conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY, BKJMUtil .createJournalURI("/bootstrapStandby").toString()); BKJMUtil.addJournalManagerDefinition(conf); conf.setBoolean(DFSConfigKeys.DFS_IMAGE_COMPRESS_KEY, true); conf.set(DFSConfigKeys.DFS_IMAGE_COMPRESSION_CODEC_KEY, SlowCodec.class.getCanonicalName()); CompressionCodecFactory.setCodecClasses(conf, ImmutableList.<Class> of(SlowCodec.class)); MiniDFSNNTopology topology = new MiniDFSNNTopology() .addNameservice(new MiniDFSNNTopology.NSConf("ns1").addNN( new MiniDFSNNTopology.NNConf("nn1").setHttpPort(10001)).addNN( new MiniDFSNNTopology.NNConf("nn2").setHttpPort(10002))); cluster = new MiniDFSCluster.Builder(conf).nnTopology(topology) .numDataNodes(1).manageNameDfsSharedDirs(false).build(); cluster.waitActive(); }
public PossiblyDecompressedInputStream(Path inputPath, Configuration conf) throws IOException { CompressionCodecFactory codecs = new CompressionCodecFactory(conf); CompressionCodec inputCodec = codecs.getCodec(inputPath); FileSystem ifs = inputPath.getFileSystem(conf); FSDataInputStream fileIn = ifs.open(inputPath); if (inputCodec == null) { decompressor = null; coreInputStream = fileIn; } else { decompressor = CodecPool.getDecompressor(inputCodec); coreInputStream = inputCodec.createInputStream(fileIn, decompressor); } }
private JsonGenerator createJsonGenerator(Configuration conf, Path path) throws IOException { FileSystem outFS = path.getFileSystem(conf); CompressionCodec codec = new CompressionCodecFactory(conf).getCodec(path); OutputStream output; Compressor compressor = null; if (codec != null) { compressor = CodecPool.getCompressor(codec); output = codec.createOutputStream(outFS.create(path), compressor); } else { output = outFS.create(path); } JsonGenerator outGen = outFactory.createJsonGenerator(output, JsonEncoding.UTF8); outGen.useDefaultPrettyPrinter(); return outGen; }
public SingleFastqRecordReader(Configuration conf, FileSplit split) throws IOException { file = split.getPath(); start = split.getStart(); end = start + split.getLength(); FileSystem fs = file.getFileSystem(conf); FSDataInputStream fileIn = fs.open(file); CompressionCodecFactory codecFactory = new CompressionCodecFactory(conf); CompressionCodec codec = codecFactory.getCodec(file); if (codec == null) { // no codec. Uncompressed file. positionAtFirstRecord(fileIn); inputStream = fileIn; } else { // compressed file if (start != 0) { throw new RuntimeException("Start position for compressed file is not 0! (found " + start + ")"); } inputStream = codec.createInputStream(fileIn); end = Long.MAX_VALUE; // read until the end of the file } lineReader = new LineReader(inputStream); }
private InputStream openFile(Path path) throws IOException { CompressionCodec codec=new CompressionCodecFactory(miniCluster.getConfig()).getCodec(path); FSDataInputStream fileIn=dfsCluster.getFileSystem().open(path); // check if compressed if (codec==null) { // uncompressed return fileIn; } else { // compressed Decompressor decompressor = CodecPool.getDecompressor(codec); this.openDecompressors.add(decompressor); // to be returned later using close if (codec instanceof SplittableCompressionCodec) { long end = dfsCluster.getFileSystem().getFileStatus(path).getLen(); final SplitCompressionInputStream cIn =((SplittableCompressionCodec)codec).createInputStream(fileIn, decompressor, 0, end,SplittableCompressionCodec.READ_MODE.CONTINUOUS); return cIn; } else { return codec.createInputStream(fileIn,decompressor); } } }
private InputStream openFile(Path path) throws IOException { CompressionCodec codec=new CompressionCodecFactory(conf).getCodec(path); FSDataInputStream fileIn=dfsCluster.getFileSystem().open(path); // check if compressed if (codec==null) { // uncompressed return fileIn; } else { // compressed Decompressor decompressor = CodecPool.getDecompressor(codec); this.openDecompressors.add(decompressor); // to be returned later using close if (codec instanceof SplittableCompressionCodec) { long end = dfsCluster.getFileSystem().getFileStatus(path).getLen(); final SplitCompressionInputStream cIn =((SplittableCompressionCodec)codec).createInputStream(fileIn, decompressor, 0, end,SplittableCompressionCodec.READ_MODE.CONTINUOUS); return cIn; } else { return codec.createInputStream(fileIn,decompressor); } } }
@Override protected boolean isSplitable(JobContext context, Path file) { CompressionCodec codec; Configuration job = context.getConfiguration(); legionObjective = ObjectiveDeserializer.deserialize(job.get("legion_objective")); if (legionObjective.getCodecOverride() != null) { codec = new CompressionCodecFactory(context.getConfiguration()) .getCodecByClassName(legionObjective.getCodecOverride()); } else { codec = new CompressionCodecFactory(context.getConfiguration()) .getCodec(file); } if (null == codec) { return true; } return codec instanceof SplittableCompressionCodec; }
@Override public void initialize(final InputSplit genericSplit, final TaskAttemptContext context) throws IOException { final FileSplit split = (FileSplit) genericSplit; final Configuration configuration = context.getConfiguration(); if (configuration.get(Constants.GREMLIN_HADOOP_GRAPH_FILTER, null) != null) this.graphFilter = VertexProgramHelper.deserialize(ConfUtil.makeApacheConfiguration(configuration), Constants.GREMLIN_HADOOP_GRAPH_FILTER); KryoShimServiceLoader.applyConfiguration(ConfUtil.makeApacheConfiguration(configuration)); this.gryoReader = HadoopPools.getGryoPool().takeReader(); long start = split.getStart(); final Path file = split.getPath(); if (null != new CompressionCodecFactory(configuration).getCodec(file)) { throw new IllegalStateException("Compression is not supported for the (binary) Gryo format"); } // open the file and seek to the start of the split this.inputStream = file.getFileSystem(configuration).open(split.getPath()); this.splitLength = split.getLength(); if (this.splitLength > 0) this.splitLength -= (seekToHeader(this.inputStream, start) - start); }
public static BufferedReader getBufferedReader(File file, MapredContext context) throws IOException { URI fileuri = file.toURI(); Path path = new Path(fileuri); Configuration conf = context.getJobConf(); CompressionCodecFactory ccf = new CompressionCodecFactory(conf); CompressionCodec codec = ccf.getCodec(path); if (codec == null) { return new BufferedReader(new FileReader(file)); } else { Decompressor decompressor = CodecPool.getDecompressor(codec); FileInputStream fis = new FileInputStream(file); CompressionInputStream cis = codec.createInputStream(fis, decompressor); BufferedReader br = new BufferedReaderExt(new InputStreamReader(cis), decompressor); return br; } }
@Override public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { FileSplit split = (FileSplit)inputSplit; Configuration conf = taskAttemptContext.getConfiguration(); Path path = split.getPath(); FileSystem fs = path.getFileSystem(conf); // Open the stream fsin = fs.open(path); CompressionCodec codec = new CompressionCodecFactory(conf).getCodec(path); if (codec != null) { decompressor = CodecPool.getDecompressor(codec); in = new BufferedInputStream(codec.createInputStream(fsin, decompressor)); } else { in = new BufferedInputStream(fsin); } }
/** * Constructor. * * @param path * Path to the JSON data file, possibly compressed. * @param conf * @throws IOException */ public JsonObjectMapperParser(Path path, Class<? extends T> clazz, Configuration conf) throws IOException { mapper = new ObjectMapper(); mapper.configure( DeserializationConfig.Feature.CAN_OVERRIDE_ACCESS_MODIFIERS, true); this.clazz = clazz; FileSystem fs = path.getFileSystem(conf); CompressionCodec codec = new CompressionCodecFactory(conf).getCodec(path); InputStream input; if (codec == null) { input = fs.open(path); decompressor = null; } else { FSDataInputStream fsdis = fs.open(path); decompressor = CodecPool.getDecompressor(codec); input = codec.createInputStream(fsdis, decompressor); } jsonParser = mapper.getJsonFactory().createJsonParser(input); }
/** * This function opens a stream to read a compressed file. Stream is not * closed, the user has to close it when read is finished. * * @param filePath * @return */ public static InputStream openCompressedFile(Path filePath, Configuration conf) { CompressionCodecFactory factory = new CompressionCodecFactory(conf); CompressionCodec codec = factory.getCodec(filePath); if (codec == null) { log.error("No codec found for file " + filePath.toString()); return null; } try { FileSystem fs = filePath.getFileSystem(conf); Decompressor decompressor = codec.createDecompressor(); return codec.createInputStream(fs.open(filePath), decompressor); } catch (Exception e) { log.error("Error opening compressed file: " + e.getMessage()); e.printStackTrace(); } return null; }
@Override public RecordReader<LongWritable, ListWritable<Text>> getRecordReader(InputSplit inputSplit, JobConf conf, Reporter reporter) throws IOException { String charsetName = conf.get(CHARSET); Charset charset = charsetName != null ? Charset.forName(charsetName) : StandardCharsets.UTF_8; FileSplit split = (FileSplit) inputSplit; Path path = split.getPath(); FileSystem fs = path.getFileSystem(conf); InputStream is = fs.open(path); // If the input is compressed, load the compression codec. CompressionCodecFactory codecFactory = new CompressionCodecFactory(conf); CompressionCodec codec = codecFactory.getCodec(path); if (codec != null) { Decompressor decompressor = CodecPool.getDecompressor(codec); is = codec.createInputStream(is, decompressor); } return new CsvRecordReader(new InputStreamReader(is, charset), createFormat(conf), split.getLength(), conf.getBoolean(STRICT_MODE, true)); }
@Override public InputStream getInputStream() { if (inputstream == null) { try { inputstream = fs.open(path, 1000000); if (path.getName().endsWith(".lz4")) { inputstream = new LZ4FrameInputStream(inputstream); isCompressed = true; } else { CompressionCodecFactory compressionCodecs = new CompressionCodecFactory(fs.getConf()); CompressionCodec codec = compressionCodecs.getCodec(path); if (codec != null) { inputstream = codec.createInputStream(inputstream); isCompressed = true; } } } catch (IOException ex) { log.fatalexception(ex, "getInputStream()"); } } return inputstream; }
protected boolean isSplitable(Configuration conf, Path file) { final CompressionCodec codec = new CompressionCodecFactory(conf).getCodec(file); if (null == codec) { return true; } return codec instanceof SplittableCompressionCodec; }
private static void decompress(FileSystem fs, String in, String outpath) throws IOException { Configuration conf = new Configuration(); CompressionCodecFactory factory = new CompressionCodecFactory(conf); CompressionCodec codec = factory.getCodec(new Path(in)); //Decompressing zip file. InputStream is = codec.createInputStream(fs.open(new Path(in))); OutputStream out = fs.create(new Path(outpath)); //Write decompressed out IOUtils.copyBytes(is, out, conf); is.close(); out.close(); }
private static void decompress(FileSystem fs, String in, String outpath) throws IOException { Configuration conf = new Configuration(); CompressionCodecFactory factory = new CompressionCodecFactory(conf); // the correct codec will be discovered by the extension of the file CompressionCodec codec = factory.getCodec(new Path(in)); //Decompressing zip file. InputStream is = codec.createInputStream(fs.open(new Path(in))); OutputStream out = fs.create(new Path(outpath)); //Write decompressed out IOUtils.copyBytes(is, out, conf); is.close(); out.close(); }
private static FileStatus decompress(FileSystem fs, String in, String outpath) throws IOException { Configuration conf = new Configuration(); CompressionCodecFactory factory = new CompressionCodecFactory(conf); CompressionCodec codec = factory.getCodec(new Path(in)); //Decompressing zip file. InputStream is = codec.createInputStream(fs.open(new Path(in))); OutputStream out = fs.create(new Path(outpath)); //Write decompressed out IOUtils.copyBytes(is, out, conf); is.close(); out.close(); return fs.getFileStatus(new Path(outpath)); }
