/** * Actually instantiate the user's chosen RecordReader implementation. */ @SuppressWarnings("unchecked") private void createChildReader() throws IOException, InterruptedException { LOG.debug("ChildSplit operates on: " + split.getPath(index)); Configuration conf = context.getConfiguration(); // Determine the file format we're reading. Class rrClass; if (ExportJobBase.isSequenceFiles(conf, split.getPath(index))) { rrClass = SequenceFileRecordReader.class; } else { rrClass = LineRecordReader.class; } // Create the appropriate record reader. this.rr = (RecordReader<LongWritable, Object>) ReflectionUtils.newInstance(rrClass, conf); }
@SuppressWarnings("unchecked") @Override public void initialize(InputSplit curSplit, TaskAttemptContext curContext) throws IOException, InterruptedException { this.split = (CombineFileSplit) curSplit; this.context = curContext; if (null == rr) { rr = ReflectionUtils.newInstance(SequenceFileRecordReader.class, context.getConfiguration()); } FileSplit fileSplit = new FileSplit(this.split.getPath(index), this.split.getOffset(index), this.split.getLength(index), this.split.getLocations()); this.rr.initialize(fileSplit, this.context); }
private void openForRead(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { reader = new SequenceFileRecordReader<K, V>(); reader.initialize(new FileSplit(chunkFilePath, 0, DistCpUtils.getFileSize(chunkFilePath, configuration), null), taskAttemptContext); }
/** * Returns a reader for this split of the distributed cache file list. */ @Override public RecordReader<LongWritable, BytesWritable> createRecordReader( InputSplit split, final TaskAttemptContext taskContext) throws IOException, InterruptedException { return new SequenceFileRecordReader<LongWritable, BytesWritable>(); }
private void openForRead(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { reader = new SequenceFileRecordReader<K, V>(); reader.initialize(new FileSplit(chunkFilePath, 0, DistCpUtils.getFileSize(chunkFilePath, chunkContext.getConfiguration()), null), taskAttemptContext); }
@SuppressWarnings("unchecked") public CombineSequenceFileRecordReader(CombineFileSplit split, TaskAttemptContext context, Integer index) throws IOException, InterruptedException { this.index = index; this.split = (CombineFileSplit) split; this.context = context; this.rr = ReflectionUtils.newInstance(SequenceFileRecordReader.class, context.getConfiguration()); }
@Override public RecordReader<NullWritable, V> createRecordReader(final InputSplit split, final TaskAttemptContext context) throws IOException, InterruptedException { final SequenceFileRecordReader<NullWritable, V> reader = new SequenceFileRecordReader<NullWritable, V>(); reader.initialize(split, context); return reader; }
private void openForRead(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { reader = new SequenceFileRecordReader<>(); reader.initialize(new FileSplit(chunkFilePath, 0, getFileSize(chunkFilePath, configuration), null), taskAttemptContext); }
/** * Getter for the record-reader, opened to the chunk-file. * @return Opened Sequence-file reader. */ public SequenceFileRecordReader<K,V> getReader() { assert reader != null : "Reader un-initialized!"; return reader; }
@SuppressWarnings("unchecked") @Override public void prepareToRead(RecordReader reader, PigSplit split) throws IOException { this.reader = (SequenceFileRecordReader) reader; }
public VectorSequenceFileRecordReader() { // recordReader = new SequenceFileRecordReader<LongWritable, ArrayListWritable>(); recordReader = new SequenceFileRecordReader<LongWritable, NumberListWritable<Double>>(); pair = new PairOfByteBuffers(); }
@Override public RecordReader<RecordKey, Text> getRecordReader(InputSplit split, TaskAttemptContext context, Integer index) { return new SequenceFileRecordReader<>(); }
public SequenceFileAsJSONRecordBatchReader() throws IOException { sequenceFileRecordReader = new SequenceFileRecordReader<WritableComparable<?>, Writable>(); }