private static SortedSet<byte[]> readFileToSearch(final Configuration conf, final FileSystem fs, final LocatedFileStatus keyFileStatus) throws IOException, InterruptedException { SortedSet<byte []> result = new TreeSet<byte []>(Bytes.BYTES_COMPARATOR); // Return entries that are flagged Counts.UNDEFINED in the value. Return the row. This is // what is missing. TaskAttemptContext context = new TaskAttemptContextImpl(conf, new TaskAttemptID()); try (SequenceFileAsBinaryInputFormat.SequenceFileAsBinaryRecordReader rr = new SequenceFileAsBinaryInputFormat.SequenceFileAsBinaryRecordReader()) { InputSplit is = new FileSplit(keyFileStatus.getPath(), 0, keyFileStatus.getLen(), new String [] {}); rr.initialize(is, context); while (rr.nextKeyValue()) { rr.getCurrentKey(); BytesWritable bw = rr.getCurrentValue(); if (Verify.VerifyReducer.whichType(bw.getBytes()) == Verify.Counts.UNDEFINED) { byte[] key = new byte[rr.getCurrentKey().getLength()]; System.arraycopy(rr.getCurrentKey().getBytes(), 0, key, 0, rr.getCurrentKey() .getLength()); result.add(key); } } } return result; }
private static SortedSet<byte[]> readFileToSearch(final Configuration conf, final FileSystem fs, final LocatedFileStatus keyFileStatus) throws IOException, InterruptedException { SortedSet<byte []> result = new TreeSet<>(Bytes.BYTES_COMPARATOR); // Return entries that are flagged Counts.UNDEFINED in the value. Return the row. This is // what is missing. TaskAttemptContext context = new TaskAttemptContextImpl(conf, new TaskAttemptID()); try (SequenceFileAsBinaryInputFormat.SequenceFileAsBinaryRecordReader rr = new SequenceFileAsBinaryInputFormat.SequenceFileAsBinaryRecordReader()) { InputSplit is = new FileSplit(keyFileStatus.getPath(), 0, keyFileStatus.getLen(), new String [] {}); rr.initialize(is, context); while (rr.nextKeyValue()) { rr.getCurrentKey(); BytesWritable bw = rr.getCurrentValue(); if (Verify.VerifyReducer.whichType(bw.getBytes()) == Verify.Counts.UNDEFINED) { byte[] key = new byte[rr.getCurrentKey().getLength()]; System.arraycopy(rr.getCurrentKey().getBytes(), 0, key, 0, rr.getCurrentKey() .getLength()); result.add(key); } } } return result; }
/** * Java wrapper for * {@link SequenceFileAsBinaryInputFormat#addInputPath(org.apache.hadoop.mapreduce.Job, org.apache.hadoop.fs.Path)}. * * @param ctx the JavaScript context * @param thisObj the 'this' object * @param args the function arguments * @param func the function being called */ @JSStaticFunction public static void addInputPath(final Context ctx, final Scriptable thisObj, final Object[] args, final Function func) { FileInputFormatHelper.addInputPath(SequenceFileAsBinaryInputFormat.class, ctx, thisObj, args); }
/** * Java wrapper for {@link SequenceFileAsBinaryInputFormat#addInputPaths(org.apache.hadoop.mapreduce.Job, String)}. * * @param ctx the JavaScript context * @param thisObj the 'this' object * @param args the function arguments * @param func the function being called */ @JSStaticFunction public static void addInputPaths(final Context ctx, final Scriptable thisObj, final Object[] args, final Function func) { FileInputFormatHelper.addInputPaths(SequenceFileAsBinaryInputFormat.class, ctx, thisObj, args); }
/** * Java wrapper for * {@link SequenceFileAsBinaryInputFormat#getInputPathFilter(org.apache.hadoop.mapreduce.JobContext)}. * * @param ctx the JavaScript context * @param thisObj the 'this' object * @param args the function arguments * @param func the function being called * * @return class name for the input path filter or undefined */ @JSStaticFunction public static Object getInputPathFilter(final Context ctx, final Scriptable thisObj, final Object[] args, final Function func) { return FileInputFormatHelper.getInputPathFilter(SequenceFileAsBinaryInputFormat.class, ctx, thisObj, args); }
/** * Java wrapper for {@link SequenceFileAsBinaryInputFormat#getInputPaths(org.apache.hadoop.mapreduce.JobContext)}. * * @param ctx the JavaScript context * @param thisObj the 'this' object * @param args the function arguments * @param func the function being called * * @return array of input paths */ @JSStaticFunction public static Object getInputPaths(final Context ctx, final Scriptable thisObj, final Object[] args, final Function func) { return FileInputFormatHelper.getInputPaths(SequenceFileAsBinaryInputFormat.class, ctx, thisObj, args); }
/** * Java wrapper for {@link SequenceFileAsBinaryInputFormat#getMaxSplitSize(org.apache.hadoop.mapreduce.JobContext)}. * * @param ctx the JavaScript context * @param thisObj the 'this' object * @param args the function arguments * @param func the function being called * * @return the max split size */ @JSStaticFunction public static Object getMaxSplitSize(final Context ctx, final Scriptable thisObj, final Object[] args, final Function func) { return FileInputFormatHelper.getMaxSplitSize(SequenceFileAsBinaryInputFormat.class, ctx, thisObj, args); }
/** * Java wrapper for {@link SequenceFileAsBinaryInputFormat#getMinSplitSize(org.apache.hadoop.mapreduce.JobContext)}. * * @param ctx the JavaScript context * @param thisObj the 'this' object * @param args the function arguments * @param func the function being called * * @return the max split size */ @JSStaticFunction public static Object getMinSplitSize(final Context ctx, final Scriptable thisObj, final Object[] args, final Function func) { return FileInputFormatHelper.getMinSplitSize(SequenceFileAsBinaryInputFormat.class, ctx, thisObj, args); }
/** * Wraps {@link SequenceFileAsBinaryInputFormat#setInputPathFilter(org.apache.hadoop.mapreduce.Job, Class)}. * * @param ctx the JavaScript context (unused) * @param thisObj the 'this' object of the caller * @param args the arguments for the call * @param func the function called (unused) */ @JSStaticFunction public static void setInputPathFilter(final Context ctx, final Scriptable thisObj, final Object[] args, final Function func) { FileInputFormatHelper.setInputPathFilter(SequenceFileAsBinaryInputFormat.class, ctx, thisObj, args); }
/** * Java wrapper for * {@link SequenceFileAsBinaryInputFormat#setInputPaths(org.apache.hadoop.mapreduce.Job, * org.apache.hadoop.fs.Path...)} and * {@link SequenceFileAsBinaryInputFormat#setInputPaths(org.apache.hadoop.mapreduce.Job, String)}. * * @param ctx the JavaScript context * @param thisObj the 'this' object * @param args the function arguments * @param func the function being called */ @JSStaticFunction public static void setInputPaths(final Context ctx, final Scriptable thisObj, final Object[] args, final Function func) { FileInputFormatHelper.setInputPaths(SequenceFileAsBinaryInputFormat.class, ctx, thisObj, args); }
/** * Java wrapper for * {@link SequenceFileAsBinaryInputFormat#setMaxInputSplitSize(org.apache.hadoop.mapreduce.Job, long)}. * * @param ctx the JavaScript context * @param thisObj the 'this' object * @param args the function arguments * @param func the function being called */ @JSStaticFunction public static void setMaxInputSplitSize(final Context ctx, final Scriptable thisObj, final Object[] args, final Function func) { FileInputFormatHelper.setMaxInputSplitSize(SequenceFileAsBinaryInputFormat.class, ctx, thisObj, args); }
/** * Java wrapper for * {@link SequenceFileAsBinaryInputFormat#setMinInputSplitSize(org.apache.hadoop.mapreduce.Job, long)}. * * @param ctx the JavaScript context * @param thisObj the 'this' object * @param args the function arguments * @param func the function being called */ @JSStaticFunction public static void setMinInputSplitSize(final Context ctx, final Scriptable thisObj, final Object[] args, final Function func) { FileInputFormatHelper.setMinInputSplitSize(SequenceFileAsBinaryInputFormat.class, ctx, thisObj, args); }