/** Sanity check for source */ protected static void checkSource(Configuration conf, List<Path> srcs ) throws InvalidInputException { List<IOException> ioes = new ArrayList<IOException>(); for(Path p : srcs) { try { if (!p.getFileSystem(conf).exists(p)) { ioes.add(new FileNotFoundException("Source "+p+" does not exist.")); } } catch(IOException e) {ioes.add(e);} } if (!ioes.isEmpty()) { throw new InvalidInputException(ioes); } }
/** Sanity check for srcPath */ private static void checkSrcPath(JobConf jobConf, List<Path> srcPaths) throws IOException { List<IOException> rslt = new ArrayList<IOException>(); Path[] ps = new Path[srcPaths.size()]; ps = srcPaths.toArray(ps); TokenCache.obtainTokensForNamenodes(jobConf.getCredentials(), ps, jobConf); for (Path p : srcPaths) { FileSystem fs = p.getFileSystem(jobConf); if (!fs.exists(p)) { rslt.add(new IOException("Input source " + p + " does not exist.")); } } if (!rslt.isEmpty()) { throw new InvalidInputException(rslt); } }
/** Sanity check for srcPath */ private static void checkSrcPath(Configuration conf, List<Path> srcPaths ) throws IOException { List<IOException> rslt = new ArrayList<IOException>(); List<Path> unglobbed = new LinkedList<Path>(); for (Path p : srcPaths) { FileSystem fs = p.getFileSystem(conf); FileStatus[] inputs = fs.globStatus(p); if(inputs != null && inputs.length > 0) { for (FileStatus onePath: inputs) { unglobbed.add(onePath.getPath()); } } else { rslt.add(new IOException("Input source " + p + " does not exist.")); } } if (!rslt.isEmpty()) { throw new InvalidInputException(rslt); } srcPaths.clear(); srcPaths.addAll(unglobbed); }
/** Sanity check for srcPath */ public static void checkSrcPath(JobConf jobConf, List<Path> srcPaths) throws IOException { List<IOException> rslt = new ArrayList<IOException>(); Path[] ps = new Path[srcPaths.size()]; ps = srcPaths.toArray(ps); TokenCache.obtainTokensForNamenodes(jobConf.getCredentials(), ps, jobConf); for (Path p : srcPaths) { FileSystem fs = p.getFileSystem(jobConf); if (!fs.exists(p)) { rslt.add(new IOException("Input source " + p + " does not exist.")); } } if (!rslt.isEmpty()) { throw new InvalidInputException(rslt); } }
/** Sanity check for srcPath */ private static void checkSrcPath(Configuration conf, List<Path> srcPaths ) throws IOException { List<IOException> rslt = new ArrayList<IOException>(); List<Path> unglobbed = new LinkedList<Path>(); for (Path p : srcPaths) { FileSystem fs = p.getFileSystem(conf); FileStatus[] inputs = fs.globStatus(p); if(inputs.length > 0) { for (FileStatus onePath: inputs) { unglobbed.add(onePath.getPath()); } } else { rslt.add(new IOException("Input source " + p + " does not exist.")); } } if (!rslt.isEmpty()) { throw new InvalidInputException(rslt); } srcPaths.clear(); srcPaths.addAll(unglobbed); }
private static void check(Configuration conf, List<FileOperation> ops ) throws InvalidInputException { List<Path> srcs = new ArrayList<Path>(); for(FileOperation op : ops) { srcs.add(op.src); } DistTool.checkSource(conf, srcs); }
/** Sanity check for srcPath */ private static void checkSrcPath(JobConf jobConf, List<Path> srcPaths) throws IOException { List<IOException> rslt = new ArrayList<IOException>(); List<Path> unglobbed = new LinkedList<Path>(); Path[] ps = new Path[srcPaths.size()]; ps = srcPaths.toArray(ps); TokenCache.obtainTokensForNamenodes(jobConf.getCredentials(), ps, jobConf); for (Path p : srcPaths) { FileSystem fs = p.getFileSystem(jobConf); FileStatus[] inputs = fs.globStatus(p); if(inputs != null && inputs.length > 0) { for (FileStatus onePath: inputs) { unglobbed.add(onePath.getPath()); } } else { rslt.add(new IOException("Input source " + p + " does not exist.")); } } if (!rslt.isEmpty()) { throw new InvalidInputException(rslt); } srcPaths.clear(); srcPaths.addAll(unglobbed); }