1. 包列表
  2. 类列表
  3. org.apache.hadoop.mapred.Utils.OutputFileUtils.OutputFilesFilter

Java 类org.apache.hadoop.mapred.Utils.OutputFileUtils.OutputFilesFilter 实例源码

项目：pbase    文件：TestImportTsv.java    
/**
 * Confirm ImportTsv via HFiles on fs.
 */
private static void validateHFiles(FileSystem fs, String outputPath, String family)
    throws IOException {

  // validate number and content of output columns
  LOG.debug("Validating HFiles.");
  Set<String> configFamilies = new HashSet<String>();
  configFamilies.add(family);
  Set<String> foundFamilies = new HashSet<String>();
  for (FileStatus cfStatus : fs.listStatus(new Path(outputPath), new OutputFilesFilter())) {
    String[] elements = cfStatus.getPath().toString().split(Path.SEPARATOR);
    String cf = elements[elements.length - 1];
    foundFamilies.add(cf);
    assertTrue(
      String.format(
        "HFile ouput contains a column family (%s) not present in input families (%s)",
        cf, configFamilies),
        configFamilies.contains(cf));
    for (FileStatus hfile : fs.listStatus(cfStatus.getPath())) {
      assertTrue(
        String.format("HFile %s appears to contain no data.", hfile.getPath()),
        hfile.getLen() > 0);
    }
  }
}
项目：pbase    文件：TestImportTSVWithVisibilityLabels.java    
/**
 * Confirm ImportTsv via HFiles on fs.
 */
private static void validateHFiles(FileSystem fs, String outputPath, String family)
    throws IOException {

  // validate number and content of output columns
  LOG.debug("Validating HFiles.");
  Set<String> configFamilies = new HashSet<String>();
  configFamilies.add(family);
  Set<String> foundFamilies = new HashSet<String>();
  for (FileStatus cfStatus : fs.listStatus(new Path(outputPath), new OutputFilesFilter())) {
    LOG.debug("The output path has files");
    String[] elements = cfStatus.getPath().toString().split(Path.SEPARATOR);
    String cf = elements[elements.length - 1];
    foundFamilies.add(cf);
    assertTrue(String.format(
        "HFile ouput contains a column family (%s) not present in input families (%s)", cf,
        configFamilies), configFamilies.contains(cf));
    for (FileStatus hfile : fs.listStatus(cfStatus.getPath())) {
      assertTrue(String.format("HFile %s appears to contain no data.", hfile.getPath()),
          hfile.getLen() > 0);
    }
  }
}
项目：HIndex    文件：TestImportTsv.java    
/**
 * Confirm ImportTsv via HFiles on fs.
 */
private static void validateHFiles(FileSystem fs, String outputPath, String family)
    throws IOException {

  // validate number and content of output columns
  LOG.debug("Validating HFiles.");
  Set<String> configFamilies = new HashSet<String>();
  configFamilies.add(family);
  Set<String> foundFamilies = new HashSet<String>();
  for (FileStatus cfStatus : fs.listStatus(new Path(outputPath), new OutputFilesFilter())) {
    String[] elements = cfStatus.getPath().toString().split(Path.SEPARATOR);
    String cf = elements[elements.length - 1];
    foundFamilies.add(cf);
    assertTrue(
      String.format(
        "HFile ouput contains a column family (%s) not present in input families (%s)",
        cf, configFamilies),
        configFamilies.contains(cf));
    for (FileStatus hfile : fs.listStatus(cfStatus.getPath())) {
      assertTrue(
        String.format("HFile %s appears to contain no data.", hfile.getPath()),
        hfile.getLen() > 0);
    }
  }
}
项目：HIndex    文件：TestImportTSVWithVisibilityLabels.java    
/**
 * Confirm ImportTsv via HFiles on fs.
 */
private static void validateHFiles(FileSystem fs, String outputPath, String family)
    throws IOException {

  // validate number and content of output columns
  LOG.debug("Validating HFiles.");
  Set<String> configFamilies = new HashSet<String>();
  configFamilies.add(family);
  Set<String> foundFamilies = new HashSet<String>();
  for (FileStatus cfStatus : fs.listStatus(new Path(outputPath), new OutputFilesFilter())) {
    LOG.debug("The output path has files");
    String[] elements = cfStatus.getPath().toString().split(Path.SEPARATOR);
    String cf = elements[elements.length - 1];
    foundFamilies.add(cf);
    assertTrue(String.format(
        "HFile ouput contains a column family (%s) not present in input families (%s)", cf,
        configFamilies), configFamilies.contains(cf));
    for (FileStatus hfile : fs.listStatus(cfStatus.getPath())) {
      assertTrue(String.format("HFile %s appears to contain no data.", hfile.getPath()),
          hfile.getLen() > 0);
    }
  }
}
项目：PyroDB    文件：TestImportTsv.java    
/**
 * Confirm ImportTsv via HFiles on fs.
 */
private static void validateHFiles(FileSystem fs, String outputPath, String family)
    throws IOException {

  // validate number and content of output columns
  LOG.debug("Validating HFiles.");
  Set<String> configFamilies = new HashSet<String>();
  configFamilies.add(family);
  Set<String> foundFamilies = new HashSet<String>();
  for (FileStatus cfStatus : fs.listStatus(new Path(outputPath), new OutputFilesFilter())) {
    String[] elements = cfStatus.getPath().toString().split(Path.SEPARATOR);
    String cf = elements[elements.length - 1];
    foundFamilies.add(cf);
    assertTrue(
      String.format(
        "HFile ouput contains a column family (%s) not present in input families (%s)",
        cf, configFamilies),
        configFamilies.contains(cf));
    for (FileStatus hfile : fs.listStatus(cfStatus.getPath())) {
      assertTrue(
        String.format("HFile %s appears to contain no data.", hfile.getPath()),
        hfile.getLen() > 0);
    }
  }
}
项目：PyroDB    文件：TestImportTSVWithVisibilityLabels.java    
/**
 * Confirm ImportTsv via HFiles on fs.
 */
private static void validateHFiles(FileSystem fs, String outputPath, String family)
    throws IOException {

  // validate number and content of output columns
  LOG.debug("Validating HFiles.");
  Set<String> configFamilies = new HashSet<String>();
  configFamilies.add(family);
  Set<String> foundFamilies = new HashSet<String>();
  for (FileStatus cfStatus : fs.listStatus(new Path(outputPath), new OutputFilesFilter())) {
    LOG.debug("The output path has files");
    String[] elements = cfStatus.getPath().toString().split(Path.SEPARATOR);
    String cf = elements[elements.length - 1];
    foundFamilies.add(cf);
    assertTrue(String.format(
        "HFile ouput contains a column family (%s) not present in input families (%s)", cf,
        configFamilies), configFamilies.contains(cf));
    for (FileStatus hfile : fs.listStatus(cfStatus.getPath())) {
      assertTrue(String.format("HFile %s appears to contain no data.", hfile.getPath()),
          hfile.getLen() > 0);
    }
  }
}
项目：c5    文件：TestImportTsv.java    
/**
 * Confirm ImportTsv via HFiles on fs.
 */
private static void validateHFiles(FileSystem fs, String outputPath, String family)
    throws IOException {

  // validate number and content of output columns
  LOG.debug("Validating HFiles.");
  Set<String> configFamilies = new HashSet<String>();
  configFamilies.add(family);
  Set<String> foundFamilies = new HashSet<String>();
  for (FileStatus cfStatus : fs.listStatus(new Path(outputPath), new OutputFilesFilter())) {
    String[] elements = cfStatus.getPath().toString().split(Path.SEPARATOR);
    String cf = elements[elements.length - 1];
    foundFamilies.add(cf);
    assertTrue(
      String.format(
        "HFile ouput contains a column family (%s) not present in input families (%s)",
        cf, configFamilies),
        configFamilies.contains(cf));
    for (FileStatus hfile : fs.listStatus(cfStatus.getPath())) {
      assertTrue(
        String.format("HFile %s appears to contain no data.", hfile.getPath()),
        hfile.getLen() > 0);
    }
  }
}
项目：ditb    文件：TestImportTsv.java    
/**
 * Confirm ImportTsv via HFiles on fs.
 */
private static void validateHFiles(FileSystem fs, String outputPath, String family,
    int expectedKVCount) throws IOException {
  // validate number and content of output columns
  LOG.debug("Validating HFiles.");
  Set<String> configFamilies = new HashSet<String>();
  configFamilies.add(family);
  Set<String> foundFamilies = new HashSet<String>();
  int actualKVCount = 0;
  for (FileStatus cfStatus : fs.listStatus(new Path(outputPath), new OutputFilesFilter())) {
    String[] elements = cfStatus.getPath().toString().split(Path.SEPARATOR);
    String cf = elements[elements.length - 1];
    foundFamilies.add(cf);
    assertTrue(
      String.format(
        "HFile output contains a column family (%s) not present in input families (%s)",
        cf, configFamilies),
        configFamilies.contains(cf));
    for (FileStatus hfile : fs.listStatus(cfStatus.getPath())) {
      assertTrue(
        String.format("HFile %s appears to contain no data.", hfile.getPath()),
        hfile.getLen() > 0);
      // count the number of KVs from all the hfiles
      if (expectedKVCount > -1) {
        actualKVCount += getKVCountFromHfile(fs, hfile.getPath());
      }
    }
  }
  if (expectedKVCount > -1) {
    assertTrue(String.format(
      "KV count in output hfile=<%d> doesn't match with expected KV count=<%d>", actualKVCount,
      expectedKVCount), actualKVCount == expectedKVCount);
  }
}
项目：ditb    文件：TestImportTSVWithVisibilityLabels.java    
/**
 * Confirm ImportTsv via HFiles on fs.
 */
private static void validateHFiles(FileSystem fs, String outputPath, String family,
    int expectedKVCount) throws IOException {

  // validate number and content of output columns
  LOG.debug("Validating HFiles.");
  Set<String> configFamilies = new HashSet<String>();
  configFamilies.add(family);
  Set<String> foundFamilies = new HashSet<String>();
  int actualKVCount = 0;
  for (FileStatus cfStatus : fs.listStatus(new Path(outputPath), new OutputFilesFilter())) {
    LOG.debug("The output path has files");
    String[] elements = cfStatus.getPath().toString().split(Path.SEPARATOR);
    String cf = elements[elements.length - 1];
    foundFamilies.add(cf);
    assertTrue(String.format(
        "HFile ouput contains a column family (%s) not present in input families (%s)", cf,
        configFamilies), configFamilies.contains(cf));
    for (FileStatus hfile : fs.listStatus(cfStatus.getPath())) {
      assertTrue(String.format("HFile %s appears to contain no data.", hfile.getPath()),
          hfile.getLen() > 0);
      if (expectedKVCount > -1) {
        actualKVCount += getKVCountFromHfile(fs, hfile.getPath());
      }
    }
  }
  if (expectedKVCount > -1) {
    assertTrue(String.format(
      "KV count in output hfile=<%d> doesn't match with expected KV count=<%d>", actualKVCount,
      expectedKVCount), actualKVCount == expectedKVCount);
  }
}
项目：hbase    文件：TestImportTsv.java    
/**
 * Confirm ImportTsv via HFiles on fs.
 */
private static void validateHFiles(FileSystem fs, String outputPath, String family,
    int expectedKVCount) throws IOException {
  // validate number and content of output columns
  LOG.debug("Validating HFiles.");
  Set<String> configFamilies = new HashSet<>();
  configFamilies.add(family);
  Set<String> foundFamilies = new HashSet<>();
  int actualKVCount = 0;
  for (FileStatus cfStatus : fs.listStatus(new Path(outputPath), new OutputFilesFilter())) {
    String[] elements = cfStatus.getPath().toString().split(Path.SEPARATOR);
    String cf = elements[elements.length - 1];
    foundFamilies.add(cf);
    assertTrue(
      String.format(
        "HFile output contains a column family (%s) not present in input families (%s)",
        cf, configFamilies),
        configFamilies.contains(cf));
    for (FileStatus hfile : fs.listStatus(cfStatus.getPath())) {
      assertTrue(
        String.format("HFile %s appears to contain no data.", hfile.getPath()),
        hfile.getLen() > 0);
      // count the number of KVs from all the hfiles
      if (expectedKVCount > -1) {
        actualKVCount += getKVCountFromHfile(fs, hfile.getPath());
      }
    }
  }
  assertTrue(String.format("HFile output does not contain the input family '%s'.", family),
      foundFamilies.contains(family));
  if (expectedKVCount > -1) {
    assertTrue(String.format(
      "KV count in ouput hfile=<%d> doesn't match with expected KV count=<%d>", actualKVCount,
      expectedKVCount), actualKVCount == expectedKVCount);
  }
}
项目：hbase    文件：TestImportTSVWithVisibilityLabels.java    
/**
 * Confirm ImportTsv via HFiles on fs.
 */
private static void validateHFiles(FileSystem fs, String outputPath, String family,
    int expectedKVCount) throws IOException {

  // validate number and content of output columns
  LOG.debug("Validating HFiles.");
  Set<String> configFamilies = new HashSet<>();
  configFamilies.add(family);
  Set<String> foundFamilies = new HashSet<>();
  int actualKVCount = 0;
  for (FileStatus cfStatus : fs.listStatus(new Path(outputPath), new OutputFilesFilter())) {
    LOG.debug("The output path has files");
    String[] elements = cfStatus.getPath().toString().split(Path.SEPARATOR);
    String cf = elements[elements.length - 1];
    foundFamilies.add(cf);
    assertTrue(String.format(
        "HFile ouput contains a column family (%s) not present in input families (%s)", cf,
        configFamilies), configFamilies.contains(cf));
    for (FileStatus hfile : fs.listStatus(cfStatus.getPath())) {
      assertTrue(String.format("HFile %s appears to contain no data.", hfile.getPath()),
          hfile.getLen() > 0);
      if (expectedKVCount > -1) {
        actualKVCount += getKVCountFromHfile(fs, hfile.getPath());
      }
    }
  }
  if (expectedKVCount > -1) {
    assertTrue(String.format(
      "KV count in output hfile=<%d> doesn't match with expected KV count=<%d>", actualKVCount,
      expectedKVCount), actualKVCount == expectedKVCount);
  }
}