Java 类org.apache.hadoop.util.GenericOptionsParser 实例源码

项目：hadoop 文件：ResourceManager.java

public static void main(String argv[]) {
  Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
  StringUtils.startupShutdownMessage(ResourceManager.class, argv, LOG);
  try {
    Configuration conf = new YarnConfiguration();
    GenericOptionsParser hParser = new GenericOptionsParser(conf, argv);
    argv = hParser.getRemainingArgs();
    // If -format-state-store, then delete RMStateStore; else startup normally
    if (argv.length == 1 && argv[0].equals("-format-state-store")) {
      deleteRMStateStore(conf);
    } else {
      ResourceManager resourceManager = new ResourceManager();
      ShutdownHookManager.get().addShutdownHook(
        new CompositeServiceShutdownHook(resourceManager),
        SHUTDOWN_HOOK_PRIORITY);
      resourceManager.init(conf);
      resourceManager.start();
    }
  } catch (Throwable t) {
    LOG.fatal("Error starting ResourceManager", t);
    System.exit(-1);
  }
}

项目：hadoop 文件：ApplicationHistoryServer.java

static ApplicationHistoryServer launchAppHistoryServer(String[] args) {
  Thread
    .setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
  StringUtils.startupShutdownMessage(ApplicationHistoryServer.class, args,
    LOG);
  ApplicationHistoryServer appHistoryServer = null;
  try {
    appHistoryServer = new ApplicationHistoryServer();
    ShutdownHookManager.get().addShutdownHook(
      new CompositeServiceShutdownHook(appHistoryServer),
      SHUTDOWN_HOOK_PRIORITY);
    YarnConfiguration conf = new YarnConfiguration();
    new GenericOptionsParser(conf, args);
    appHistoryServer.init(conf);
    appHistoryServer.start();
  } catch (Throwable t) {
    LOG.fatal("Error starting ApplicationHistoryServer", t);
    ExitUtil.terminate(-1, "Error starting ApplicationHistoryServer");
  }
  return appHistoryServer;
}

项目：hadoop 文件：Submitter.java

void printUsage() {
  // The CLI package should do this for us, but I can't figure out how
  // to make it print something reasonable.
  System.out.println("bin/hadoop pipes");
  System.out.println("  [-input <path>] // Input directory");
  System.out.println("  [-output <path>] // Output directory");
  System.out.println("  [-jar <jar file> // jar filename");
  System.out.println("  [-inputformat <class>] // InputFormat class");
  System.out.println("  [-map <class>] // Java Map class");
  System.out.println("  [-partitioner <class>] // Java Partitioner");
  System.out.println("  [-reduce <class>] // Java Reduce class");
  System.out.println("  [-writer <class>] // Java RecordWriter");
  System.out.println("  [-program <executable>] // executable URI");
  System.out.println("  [-reduces <num>] // number of reduces");
  System.out.println("  [-lazyOutput <true/false>] // createOutputLazily");
  System.out.println();
  GenericOptionsParser.printGenericCommandUsage(System.out);
}

项目：hadoop 文件：JobHistoryServer.java

static JobHistoryServer launchJobHistoryServer(String[] args) {
  Thread.
      setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
  StringUtils.startupShutdownMessage(JobHistoryServer.class, args, LOG);
  JobHistoryServer jobHistoryServer = null;
  try {
    jobHistoryServer = new JobHistoryServer();
    ShutdownHookManager.get().addShutdownHook(
        new CompositeServiceShutdownHook(jobHistoryServer),
        SHUTDOWN_HOOK_PRIORITY);
    YarnConfiguration conf = new YarnConfiguration(new JobConf());
    new GenericOptionsParser(conf, args);
    jobHistoryServer.init(conf);
    jobHistoryServer.start();
  } catch (Throwable t) {
    LOG.fatal("Error starting JobHistoryServer", t);
    ExitUtil.terminate(-1, "Error starting JobHistoryServer");
  }
  return jobHistoryServer;
}

项目：hadoop 文件：WordCount.java

public static void main(String[] args) throws Exception {
  Configuration conf = new Configuration();
  String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
  if (otherArgs.length < 2) {
    System.err.println("Usage: wordcount <in> [<in>...] <out>");
    System.exit(2);
  }
  Job job = Job.getInstance(conf, "word count");
  job.setJarByClass(WordCount.class);
  job.setMapperClass(TokenizerMapper.class);
  job.setCombinerClass(IntSumReducer.class);
  job.setReducerClass(IntSumReducer.class);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(IntWritable.class);
  for (int i = 0; i < otherArgs.length - 1; ++i) {
    FileInputFormat.addInputPath(job, new Path(otherArgs[i]));
  }
  FileOutputFormat.setOutputPath(job,
    new Path(otherArgs[otherArgs.length - 1]));
  System.exit(job.waitForCompletion(true) ? 0 : 1);
}

项目：hadoop 文件：DFSZKFailoverController.java

public static void main(String args[])
    throws Exception {
  if (DFSUtil.parseHelpArgument(args, 
      ZKFailoverController.USAGE, System.out, true)) {
    System.exit(0);
  }

  GenericOptionsParser parser = new GenericOptionsParser(
      new HdfsConfiguration(), args);
  DFSZKFailoverController zkfc = DFSZKFailoverController.create(
      parser.getConfiguration());
  int retCode = 0;
  try {
    retCode = zkfc.run(parser.getRemainingArgs());
  } catch (Throwable t) {
    LOG.fatal("Got a fatal error, exiting now", t);
  }
  System.exit(retCode);
}

项目：ditb 文件：Canary.java

public static void main(String[] args) throws Exception {
  final Configuration conf = HBaseConfiguration.create();
  final ChoreService choreService = new ChoreService("CANARY_TOOL");
  final ScheduledChore authChore = AuthUtil.getAuthChore(conf);
  if (authChore != null) {
    choreService.scheduleChore(authChore);
  }

  // loading the generic options to conf
  new GenericOptionsParser(conf, args);

  int numThreads = conf.getInt("hbase.canary.threads.num", MAX_THREADS_NUM);
  LOG.info("Number of exection threads " + numThreads);

  ExecutorService executor = new ScheduledThreadPoolExecutor(numThreads);

  Class<? extends Sink> sinkClass =
      conf.getClass("hbase.canary.sink.class", RegionServerStdOutSink.class, Sink.class);
  Sink sink = ReflectionUtils.newInstance(sinkClass);

  int exitCode = ToolRunner.run(conf, new Canary(executor, sink), args);
  choreService.shutdown();
  executor.shutdown();
  System.exit(exitCode);
}

项目：ditb 文件：CellCounter.java

/**
 * Main entry point.
 *
 * @param args The command line parameters.
 * @throws Exception When running the job fails.
 */
public static void main(String[] args) throws Exception {
  Configuration conf = HBaseConfiguration.create();
  String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
  if (otherArgs.length < 2) {
    System.err.println("ERROR: Wrong number of parameters: " + args.length);
    System.err.println("Usage: CellCounter ");
    System.err.println("       <tablename> <outputDir> <reportSeparator> [^[regex pattern] or " +
      "[Prefix] for row filter]] --starttime=[starttime] --endtime=[endtime]");
    System.err.println("  Note: -D properties will be applied to the conf used. ");
    System.err.println("  Additionally, the following SCAN properties can be specified");
    System.err.println("  to get fine grained control on what is counted..");
    System.err.println("   -D " + TableInputFormat.SCAN_COLUMN_FAMILY + "=<familyName>");
    System.err.println(" <reportSeparator> parameter can be used to override the default report separator " +
        "string : used to separate the rowId/column family name and qualifier name.");
    System.err.println(" [^[regex pattern] or [Prefix] parameter can be used to limit the cell counter count " +
        "operation to a limited subset of rows from the table based on regex or prefix pattern.");
    System.exit(-1);
  }
  Job job = createSubmittableJob(conf, otherArgs);
  System.exit(job.waitForCompletion(true) ? 0 : 1);
}

项目：ditb 文件：HashTable.java

@Override
public int run(String[] args) throws Exception {
  String[] otherArgs = new GenericOptionsParser(getConf(), args).getRemainingArgs();
  if (!doCommandLine(otherArgs)) {
    return 1;
  }

  Job job = createSubmittableJob(otherArgs);
  writeTempManifestFile();
  if (!job.waitForCompletion(true)) {
    LOG.info("Map-reduce job failed!");
    return 1;
  }
  completeManifest();
  return 0;
}

项目：ditb 文件：Merge.java

private int parseArgs(String[] args) throws IOException {
  GenericOptionsParser parser =
    new GenericOptionsParser(getConf(), args);

  String[] remainingArgs = parser.getRemainingArgs();
  if (remainingArgs.length != 3) {
    usage();
    return -1;
  }
  tableName = TableName.valueOf(remainingArgs[0]);

  region1 = Bytes.toBytesBinary(remainingArgs[1]);
  region2 = Bytes.toBytesBinary(remainingArgs[2]);
  int status = 0;
  if (notInTable(tableName, region1) || notInTable(tableName, region2)) {
    status = -1;
  } else if (Bytes.equals(region1, region2)) {
    LOG.error("Can't merge a region with itself");
    status = -1;
  }
  return status;
}

项目：ditb 文件：TestImportTsv.java

@Test
public void testJobConfigurationsWithTsvImporterTextMapper() throws Exception {
  String table = "test-" + UUID.randomUUID();
  Path bulkOutputPath = new Path(util.getDataTestDirOnTestFS(table),"hfiles");
  String INPUT_FILE = "InputFile1.csv";
  // Prepare the arguments required for the test.
  String[] args =
      new String[] {
          "-D" + ImportTsv.MAPPER_CONF_KEY
              + "=org.apache.hadoop.hbase.mapreduce.TsvImporterTextMapper",
          "-D" + ImportTsv.COLUMNS_CONF_KEY
              + "=HBASE_ROW_KEY,FAM:A,FAM:B",
          "-D" + ImportTsv.SEPARATOR_CONF_KEY + "=,",
          "-D" + ImportTsv.BULK_OUTPUT_CONF_KEY + "=" + bulkOutputPath.toString(), table,
          INPUT_FILE
          };
  GenericOptionsParser opts = new GenericOptionsParser(util.getConfiguration(), args);
  args = opts.getRemainingArgs();
  Job job = ImportTsv.createSubmittableJob(util.getConfiguration(), args);
  assertTrue(job.getMapperClass().equals(TsvImporterTextMapper.class));
  assertTrue(job.getReducerClass().equals(TextSortReducer.class));
  assertTrue(job.getMapOutputValueClass().equals(Text.class));
}

项目：ditb 文件：IntegrationTestImportTsv.java

public int run(String[] args) throws Exception {
  if (args.length != 0) {
    System.err.println(format("%s [genericOptions]", NAME));
    System.err.println("  Runs ImportTsv integration tests against a distributed cluster.");
    System.err.println();
    GenericOptionsParser.printGenericCommandUsage(System.err);
    return 1;
  }

  // adding more test methods? Don't forget to add them here... or consider doing what
  // IntegrationTestsDriver does.
  provisionCluster();
  testGenerateAndLoad();
  releaseCluster();

  return 0;
}

项目：aliyun-oss-hadoop-fs 文件：ApplicationHistoryServer.java

static ApplicationHistoryServer launchAppHistoryServer(String[] args) {
  Thread
    .setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
  StringUtils.startupShutdownMessage(ApplicationHistoryServer.class, args,
    LOG);
  ApplicationHistoryServer appHistoryServer = null;
  try {
    appHistoryServer = new ApplicationHistoryServer();
    ShutdownHookManager.get().addShutdownHook(
      new CompositeServiceShutdownHook(appHistoryServer),
      SHUTDOWN_HOOK_PRIORITY);
    YarnConfiguration conf = new YarnConfiguration();
    new GenericOptionsParser(conf, args);
    appHistoryServer.init(conf);
    appHistoryServer.start();
  } catch (Throwable t) {
    LOG.fatal("Error starting ApplicationHistoryServer", t);
    ExitUtil.terminate(-1, "Error starting ApplicationHistoryServer");
  }
  return appHistoryServer;
}

项目：aliyun-oss-hadoop-fs 文件：TimelineServicePerformance.java

protected static int printUsage() {
  System.err.println(
      "Usage: [-m <maps>] number of mappers (default: " + NUM_MAPS_DEFAULT +
          ")\n" +
      "     [-v] timeline service version\n" +
      "     [-mtype <mapper type in integer>]\n" +
      "          1. simple entity write mapper\n" +
      "          2. jobhistory files replay mapper\n" +
      "     [-s <(KBs)test>] number of KB per put (mtype=1, default: " +
           SimpleEntityWriterV1.KBS_SENT_DEFAULT + " KB)\n" +
      "     [-t] package sending iterations per mapper (mtype=1, default: " +
           SimpleEntityWriterV1.TEST_TIMES_DEFAULT + ")\n" +
      "     [-d <path>] root path of job history files (mtype=2)\n" +
      "     [-r <replay mode>] (mtype=2)\n" +
      "          1. write all entities for a job in one put (default)\n" +
      "          2. write one entity at a time\n");
  GenericOptionsParser.printGenericCommandUsage(System.err);
  return -1;
}

项目：aliyun-oss-hadoop-fs 文件：WordCount.java

public static void main(String[] args) throws Exception {
  final Configuration conf = new Configuration();
  final String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
  if (otherArgs.length != 2) {
    System.err.println("Usage: wordcount <in> <out>");
    System.exit(2);
  }
  final Job job = Job.getInstance(conf,
                                  conf.get(MRJobConfig.JOB_NAME, "word count"));
  job.setJarByClass(WordCount.class);
  job.setMapperClass(TokenizerMapper.class);
  job.setCombinerClass(IntSumReducer.class);
  job.setReducerClass(IntSumReducer.class);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(IntWritable.class);
  FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
  FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
  System.exit(job.waitForCompletion(true) ? 0 : 1);
}

项目：aliyun-oss-hadoop-fs 文件：Submitter.java

void printUsage() {
  // The CLI package should do this for us, but I can't figure out how
  // to make it print something reasonable.
  System.out.println("Usage: pipes ");
  System.out.println("  [-input <path>] // Input directory");
  System.out.println("  [-output <path>] // Output directory");
  System.out.println("  [-jar <jar file> // jar filename");
  System.out.println("  [-inputformat <class>] // InputFormat class");
  System.out.println("  [-map <class>] // Java Map class");
  System.out.println("  [-partitioner <class>] // Java Partitioner");
  System.out.println("  [-reduce <class>] // Java Reduce class");
  System.out.println("  [-writer <class>] // Java RecordWriter");
  System.out.println("  [-program <executable>] // executable URI");
  System.out.println("  [-reduces <num>] // number of reduces");
  System.out.println("  [-lazyOutput <true/false>] // createOutputLazily");
  System.out.println();
  GenericOptionsParser.printGenericCommandUsage(System.out);
}

项目：aliyun-oss-hadoop-fs 文件：JobHistoryServer.java

static JobHistoryServer launchJobHistoryServer(String[] args) {
  Thread.
      setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
  StringUtils.startupShutdownMessage(JobHistoryServer.class, args, LOG);
  JobHistoryServer jobHistoryServer = null;
  try {
    jobHistoryServer = new JobHistoryServer();
    ShutdownHookManager.get().addShutdownHook(
        new CompositeServiceShutdownHook(jobHistoryServer),
        SHUTDOWN_HOOK_PRIORITY);
    YarnConfiguration conf = new YarnConfiguration(new JobConf());
    new GenericOptionsParser(conf, args);
    jobHistoryServer.init(conf);
    jobHistoryServer.start();
  } catch (Throwable t) {
    LOG.fatal("Error starting JobHistoryServer", t);
    ExitUtil.terminate(-1, "Error starting JobHistoryServer");
  }
  return jobHistoryServer;
}

项目：aliyun-oss-hadoop-fs 文件：WordCount.java

public static void main(String[] args) throws Exception {
  Configuration conf = new Configuration();
  String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
  if (otherArgs.length < 2) {
    System.err.println("Usage: wordcount <in> [<in>...] <out>");
    System.exit(2);
  }
  Job job = Job.getInstance(conf, "word count");
  job.setJarByClass(WordCount.class);
  job.setMapperClass(TokenizerMapper.class);
  job.setCombinerClass(IntSumReducer.class);
  job.setReducerClass(IntSumReducer.class);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(IntWritable.class);
  for (int i = 0; i < otherArgs.length - 1; ++i) {
    FileInputFormat.addInputPath(job, new Path(otherArgs[i]));
  }
  FileOutputFormat.setOutputPath(job,
    new Path(otherArgs[otherArgs.length - 1]));
  System.exit(job.waitForCompletion(true) ? 0 : 1);
}

项目：aliyun-oss-hadoop-fs 文件：DataNode.java

/** Instantiate a single datanode object, along with its secure resources. 
 * This must be run by invoking{@link DataNode#runDatanodeDaemon()} 
 * subsequently. 
 */
public static DataNode instantiateDataNode(String args [], Configuration conf,
    SecureResources resources) throws IOException {
  if (conf == null)
    conf = new HdfsConfiguration();

  if (args != null) {
    // parse generic hadoop options
    GenericOptionsParser hParser = new GenericOptionsParser(conf, args);
    args = hParser.getRemainingArgs();
  }

  if (!parseArguments(args, conf)) {
    printUsage(System.err);
    return null;
  }
  Collection<StorageLocation> dataLocations = getStorageLocations(conf);
  UserGroupInformation.setConfiguration(conf);
  SecurityUtil.login(conf, DFS_DATANODE_KEYTAB_FILE_KEY,
      DFS_DATANODE_KERBEROS_PRINCIPAL_KEY, getHostName(conf));
  return makeInstance(dataLocations, conf, resources);
}

项目：aliyun-oss-hadoop-fs 文件：DFSZKFailoverController.java

public static void main(String args[])
    throws Exception {
  StringUtils.startupShutdownMessage(DFSZKFailoverController.class,
      args, LOG);
  if (DFSUtil.parseHelpArgument(args, 
      ZKFailoverController.USAGE, System.out, true)) {
    System.exit(0);
  }

  GenericOptionsParser parser = new GenericOptionsParser(
      new HdfsConfiguration(), args);
  DFSZKFailoverController zkfc = DFSZKFailoverController.create(
      parser.getConfiguration());
  int retCode = 0;
  try {
    retCode = zkfc.run(parser.getRemainingArgs());
  } catch (Throwable t) {
    LOG.fatal("Got a fatal error, exiting now", t);
  }
  System.exit(retCode);
}

项目：aliyun-oss-hadoop-fs 文件：DelegationTokenFetcher.java

private static void printUsage(PrintStream err) {
  err.println("fetchdt retrieves delegation tokens from the NameNode");
  err.println();
  err.println("fetchdt <opts> <token file>");
  err.println("Options:");
  err.println("  --webservice <url>  Url to contact NN on (starts with " +
          "http:// or https://)");
  err.println("  --renewer <name>    Name of the delegation token renewer");
  err.println("  --cancel            Cancel the delegation token");
  err.println("  --renew             Renew the delegation token.  " +
          "Delegation " + "token must have been fetched using the --renewer" +
          " <name> option.");
  err.println("  --print             Print the delegation token");
  err.println();
  GenericOptionsParser.printGenericCommandUsage(err);
  ExitUtil.terminate(1);
}

项目：wifi 文件：WordCount.java

public static void main(String[] args) throws Exception {
        Configuration conf = new Configuration();
        String[] otherArgs = new GenericOptionsParser(conf,args).getRemainingArgs();
//      System.out.println(otherArgs);
        if(otherArgs.length != 2) {
            System.out.println("Usage:wordcount <in> <out>");
            System.exit(2);
        }
//      if(args.length != 2) {
//          System.out.println("param error!");
//          System.exit(-1);
//      }

        Job job = new Job(conf, "word count");
        job.setJarByClass(WordCount.class);
        job.setMapperClass(TokenizerMapper.class);
        job.setCombinerClass(IntSumReducer.class);
        job.setReducerClass(IntSumReducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);
        FileInputFormat.addInputPath(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));
        System.exit(job.waitForCompletion(true) ? 0 : 1);

    }

项目：marklogic-contentpump 文件：LinkCountHDFS.java

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    if (args.length < 2) {
        System.err.println("Usage: LinkCountHDFS inputDir outputDir");
        System.exit(2);
    }
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();

    Job job = Job.getInstance(conf, "link count hdfs");
    job.setJarByClass(LinkCountHDFS.class);
    job.setInputFormatClass(HDFSInputFormat.class);
    job.setMapperClass(RefMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(IntWritable.class);

    job.setReducerClass(IntSumReducer.class);
    job.setOutputFormatClass(TextOutputFormat.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);
    HDFSInputFormat.setInputPaths(job, new Path(otherArgs[0]));
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));

    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

项目：marklogic-contentpump 文件：LinkCountInProperty.java

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    if (args.length < 1) {
        System.err.println("Usage: LinkCountInProperty configFile");
        System.exit(2);
    }
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();

    Job job = Job.getInstance(conf, "link count in property");
    job.setJarByClass(LinkCountInProperty.class);
    job.setInputFormatClass(ValueInputFormat.class);
    job.setMapperClass(RefMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(IntWritable.class);
    job.setReducerClass(IntSumReducer.class);
    job.setOutputFormatClass(PropertyOutputFormat.class);
    job.setOutputKeyClass(DocumentURI.class);
    job.setOutputValueClass(MarkLogicNode.class);

    conf = job.getConfiguration();
    conf.addResource(otherArgs[0]);
    conf.setClass(MarkLogicConstants.INPUT_VALUE_CLASS, Text.class, 
            Writable.class);

    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

项目：marklogic-contentpump 文件：RevisionGrouper.java

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    if (args.length < 1) {
        System.err.println("Usage: RevisionGrouper configFile");
        System.exit(2);
    }
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();

    Job job = Job.getInstance(conf, "revision grouper");
    job.setJarByClass(RevisionGrouper.class);
    job.setInputFormatClass(NodeInputFormat.class);
    job.setMapperClass(RevisionMapper.class);

    job.setOutputFormatClass(KeyValueOutputFormat.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    conf = job.getConfiguration();
    conf.addResource(otherArgs[0]);

    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

项目：marklogic-contentpump 文件：ContentReader.java

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    if (args.length < 2) {
        System.err.println("Usage: ContentReader configFile outputDir");
        System.exit(2);
    }
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();

    Job job = Job.getInstance(conf, "content reader");
    job.setJarByClass(ContentReader.class);
    job.setInputFormatClass(DocumentInputFormat.class);
    job.setMapperClass(DocMapper.class);
    job.setMapOutputKeyClass(DocumentURI.class);
    job.setMapOutputValueClass(DatabaseDocument.class);
    job.setOutputFormatClass(CustomOutputFormat.class);

    CustomOutputFormat.setOutputPath(job, new Path(otherArgs[1]));

    conf = job.getConfiguration();
    conf.addResource(otherArgs[0]);
    conf.setClass(MarkLogicConstants.INPUT_SSL_OPTIONS_CLASS,  
            SslOptions.class, SslConfigOptions.class);

    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

项目：marklogic-contentpump 文件：ContentLoader.java

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    if (args.length < 2) {
        System.err.println("Usage: ContentLoader configFile inputDir");
        System.exit(2);
    }
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();

    Job job = Job.getInstance(conf, "content loader");
    job.setJarByClass(ContentLoader.class);
    job.setInputFormatClass(ContentInputFormat.class);
    job.setMapperClass(ContentMapper.class);
    job.setMapOutputKeyClass(DocumentURI.class);
    job.setMapOutputValueClass(Text.class);
    job.setOutputFormatClass(ContentOutputFormat.class);

    ContentInputFormat.setInputPaths(job, new Path(otherArgs[1]));

    conf = job.getConfiguration();
    conf.addResource(otherArgs[0]);

    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

项目：marklogic-contentpump 文件：LinkCountInDoc.java

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    if (args.length < 1) {
        System.err.println("Usage: LinkCountInDoc configFile");
        System.exit(2);
    }
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();

    Job job = Job.getInstance(conf, "link count in doc");
    job.setJarByClass(LinkCountInDoc.class);
    job.setInputFormatClass(NodeInputFormat.class);
    job.setMapperClass(RefMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(IntWritable.class);
    job.setReducerClass(IntSumReducer.class);
    job.setOutputFormatClass(NodeOutputFormat.class);
    job.setOutputKeyClass(NodePath.class);
    job.setOutputValueClass(MarkLogicNode.class);

    conf = job.getConfiguration();
    conf.addResource(otherArgs[0]);

    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

项目：marklogic-contentpump 文件：WikiLoader.java

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    if (args.length < 2) {
        System.err.println("Usage: WikiLoader configFile inputDir");
        System.exit(2);
    }
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();

    Job job = Job.getInstance(conf, "wiki loader");
    job.setJarByClass(WikiLoader.class);
    job.setInputFormatClass(WikiInputFormat.class);
    job.setMapperClass(ArticleMapper.class);
    job.setMapOutputKeyClass(DocumentURI.class);
    job.setMapOutputValueClass(Text.class);
    job.setOutputFormatClass(ContentOutputFormat.class);

    ContentInputFormat.setInputPaths(job, new Path(otherArgs[1]));

    conf = job.getConfiguration();
    conf.addResource(otherArgs[0]);

    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

项目：marklogic-contentpump 文件：BinaryReader.java

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();

    if (args.length < 2) {
        System.err.println("Usage: BinaryReader configFile outputDir");
        System.exit(2);
    }
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    Job job = Job.getInstance(conf, "binary reader");
    job.setJarByClass(BinaryReader.class);
    job.setInputFormatClass(DocumentInputFormat.class);
    job.setMapperClass(DocMapper.class);
    job.setMapOutputKeyClass(DocumentURI.class);
    job.setMapOutputValueClass(BytesWritable.class);
    job.setOutputFormatClass(BinaryOutputFormat.class);
    job.setOutputKeyClass(DocumentURI.class);
    job.setOutputValueClass(BytesWritable.class);
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));

    conf = job.getConfiguration();
    conf.addResource(otherArgs[0]);

    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

项目：marklogic-contentpump 文件：ZipContentLoader.java

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length != 2) {
      System.err.println("Usage: ZipContentLoader configFile inputDir");
      System.exit(2);
    }

    Job job = Job.getInstance(conf, "zip content loader");
    job.setJarByClass(ZipContentLoader.class);
    job.setInputFormatClass(ZipContentInputFormat.class);
    job.setMapperClass(ZipContentMapper.class);
    job.setMapOutputKeyClass(DocumentURI.class);
    job.setMapOutputValueClass(Text.class);
    job.setOutputFormatClass(ContentOutputFormat.class);

    ZipContentInputFormat.setInputPaths(job, new Path(otherArgs[1]));

    conf = job.getConfiguration();
    conf.addResource(otherArgs[0]);

    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

项目：marklogic-contentpump 文件：HelloWorld.java

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();

    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    Job job = Job.getInstance(conf, "hello world");
    job.setJarByClass(HelloWorld.class);

    // Map related configuration
    job.setInputFormatClass(DocumentInputFormat.class);
    job.setMapperClass(MyMapper.class);
    job.setMapOutputKeyClass(IntWritable.class);
    job.setMapOutputValueClass(Text.class);

    // Reduce related configuration
    job.setReducerClass(MyReducer.class);
    job.setOutputFormatClass(ContentOutputFormat.class);
    job.setOutputKeyClass(DocumentURI.class);
    job.setOutputValueClass(Text.class);

    conf = job.getConfiguration();
    conf.addResource("marklogic-hello-world.xml");

    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

项目：marklogic-contentpump 文件：ElementValuesTest.java

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 2) {
        System.err.println("Usage: ElementValuesTest configFile outputDir");
        System.exit(2);
    }

    Job job = Job.getInstance(conf);
    job.setJarByClass(ElementValuesTest.class);
    job.setInputFormatClass(ValueInputFormat.class);
    job.setMapperClass(ElementValueMapper.class);
    job.setMapOutputKeyClass(LongWritable.class);
    job.setMapOutputValueClass(Text.class);
    job.setOutputFormatClass(TextOutputFormat.class);
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));

    conf = job.getConfiguration();
    conf.addResource(otherArgs[0]);
    conf.setClass(MarkLogicConstants.INPUT_VALUE_CLASS, Text.class, 
            Writable.class);
    conf.setClass(MarkLogicConstants.INPUT_LEXICON_FUNCTION_CLASS, 
        ElementValuesFunction.class, ElementValues.class);

    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

项目：marklogic-contentpump 文件：ElementValueMatchTest.java

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 1) {
        System.err.println("Usage: ElementValueMatchTest configFile outputDir");
        System.exit(2);
    }

    Job job = Job.getInstance(conf);
    job.setJarByClass(ElementValueMatchTest.class);
    job.setInputFormatClass(ValueInputFormat.class);
    job.setMapperClass(ElementValueMatchMapper.class);
    job.setMapOutputKeyClass(LongWritable.class);
    job.setMapOutputValueClass(Text.class);
    job.setOutputFormatClass(TextOutputFormat.class);
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));

    conf = job.getConfiguration();
    conf.addResource(otherArgs[0]);
    conf.setClass(MarkLogicConstants.INPUT_VALUE_CLASS, Text.class, 
            Writable.class);
    conf.setClass(MarkLogicConstants.INPUT_LEXICON_FUNCTION_CLASS, 
        ElementValueMatchFunction.class, ElementValueMatch.class);

    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

项目：marklogic-contentpump 文件：ElementAttributeValuesTest.java

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 2) {
        System.err.println("Usage: ElementAttributeValuesTest configFile outputDir");
        System.exit(2);
    }

    Job job = Job.getInstance(conf);
    job.setJarByClass(ElementAttributeValuesTest.class);
    job.setInputFormatClass(ValueInputFormat.class);
    job.setMapperClass(ElementAttrValueMapper.class);
    job.setMapOutputKeyClass(LongWritable.class);
    job.setMapOutputValueClass(Text.class);
    job.setOutputFormatClass(TextOutputFormat.class);
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));

    conf = job.getConfiguration();
    conf.addResource(otherArgs[0]);
    conf.setClass(MarkLogicConstants.INPUT_VALUE_CLASS, Text.class, 
            Writable.class);
    conf.setClass(MarkLogicConstants.INPUT_LEXICON_FUNCTION_CLASS, 
        ElementAttributeValuesFunction.class, ElementAttributeValues.class);

    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

项目：marklogic-contentpump 文件：MapTreeReduceTree.java

@Override
public int run(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 2) {
        System.err.println("Usage: MapTreeReduceTree inputpath outputpath");
        System.exit(2);
    }
    Job job = Job.getInstance(super.getConf());
    job.setJarByClass(MapTreeReduceTree.class);

    // Map related configuration
    job.setInputFormatClass(ForestInputFormat.class);
    job.setMapperClass(MyMapper.class);
    job.setMapOutputKeyClass(DocumentURIWithSourceInfo.class);
    job.setMapOutputValueClass(DOMDocument.class);
    job.setReducerClass(MyReducer.class);
    FileInputFormat.setInputPaths(job, new Path(otherArgs[0]));
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));

    job.waitForCompletion(true);
    return 0;
}

项目：marklogic-contentpump 文件：ValueMatchTest.java

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 2) {
        System.err.println("Usage: ValueMatchTest configFile outputDir");
        System.exit(2);
    }

    Job job = Job.getInstance(conf);
    job.setJarByClass(ValueMatchTest.class);
    job.setInputFormatClass(ValueInputFormat.class);
    job.setMapperClass(ValueMatchMapper.class);
    job.setMapOutputKeyClass(LongWritable.class);
    job.setMapOutputValueClass(Text.class);
    job.setOutputFormatClass(TextOutputFormat.class);
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));

    conf = job.getConfiguration();
    conf.addResource(otherArgs[0]);
    conf.setClass(MarkLogicConstants.INPUT_VALUE_CLASS, Text.class, 
            Writable.class);
    conf.setClass(MarkLogicConstants.INPUT_LEXICON_FUNCTION_CLASS, 
        ValueMatchFunction.class, ValueMatch.class);

    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

项目：marklogic-contentpump 文件：MultithreadedZipContentLoader.java

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 2) {
        System.err.println("Usage: MultithreadedZipContentLoader configFile inputDir threadCount");
        System.exit(2);
    }

    Job job = Job.getInstance(conf);
    job.setJarByClass(MultithreadedZipContentLoader.class);
    job.setInputFormatClass(ZipContentInputFormat.class);
    job.setMapperClass(MultithreadedMapper.class);
    MultithreadedMapper.setMapperClass(job, ZipContentMapper.class);
    MultithreadedMapper.setNumberOfThreads(job, Integer.parseInt(args[2]));
    job.setMapOutputKeyClass(DocumentURI.class);
    job.setMapOutputValueClass(Text.class);
    job.setOutputFormatClass(ContentOutputFormat.class);

    ZipContentInputFormat.setInputPaths(job, new Path(otherArgs[1]));

    conf = job.getConfiguration();
    conf.addResource(otherArgs[0]);

    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

项目：marklogic-contentpump 文件：WordsTest.java

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 1) {
        System.err.println("Usage: WordsTest configFile outputDir");
        System.exit(2);
    }

    Job job = Job.getInstance(conf);
    job.setJarByClass(WordsTest.class);
    job.setInputFormatClass(ValueInputFormat.class);
    job.setMapperClass(WordsMapper.class);
    job.setMapOutputKeyClass(LongWritable.class);
    job.setMapOutputValueClass(Text.class);
    job.setOutputFormatClass(TextOutputFormat.class);
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));

    conf = job.getConfiguration();
    conf.addResource(otherArgs[0]);
    conf.setClass(MarkLogicConstants.INPUT_VALUE_CLASS, Text.class, 
            Writable.class);
    conf.setClass(MarkLogicConstants.INPUT_LEXICON_FUNCTION_CLASS, 
        Words.class, Words.class);

    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

项目：marklogic-contentpump 文件：CustomQuery.java

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    if (args.length < 1) {
        System.err.println("Usage: CustomQuery configFile");
        System.exit(2);
    }
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();

    Job job = Job.getInstance(conf, "custom query");
    job.setJarByClass(CustomQuery.class);

    job.setInputFormatClass(NodeInputFormat.class);
    job.setMapperClass(QueryMapper.class);
    job.setMapOutputKeyClass(IntWritable.class);
    job.setMapOutputValueClass(Text.class);

    job.setReducerClass(QueryReducer.class);
    job.setOutputFormatClass(KeyValueOutputFormat.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    conf = job.getConfiguration();
    conf.addResource(otherArgs[0]);

    System.exit(job.waitForCompletion(true) ? 0 : 1);
}