Java 类org.apache.hadoop.mapreduce.lib.db.DBConfiguration 实例源码

项目:bigdata_pattern    文件:FileToDBMapReduce.java   
public static void main(String[] args) throws IOException, Exception {  
Configuration conf = new Configuration();
Job job = new Job(conf);
job.setJarByClass(FileToDBMapReduce.class);
   job.setInputFormatClass(TextInputFormat.class);  
   job.setOutputFormatClass(DBOutputFormat.class);  

   Path inputPath = new Path("/hua/hua.bcp");
   FileInputFormat.setInputPaths(job, inputPath);  
   DBConfiguration.configureDB(conf, "com.mysql.jdbc.Driver",  
        "jdbc:mysql://192.168.3.244:3306/hadoop", "hua", "hadoop");  
   DBOutputFormat.setOutput(job, "studentinfo", "id", "name");  

   job.setMapperClass(Mapper.class);  
   job.setReducerClass(MyReducer.class);  

   job.waitForCompletion(true);  
 }
项目:bigdata_pattern    文件:DBToFileMapReduce.java   
public static void main(String[] args) throws IOException, Exception {  
 Configuration conf = new Configuration();
 Job job = new Job(conf);
 job.setJarByClass(DBToFileMapReduce.class);  
 DistributedCache.addFileToClassPath(new Path(  
       "/lib/mysql-connector-java-5.1.0-bin.jar"), conf);  

 job.setMapperClass(DBInputMapper.class);  
 job.setReducerClass(Reducer.class); 

 job.setMapOutputKeyClass(LongWritable.class);  
 job.setMapOutputValueClass(Text.class);  
 job.setOutputKeyClass(LongWritable.class);  
 job.setOutputValueClass(Text.class);  

 job.setInputFormatClass(DBInputFormat.class); 

 Path outputPath = new Path("/hua01");
 FileOutputFormat.setOutputPath(job, outputPath);  
 outputPath.getFileSystem(conf).delete(outputPath, true);

 DBConfiguration.configureDB(conf, "com.mysql.jdbc.Driver",  
       "jdbc:mysql://192.168.3.244:3306/hadoop", "hua", "hadoop");  
 String[] fields = { "id", "name" };  
 DBInputFormat.setInput(job, StudentinfoRecord.class, "studentinfo", null, "id", fields);   

 job.waitForCompletion(true); 
}
项目:Hot-Search-Terms    文件:Dispatcher.java   
public static void main(String[] args) throws ClassNotFoundException, IOException, InterruptedException {
        // configure n-gram mapreduce job
        Configuration conf1 = new Configuration();
        conf1.set("textinputformat.record.delimiter", "."); // read a complete sentence as a line
        conf1.set("GRAM_NUMBER", args[2]);
        Job job1 = Job.getInstance(conf1);
        job1.setNumReduceTasks(3);
        job1.setJobName("NGram");
        job1.setJarByClass(Dispatcher.class);
        job1.setMapperClass(NGramBuilder.NGramMapper.class);
        job1.setReducerClass(NGramBuilder.NGramReducer.class);
        job1.setOutputKeyClass(Text.class);
        job1.setOutputValueClass(IntWritable.class);
        job1.setInputFormatClass(TextInputFormat.class); // default format: reads lines of text files
        job1.setOutputFormatClass(TextOutputFormat.class); // default format: key \t value
        TextInputFormat.setInputPaths(job1, new Path(args[0]));
        TextOutputFormat.setOutputPath(job1, new Path(args[1]));
        job1.waitForCompletion(true); // language model won't start to build until the n-gram library completely built

        // configure language model mapreduce job
        Configuration conf2 = new Configuration();
        conf2.set("THRESHOLD", args[3]);
        conf2.set("TOP_K", args[4]);
        DBConfiguration.configureDB(conf2, "com.mysql.jdbc.Driver", "jdbc:mysql://127.0.0.1:3306/tp", "root", "123456"); // establish connection with mySQL database   
        Job job2 = Job.getInstance(conf2);
        job2.setNumReduceTasks(3);
        job2.setJobName("LModel");
        job2.setJarByClass(Dispatcher.class);           
        job2.addArchiveToClassPath(new Path("/mysql/mysql-connector-java-5.1.39-bin.jar")); // putting this jar file into jre/lib/ext is recommended    
        job2.setMapperClass(LanguageModel.ModelMapper.class);
        job2.setReducerClass(LanguageModel.ModelReducer.class);
        job2.setMapOutputKeyClass(Text.class); // Mapper emits different key type than the Reducer
        job2.setMapOutputValueClass(Text.class); // Mapper emits different value type than the Reducer
        job2.setOutputKeyClass(DBOutputWritable.class);
        job2.setOutputValueClass(NullWritable.class);
        job2.setInputFormatClass(TextInputFormat.class);
        job2.setOutputFormatClass(DBOutputFormat.class);
        TextInputFormat.setInputPaths(job2, new Path(args[1]));
        DBOutputFormat.setOutput(job2, "LanguageModel", new String[] {"starter", "follower", "probability"});
        System.exit(job2.waitForCompletion(true) ? 0 : 1);
}
项目:hadoop    文件:DBCountPageView.java   
@Override
//Usage DBCountPageView [driverClass dburl]
public int run(String[] args) throws Exception {

  String driverClassName = DRIVER_CLASS;
  String url = DB_URL;

  if(args.length > 1) {
    driverClassName = args[0];
    url = args[1];
  }

  initialize(driverClassName, url);
  Configuration conf = getConf();

  DBConfiguration.configureDB(conf, driverClassName, url);

  Job job = new Job(conf);

  job.setJobName("Count Pageviews of URLs");
  job.setJarByClass(DBCountPageView.class);
  job.setMapperClass(PageviewMapper.class);
  job.setCombinerClass(LongSumReducer.class);
  job.setReducerClass(PageviewReducer.class);

  DBInputFormat.setInput(job, AccessRecord.class, "Access"
      , null, "url", AccessFieldNames);

  DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);

  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(LongWritable.class);

  job.setOutputKeyClass(PageviewRecord.class);
  job.setOutputValueClass(NullWritable.class);
  int ret;
  try {
    ret = job.waitForCompletion(true) ? 0 : 1;
    boolean correct = verify();
    if(!correct) {
      throw new RuntimeException("Evaluation was not correct!");
    }
  } finally {
    shutdown();    
  }
  return ret;
}
项目:mapreduce-samples    文件:Driver.java   
public static void main(String[] args) throws ClassNotFoundException, IOException, InterruptedException {
    Configuration conf1 = new Configuration();
    conf1.set("textinputformat.record.delimiter", ".");
    conf1.set("noGram", args[2]);

    //First Job 
    Job job1 = Job.getInstance(conf1);
    job1.setJobName("NGram");
    job1.setJarByClass(Driver.class);

    job1.setMapperClass(NGramLibraryBuilder.NGramMapper.class);
    job1.setReducerClass(NGramLibraryBuilder.NGramReducer.class);

    job1.setOutputKeyClass(Text.class);
    job1.setOutputValueClass(IntWritable.class);

    job1.setInputFormatClass(TextInputFormat.class);
    job1.setOutputFormatClass(TextOutputFormat.class);

    TextInputFormat.setInputPaths(job1, new Path(args[0]));
    TextOutputFormat.setOutputPath(job1, new Path(args[1]));
    job1.waitForCompletion(true);

    //Second Job 
    Configuration conf2 = new Configuration();
    conf2.set("threashold", args[3]);
    conf2.set("n", args[4]);
    DBConfiguration.configureDB(conf2,
             "com.mysql.jdbc.Driver",   // driver class
             "jdbc:mysql://10.101.0.163:8889/test", //
             "root",    // user name
             "root"); //password

    Job job2 = Job.getInstance(conf2);
    job2.setJobName("LanguageModel");
    job2.setJarByClass(Driver.class);

    job2.addArchiveToClassPath(new Path("/mysql/mysql-connector-java-5.1.39-bin.jar"));

    job2.setMapOutputKeyClass(Text.class);
    job2.setMapOutputValueClass(Text.class);
    job2.setOutputKeyClass(Text.class);
    job2.setOutputValueClass(NullWritable.class);

    job2.setMapperClass(LanguageModel.Map.class);
    job2.setReducerClass(LanguageModel.Reduce.class);

    job2.setInputFormatClass(TextInputFormat.class);
    job2.setOutputFormatClass(DBOutputFormat.class);

    DBOutputFormat.setOutput(
             job2,
             "output",    // output table name
             new String[] { "starting_phrase", "following_word", "count" }   //table columns
             );

    //Path name for this job should match first job's output path name
    TextInputFormat.setInputPaths(job2, new Path(args[1]));
    System.exit(job2.waitForCompletion(true)?0:1);

}
项目:aliyun-oss-hadoop-fs    文件:DBCountPageView.java   
@Override
//Usage DBCountPageView [driverClass dburl]
public int run(String[] args) throws Exception {

  String driverClassName = DRIVER_CLASS;
  String url = DB_URL;

  if(args.length > 1) {
    driverClassName = args[0];
    url = args[1];
  }

  initialize(driverClassName, url);
  Configuration conf = getConf();

  DBConfiguration.configureDB(conf, driverClassName, url);

  Job job = Job.getInstance(conf);

  job.setJobName("Count Pageviews of URLs");
  job.setJarByClass(DBCountPageView.class);
  job.setMapperClass(PageviewMapper.class);
  job.setCombinerClass(LongSumReducer.class);
  job.setReducerClass(PageviewReducer.class);

  DBInputFormat.setInput(job, AccessRecord.class, "HAccess"
      , null, "url", AccessFieldNames);

  DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);

  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(LongWritable.class);

  job.setOutputKeyClass(PageviewRecord.class);
  job.setOutputValueClass(NullWritable.class);
  int ret;
  try {
    ret = job.waitForCompletion(true) ? 0 : 1;
    boolean correct = verify();
    if(!correct) {
      throw new RuntimeException("Evaluation was not correct!");
    }
  } finally {
    shutdown();    
  }
  return ret;
}
项目:big-c    文件:DBCountPageView.java   
@Override
//Usage DBCountPageView [driverClass dburl]
public int run(String[] args) throws Exception {

  String driverClassName = DRIVER_CLASS;
  String url = DB_URL;

  if(args.length > 1) {
    driverClassName = args[0];
    url = args[1];
  }

  initialize(driverClassName, url);
  Configuration conf = getConf();

  DBConfiguration.configureDB(conf, driverClassName, url);

  Job job = new Job(conf);

  job.setJobName("Count Pageviews of URLs");
  job.setJarByClass(DBCountPageView.class);
  job.setMapperClass(PageviewMapper.class);
  job.setCombinerClass(LongSumReducer.class);
  job.setReducerClass(PageviewReducer.class);

  DBInputFormat.setInput(job, AccessRecord.class, "Access"
      , null, "url", AccessFieldNames);

  DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);

  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(LongWritable.class);

  job.setOutputKeyClass(PageviewRecord.class);
  job.setOutputValueClass(NullWritable.class);
  int ret;
  try {
    ret = job.waitForCompletion(true) ? 0 : 1;
    boolean correct = verify();
    if(!correct) {
      throw new RuntimeException("Evaluation was not correct!");
    }
  } finally {
    shutdown();    
  }
  return ret;
}
项目:hadoop-2.6.0-cdh5.4.3    文件:DBCountPageView.java   
@Override
//Usage DBCountPageView [driverClass dburl]
public int run(String[] args) throws Exception {

  String driverClassName = DRIVER_CLASS;
  String url = DB_URL;

  if(args.length > 1) {
    driverClassName = args[0];
    url = args[1];
  }

  initialize(driverClassName, url);
  Configuration conf = getConf();

  DBConfiguration.configureDB(conf, driverClassName, url);

  Job job = new Job(conf);

  job.setJobName("Count Pageviews of URLs");
  job.setJarByClass(DBCountPageView.class);
  job.setMapperClass(PageviewMapper.class);
  job.setCombinerClass(LongSumReducer.class);
  job.setReducerClass(PageviewReducer.class);

  DBInputFormat.setInput(job, AccessRecord.class, "Access"
      , null, "url", AccessFieldNames);

  DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);

  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(LongWritable.class);

  job.setOutputKeyClass(PageviewRecord.class);
  job.setOutputValueClass(NullWritable.class);
  int ret;
  try {
    ret = job.waitForCompletion(true) ? 0 : 1;
    boolean correct = verify();
    if(!correct) {
      throw new RuntimeException("Evaluation was not correct!");
    }
  } finally {
    shutdown();    
  }
  return ret;
}
项目:hadoop-plus    文件:DBCountPageView.java   
@Override
//Usage DBCountPageView [driverClass dburl]
public int run(String[] args) throws Exception {

  String driverClassName = DRIVER_CLASS;
  String url = DB_URL;

  if(args.length > 1) {
    driverClassName = args[0];
    url = args[1];
  }

  initialize(driverClassName, url);
  Configuration conf = getConf();

  DBConfiguration.configureDB(conf, driverClassName, url);

  Job job = new Job(conf);

  job.setJobName("Count Pageviews of URLs");
  job.setJarByClass(DBCountPageView.class);
  job.setMapperClass(PageviewMapper.class);
  job.setCombinerClass(LongSumReducer.class);
  job.setReducerClass(PageviewReducer.class);

  DBInputFormat.setInput(job, AccessRecord.class, "Access"
      , null, "url", AccessFieldNames);

  DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);

  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(LongWritable.class);

  job.setOutputKeyClass(PageviewRecord.class);
  job.setOutputValueClass(NullWritable.class);
  int ret;
  try {
    ret = job.waitForCompletion(true) ? 0 : 1;
    boolean correct = verify();
    if(!correct) {
      throw new RuntimeException("Evaluation was not correct!");
    }
  } finally {
    shutdown();    
  }
  return ret;
}
项目:FlexMap    文件:DBCountPageView.java   
@Override
//Usage DBCountPageView [driverClass dburl]
public int run(String[] args) throws Exception {

  String driverClassName = DRIVER_CLASS;
  String url = DB_URL;

  if(args.length > 1) {
    driverClassName = args[0];
    url = args[1];
  }

  initialize(driverClassName, url);
  Configuration conf = getConf();

  DBConfiguration.configureDB(conf, driverClassName, url);

  Job job = new Job(conf);

  job.setJobName("Count Pageviews of URLs");
  job.setJarByClass(DBCountPageView.class);
  job.setMapperClass(PageviewMapper.class);
  job.setCombinerClass(LongSumReducer.class);
  job.setReducerClass(PageviewReducer.class);

  DBInputFormat.setInput(job, AccessRecord.class, "Access"
      , null, "url", AccessFieldNames);

  DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);

  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(LongWritable.class);

  job.setOutputKeyClass(PageviewRecord.class);
  job.setOutputValueClass(NullWritable.class);
  int ret;
  try {
    ret = job.waitForCompletion(true) ? 0 : 1;
    boolean correct = verify();
    if(!correct) {
      throw new RuntimeException("Evaluation was not correct!");
    }
  } finally {
    shutdown();    
  }
  return ret;
}
项目:hops    文件:DBCountPageView.java   
@Override
//Usage DBCountPageView [driverClass dburl]
public int run(String[] args) throws Exception {

  String driverClassName = DRIVER_CLASS;
  String url = DB_URL;

  if(args.length > 1) {
    driverClassName = args[0];
    url = args[1];
  }

  initialize(driverClassName, url);
  Configuration conf = getConf();

  DBConfiguration.configureDB(conf, driverClassName, url);

  Job job = new Job(conf);

  job.setJobName("Count Pageviews of URLs");
  job.setJarByClass(DBCountPageView.class);
  job.setMapperClass(PageviewMapper.class);
  job.setCombinerClass(LongSumReducer.class);
  job.setReducerClass(PageviewReducer.class);

  DBInputFormat.setInput(job, AccessRecord.class, "Access"
      , null, "url", AccessFieldNames);

  DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);

  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(LongWritable.class);

  job.setOutputKeyClass(PageviewRecord.class);
  job.setOutputValueClass(NullWritable.class);
  int ret;
  try {
    ret = job.waitForCompletion(true) ? 0 : 1;
    boolean correct = verify();
    if(!correct) {
      throw new RuntimeException("Evaluation was not correct!");
    }
  } finally {
    shutdown();    
  }
  return ret;
}
项目:glusterfs-hadoop-examples    文件:DBCountPageView.java   
@Override
//Usage DBCountPageView [driverClass dburl]
public int run(String[] args) throws Exception {

  String driverClassName = DRIVER_CLASS;
  String url = DB_URL;

  if(args.length > 1) {
    driverClassName = args[0];
    url = args[1];
  }

  initialize(driverClassName, url);
  Configuration conf = getConf();

  DBConfiguration.configureDB(conf, driverClassName, url);

  Job job = new Job(conf);

  job.setJobName("Count Pageviews of URLs");
  job.setJarByClass(DBCountPageView.class);
  job.setMapperClass(PageviewMapper.class);
  job.setCombinerClass(LongSumReducer.class);
  job.setReducerClass(PageviewReducer.class);

  DBInputFormat.setInput(job, AccessRecord.class, "Access"
      , null, "url", AccessFieldNames);

  DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);

  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(LongWritable.class);

  job.setOutputKeyClass(PageviewRecord.class);
  job.setOutputValueClass(NullWritable.class);
  int ret;
  try {
    ret = job.waitForCompletion(true) ? 0 : 1;
    boolean correct = verify();
    if(!correct) {
      throw new RuntimeException("Evaluation was not correct!");
    }
  } finally {
    shutdown();    
  }
  return ret;
}
项目:hadoop-TCP    文件:DBCountPageView.java   
@Override
//Usage DBCountPageView [driverClass dburl]
public int run(String[] args) throws Exception {

  String driverClassName = DRIVER_CLASS;
  String url = DB_URL;

  if(args.length > 1) {
    driverClassName = args[0];
    url = args[1];
  }

  initialize(driverClassName, url);
  Configuration conf = getConf();

  DBConfiguration.configureDB(conf, driverClassName, url);

  Job job = new Job(conf);

  job.setJobName("Count Pageviews of URLs");
  job.setJarByClass(DBCountPageView.class);
  job.setMapperClass(PageviewMapper.class);
  job.setCombinerClass(LongSumReducer.class);
  job.setReducerClass(PageviewReducer.class);

  DBInputFormat.setInput(job, AccessRecord.class, "Access"
      , null, "url", AccessFieldNames);

  DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);

  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(LongWritable.class);

  job.setOutputKeyClass(PageviewRecord.class);
  job.setOutputValueClass(NullWritable.class);
  int ret;
  try {
    ret = job.waitForCompletion(true) ? 0 : 1;
    boolean correct = verify();
    if(!correct) {
      throw new RuntimeException("Evaluation was not correct!");
    }
  } finally {
    shutdown();    
  }
  return ret;
}
项目:hardfs    文件:DBCountPageView.java   
@Override
//Usage DBCountPageView [driverClass dburl]
public int run(String[] args) throws Exception {

  String driverClassName = DRIVER_CLASS;
  String url = DB_URL;

  if(args.length > 1) {
    driverClassName = args[0];
    url = args[1];
  }

  initialize(driverClassName, url);
  Configuration conf = getConf();

  DBConfiguration.configureDB(conf, driverClassName, url);

  Job job = new Job(conf);

  job.setJobName("Count Pageviews of URLs");
  job.setJarByClass(DBCountPageView.class);
  job.setMapperClass(PageviewMapper.class);
  job.setCombinerClass(LongSumReducer.class);
  job.setReducerClass(PageviewReducer.class);

  DBInputFormat.setInput(job, AccessRecord.class, "Access"
      , null, "url", AccessFieldNames);

  DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);

  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(LongWritable.class);

  job.setOutputKeyClass(PageviewRecord.class);
  job.setOutputValueClass(NullWritable.class);
  int ret;
  try {
    ret = job.waitForCompletion(true) ? 0 : 1;
    boolean correct = verify();
    if(!correct) {
      throw new RuntimeException("Evaluation was not correct!");
    }
  } finally {
    shutdown();    
  }
  return ret;
}
项目:hadoop-on-lustre2    文件:DBCountPageView.java   
@Override
//Usage DBCountPageView [driverClass dburl]
public int run(String[] args) throws Exception {

  String driverClassName = DRIVER_CLASS;
  String url = DB_URL;

  if(args.length > 1) {
    driverClassName = args[0];
    url = args[1];
  }

  initialize(driverClassName, url);
  Configuration conf = getConf();

  DBConfiguration.configureDB(conf, driverClassName, url);

  Job job = new Job(conf);

  job.setJobName("Count Pageviews of URLs");
  job.setJarByClass(DBCountPageView.class);
  job.setMapperClass(PageviewMapper.class);
  job.setCombinerClass(LongSumReducer.class);
  job.setReducerClass(PageviewReducer.class);

  DBInputFormat.setInput(job, AccessRecord.class, "Access"
      , null, "url", AccessFieldNames);

  DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);

  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(LongWritable.class);

  job.setOutputKeyClass(PageviewRecord.class);
  job.setOutputValueClass(NullWritable.class);
  int ret;
  try {
    ret = job.waitForCompletion(true) ? 0 : 1;
    boolean correct = verify();
    if(!correct) {
      throw new RuntimeException("Evaluation was not correct!");
    }
  } finally {
    shutdown();    
  }
  return ret;
}
项目:mapreduce-fork    文件:DBCountPageView.java   
@Override
//Usage DBCountPageView [driverClass dburl]
public int run(String[] args) throws Exception {

  String driverClassName = DRIVER_CLASS;
  String url = DB_URL;

  if(args.length > 1) {
    driverClassName = args[0];
    url = args[1];
  }

  initialize(driverClassName, url);
  Configuration conf = getConf();

  DBConfiguration.configureDB(conf, driverClassName, url);

  Job job = new Job(conf);

  job.setJobName("Count Pageviews of URLs");
  job.setJarByClass(DBCountPageView.class);
  job.setMapperClass(PageviewMapper.class);
  job.setCombinerClass(LongSumReducer.class);
  job.setReducerClass(PageviewReducer.class);

  DBInputFormat.setInput(job, AccessRecord.class, "Access"
      , null, "url", AccessFieldNames);

  DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);

  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(LongWritable.class);

  job.setOutputKeyClass(PageviewRecord.class);
  job.setOutputValueClass(NullWritable.class);
  int ret;
  try {
    ret = job.waitForCompletion(true) ? 0 : 1;
    boolean correct = verify();
    if(!correct) {
      throw new RuntimeException("Evaluation was not correct!");
    }
  } finally {
    shutdown();    
  }
  return ret;
}