public static void main(String[] args) throws IOException, Exception { Configuration conf = new Configuration(); Job job = new Job(conf); job.setJarByClass(DBToFileMapReduce.class); DistributedCache.addFileToClassPath(new Path( "/lib/mysql-connector-java-5.1.0-bin.jar"), conf); job.setMapperClass(DBInputMapper.class); job.setReducerClass(Reducer.class); job.setMapOutputKeyClass(LongWritable.class); job.setMapOutputValueClass(Text.class); job.setOutputKeyClass(LongWritable.class); job.setOutputValueClass(Text.class); job.setInputFormatClass(DBInputFormat.class); Path outputPath = new Path("/hua01"); FileOutputFormat.setOutputPath(job, outputPath); outputPath.getFileSystem(conf).delete(outputPath, true); DBConfiguration.configureDB(conf, "com.mysql.jdbc.Driver", "jdbc:mysql://192.168.3.244:3306/hadoop", "hua", "hadoop"); String[] fields = { "id", "name" }; DBInputFormat.setInput(job, StudentinfoRecord.class, "studentinfo", null, "id", fields); job.waitForCompletion(true); }
@Override //Usage DBCountPageView [driverClass dburl] public int run(String[] args) throws Exception { String driverClassName = DRIVER_CLASS; String url = DB_URL; if(args.length > 1) { driverClassName = args[0]; url = args[1]; } initialize(driverClassName, url); Configuration conf = getConf(); DBConfiguration.configureDB(conf, driverClassName, url); Job job = new Job(conf); job.setJobName("Count Pageviews of URLs"); job.setJarByClass(DBCountPageView.class); job.setMapperClass(PageviewMapper.class); job.setCombinerClass(LongSumReducer.class); job.setReducerClass(PageviewReducer.class); DBInputFormat.setInput(job, AccessRecord.class, "Access" , null, "url", AccessFieldNames); DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(LongWritable.class); job.setOutputKeyClass(PageviewRecord.class); job.setOutputValueClass(NullWritable.class); int ret; try { ret = job.waitForCompletion(true) ? 0 : 1; boolean correct = verify(); if(!correct) { throw new RuntimeException("Evaluation was not correct!"); } } finally { shutdown(); } return ret; }
@Override //Usage DBCountPageView [driverClass dburl] public int run(String[] args) throws Exception { String driverClassName = DRIVER_CLASS; String url = DB_URL; if(args.length > 1) { driverClassName = args[0]; url = args[1]; } initialize(driverClassName, url); Configuration conf = getConf(); DBConfiguration.configureDB(conf, driverClassName, url); Job job = Job.getInstance(conf); job.setJobName("Count Pageviews of URLs"); job.setJarByClass(DBCountPageView.class); job.setMapperClass(PageviewMapper.class); job.setCombinerClass(LongSumReducer.class); job.setReducerClass(PageviewReducer.class); DBInputFormat.setInput(job, AccessRecord.class, "HAccess" , null, "url", AccessFieldNames); DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(LongWritable.class); job.setOutputKeyClass(PageviewRecord.class); job.setOutputValueClass(NullWritable.class); int ret; try { ret = job.waitForCompletion(true) ? 0 : 1; boolean correct = verify(); if(!correct) { throw new RuntimeException("Evaluation was not correct!"); } } finally { shutdown(); } return ret; }
/** * Validate whether class is {@link DBInputFormat} or a subclass. * * @param clazz the class to validate * @param ctx the context to throw the error in if class is invalid * @param scope the scope to throw the error in if class is invalid */ public static void validateClass(final Class<?> clazz, final Context ctx, final Scriptable scope) { if (!ReflectionUtils.isClassOrSubclass(DBInputFormat.class, clazz)) { throw Utils.makeError(ctx, scope, LembosMessages.makeInvalidClassErrorMessage(DBInputFormat.class, clazz)); } }
/** * Java wrapper for {@link DBInputFormat#setInput(org.apache.hadoop.mapreduce.Job, Class, String, String)} and * {@link DBInputFormat#setInput(org.apache.hadoop.mapreduce.Job, Class, String, String, String, String...)}. * * @param ctx the JavaScript context * @param thisObj the 'this' object * @param args the function arguments * @param func the function being called */ @JSStaticFunction public static void setInput(final Context ctx, final Scriptable thisObj, final Object[] args, final Function func) { DBInputFormatHelper.setInput(DBInputFormat.class, ctx, thisObj, args, func); }
