Java 类org.apache.hadoop.mapreduce.MapReduceTestUtil.DataCopyMapper 实例源码

项目:hadoop    文件:TestTaskContext.java   
/**
 * Tests context.setStatus method.
 * TODO fix testcase
 * @throws IOException
 * @throws InterruptedException
 * @throws ClassNotFoundException
 */
@Test
@Ignore
public void testContextStatus()
    throws IOException, InterruptedException, ClassNotFoundException {
  Path test = new Path(testRootTempDir, "testContextStatus");

  // test with 1 map and 0 reducers
  // test with custom task status
  int numMaps = 1;
  Job job = MapReduceTestUtil.createJob(createJobConf(), 
              new Path(test, "in"), new Path(test, "out"), numMaps, 0);
  job.setMapperClass(MyMapper.class);
  job.waitForCompletion(true);
  assertTrue("Job failed", job.isSuccessful());
  TaskReport[] reports = job.getTaskReports(TaskType.MAP);
  assertEquals(numMaps, reports.length);
  assertEquals(myStatus, reports[0].getState());

  // test with 1 map and 1 reducer
  // test with default task status
  int numReduces = 1;
  job = MapReduceTestUtil.createJob(createJobConf(), 
          new Path(test, "in"), new Path(test, "out"), numMaps, numReduces);
  job.setMapperClass(DataCopyMapper.class);
  job.setReducerClass(DataCopyReducer.class);
  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(Text.class);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(Text.class);

  // fail early
  job.setMaxMapAttempts(1);
  job.setMaxReduceAttempts(0);

  // run the job and wait for completion
  job.waitForCompletion(true);
  assertTrue("Job failed", job.isSuccessful());

  // check map task reports
  // TODO fix testcase 
  // Disabling checks for now to get builds to run
  /*
  reports = job.getTaskReports(TaskType.MAP);
  assertEquals(numMaps, reports.length);
  assertEquals("map > sort", reports[0].getState());

  // check reduce task reports
  reports = job.getTaskReports(TaskType.REDUCE);
  assertEquals(numReduces, reports.length);
  assertEquals("reduce > reduce", reports[0].getState());
  */
}
项目:aliyun-oss-hadoop-fs    文件:TestTaskContext.java   
/**
 * Tests context.setStatus method.
 * TODO fix testcase
 * @throws IOException
 * @throws InterruptedException
 * @throws ClassNotFoundException
 */
@Test
@Ignore
public void testContextStatus()
    throws IOException, InterruptedException, ClassNotFoundException {
  Path test = new Path(testRootTempDir, "testContextStatus");

  // test with 1 map and 0 reducers
  // test with custom task status
  int numMaps = 1;
  Job job = MapReduceTestUtil.createJob(createJobConf(), 
              new Path(test, "in"), new Path(test, "out"), numMaps, 0);
  job.setMapperClass(MyMapper.class);
  job.waitForCompletion(true);
  assertTrue("Job failed", job.isSuccessful());
  TaskReport[] reports = job.getTaskReports(TaskType.MAP);
  assertEquals(numMaps, reports.length);
  assertEquals(myStatus, reports[0].getState());

  // test with 1 map and 1 reducer
  // test with default task status
  int numReduces = 1;
  job = MapReduceTestUtil.createJob(createJobConf(), 
          new Path(test, "in"), new Path(test, "out"), numMaps, numReduces);
  job.setMapperClass(DataCopyMapper.class);
  job.setReducerClass(DataCopyReducer.class);
  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(Text.class);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(Text.class);

  // fail early
  job.setMaxMapAttempts(1);
  job.setMaxReduceAttempts(0);

  // run the job and wait for completion
  job.waitForCompletion(true);
  assertTrue("Job failed", job.isSuccessful());

  // check map task reports
  // TODO fix testcase 
  // Disabling checks for now to get builds to run
  /*
  reports = job.getTaskReports(TaskType.MAP);
  assertEquals(numMaps, reports.length);
  assertEquals("map > sort", reports[0].getState());

  // check reduce task reports
  reports = job.getTaskReports(TaskType.REDUCE);
  assertEquals(numReduces, reports.length);
  assertEquals("reduce > reduce", reports[0].getState());
  */
}
项目:big-c    文件:TestTaskContext.java   
/**
 * Tests context.setStatus method.
 * TODO fix testcase
 * @throws IOException
 * @throws InterruptedException
 * @throws ClassNotFoundException
 */
@Test
@Ignore
public void testContextStatus()
    throws IOException, InterruptedException, ClassNotFoundException {
  Path test = new Path(testRootTempDir, "testContextStatus");

  // test with 1 map and 0 reducers
  // test with custom task status
  int numMaps = 1;
  Job job = MapReduceTestUtil.createJob(createJobConf(), 
              new Path(test, "in"), new Path(test, "out"), numMaps, 0);
  job.setMapperClass(MyMapper.class);
  job.waitForCompletion(true);
  assertTrue("Job failed", job.isSuccessful());
  TaskReport[] reports = job.getTaskReports(TaskType.MAP);
  assertEquals(numMaps, reports.length);
  assertEquals(myStatus, reports[0].getState());

  // test with 1 map and 1 reducer
  // test with default task status
  int numReduces = 1;
  job = MapReduceTestUtil.createJob(createJobConf(), 
          new Path(test, "in"), new Path(test, "out"), numMaps, numReduces);
  job.setMapperClass(DataCopyMapper.class);
  job.setReducerClass(DataCopyReducer.class);
  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(Text.class);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(Text.class);

  // fail early
  job.setMaxMapAttempts(1);
  job.setMaxReduceAttempts(0);

  // run the job and wait for completion
  job.waitForCompletion(true);
  assertTrue("Job failed", job.isSuccessful());

  // check map task reports
  // TODO fix testcase 
  // Disabling checks for now to get builds to run
  /*
  reports = job.getTaskReports(TaskType.MAP);
  assertEquals(numMaps, reports.length);
  assertEquals("map > sort", reports[0].getState());

  // check reduce task reports
  reports = job.getTaskReports(TaskType.REDUCE);
  assertEquals(numReduces, reports.length);
  assertEquals("reduce > reduce", reports[0].getState());
  */
}
项目:hadoop-2.6.0-cdh5.4.3    文件:TestTaskContext.java   
/**
 * Tests context.setStatus method.
 * TODO fix testcase
 * @throws IOException
 * @throws InterruptedException
 * @throws ClassNotFoundException
 */
@Test
@Ignore
public void testContextStatus()
    throws IOException, InterruptedException, ClassNotFoundException {
  Path test = new Path(testRootTempDir, "testContextStatus");

  // test with 1 map and 0 reducers
  // test with custom task status
  int numMaps = 1;
  Job job = MapReduceTestUtil.createJob(createJobConf(), 
              new Path(test, "in"), new Path(test, "out"), numMaps, 0);
  job.setMapperClass(MyMapper.class);
  job.waitForCompletion(true);
  assertTrue("Job failed", job.isSuccessful());
  TaskReport[] reports = job.getTaskReports(TaskType.MAP);
  assertEquals(numMaps, reports.length);
  assertEquals(myStatus, reports[0].getState());

  // test with 1 map and 1 reducer
  // test with default task status
  int numReduces = 1;
  job = MapReduceTestUtil.createJob(createJobConf(), 
          new Path(test, "in"), new Path(test, "out"), numMaps, numReduces);
  job.setMapperClass(DataCopyMapper.class);
  job.setReducerClass(DataCopyReducer.class);
  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(Text.class);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(Text.class);

  // fail early
  job.setMaxMapAttempts(1);
  job.setMaxReduceAttempts(0);

  // run the job and wait for completion
  job.waitForCompletion(true);
  assertTrue("Job failed", job.isSuccessful());

  // check map task reports
  // TODO fix testcase 
  // Disabling checks for now to get builds to run
  /*
  reports = job.getTaskReports(TaskType.MAP);
  assertEquals(numMaps, reports.length);
  assertEquals("map > sort", reports[0].getState());

  // check reduce task reports
  reports = job.getTaskReports(TaskType.REDUCE);
  assertEquals(numReduces, reports.length);
  assertEquals("reduce > reduce", reports[0].getState());
  */
}
项目:hadoop-plus    文件:TestTaskContext.java   
/**
 * Tests context.setStatus method.
 * TODO fix testcase
 * @throws IOException
 * @throws InterruptedException
 * @throws ClassNotFoundException
 */
@Test
@Ignore
public void testContextStatus()
    throws IOException, InterruptedException, ClassNotFoundException {
  Path test = new Path(testRootTempDir, "testContextStatus");

  // test with 1 map and 0 reducers
  // test with custom task status
  int numMaps = 1;
  Job job = MapReduceTestUtil.createJob(createJobConf(), 
              new Path(test, "in"), new Path(test, "out"), numMaps, 0);
  job.setMapperClass(MyMapper.class);
  job.waitForCompletion(true);
  assertTrue("Job failed", job.isSuccessful());
  TaskReport[] reports = job.getTaskReports(TaskType.MAP);
  assertEquals(numMaps, reports.length);
  assertEquals(myStatus, reports[0].getState());

  // test with 1 map and 1 reducer
  // test with default task status
  int numReduces = 1;
  job = MapReduceTestUtil.createJob(createJobConf(), 
          new Path(test, "in"), new Path(test, "out"), numMaps, numReduces);
  job.setMapperClass(DataCopyMapper.class);
  job.setReducerClass(DataCopyReducer.class);
  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(Text.class);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(Text.class);

  // fail early
  job.setMaxMapAttempts(1);
  job.setMaxReduceAttempts(0);

  // run the job and wait for completion
  job.waitForCompletion(true);
  assertTrue("Job failed", job.isSuccessful());

  // check map task reports
  // TODO fix testcase 
  // Disabling checks for now to get builds to run
  /*
  reports = job.getTaskReports(TaskType.MAP);
  assertEquals(numMaps, reports.length);
  assertEquals("map > sort", reports[0].getState());

  // check reduce task reports
  reports = job.getTaskReports(TaskType.REDUCE);
  assertEquals(numReduces, reports.length);
  assertEquals("reduce > reduce", reports[0].getState());
  */
}
项目:FlexMap    文件:TestTaskContext.java   
/**
 * Tests context.setStatus method.
 * TODO fix testcase
 * @throws IOException
 * @throws InterruptedException
 * @throws ClassNotFoundException
 */
@Test
@Ignore
public void testContextStatus()
    throws IOException, InterruptedException, ClassNotFoundException {
  Path test = new Path(testRootTempDir, "testContextStatus");

  // test with 1 map and 0 reducers
  // test with custom task status
  int numMaps = 1;
  Job job = MapReduceTestUtil.createJob(createJobConf(), 
              new Path(test, "in"), new Path(test, "out"), numMaps, 0);
  job.setMapperClass(MyMapper.class);
  job.waitForCompletion(true);
  assertTrue("Job failed", job.isSuccessful());
  TaskReport[] reports = job.getTaskReports(TaskType.MAP);
  assertEquals(numMaps, reports.length);
  assertEquals(myStatus, reports[0].getState());

  // test with 1 map and 1 reducer
  // test with default task status
  int numReduces = 1;
  job = MapReduceTestUtil.createJob(createJobConf(), 
          new Path(test, "in"), new Path(test, "out"), numMaps, numReduces);
  job.setMapperClass(DataCopyMapper.class);
  job.setReducerClass(DataCopyReducer.class);
  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(Text.class);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(Text.class);

  // fail early
  job.setMaxMapAttempts(1);
  job.setMaxReduceAttempts(0);

  // run the job and wait for completion
  job.waitForCompletion(true);
  assertTrue("Job failed", job.isSuccessful());

  // check map task reports
  // TODO fix testcase 
  // Disabling checks for now to get builds to run
  /*
  reports = job.getTaskReports(TaskType.MAP);
  assertEquals(numMaps, reports.length);
  assertEquals("map > sort", reports[0].getState());

  // check reduce task reports
  reports = job.getTaskReports(TaskType.REDUCE);
  assertEquals(numReduces, reports.length);
  assertEquals("reduce > reduce", reports[0].getState());
  */
}
项目:hops    文件:TestTaskContext.java   
/**
 * Tests context.setStatus method.
 * TODO fix testcase
 * @throws IOException
 * @throws InterruptedException
 * @throws ClassNotFoundException
 */
@Test
@Ignore
public void testContextStatus()
    throws IOException, InterruptedException, ClassNotFoundException {
  Path test = new Path(testRootTempDir, "testContextStatus");

  // test with 1 map and 0 reducers
  // test with custom task status
  int numMaps = 1;
  Job job = MapReduceTestUtil.createJob(createJobConf(), 
              new Path(test, "in"), new Path(test, "out"), numMaps, 0);
  job.setMapperClass(MyMapper.class);
  job.waitForCompletion(true);
  assertTrue("Job failed", job.isSuccessful());
  TaskReport[] reports = job.getTaskReports(TaskType.MAP);
  assertEquals(numMaps, reports.length);
  assertEquals(myStatus, reports[0].getState());

  // test with 1 map and 1 reducer
  // test with default task status
  int numReduces = 1;
  job = MapReduceTestUtil.createJob(createJobConf(), 
          new Path(test, "in"), new Path(test, "out"), numMaps, numReduces);
  job.setMapperClass(DataCopyMapper.class);
  job.setReducerClass(DataCopyReducer.class);
  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(Text.class);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(Text.class);

  // fail early
  job.setMaxMapAttempts(1);
  job.setMaxReduceAttempts(0);

  // run the job and wait for completion
  job.waitForCompletion(true);
  assertTrue("Job failed", job.isSuccessful());

  // check map task reports
  // TODO fix testcase 
  // Disabling checks for now to get builds to run
  /*
  reports = job.getTaskReports(TaskType.MAP);
  assertEquals(numMaps, reports.length);
  assertEquals("map > sort", reports[0].getState());

  // check reduce task reports
  reports = job.getTaskReports(TaskType.REDUCE);
  assertEquals(numReduces, reports.length);
  assertEquals("reduce > reduce", reports[0].getState());
  */
}
项目:hadoop-TCP    文件:TestTaskContext.java   
/**
 * Tests context.setStatus method.
 * TODO fix testcase
 * @throws IOException
 * @throws InterruptedException
 * @throws ClassNotFoundException
 */
@Test
@Ignore
public void testContextStatus()
    throws IOException, InterruptedException, ClassNotFoundException {
  Path test = new Path(testRootTempDir, "testContextStatus");

  // test with 1 map and 0 reducers
  // test with custom task status
  int numMaps = 1;
  Job job = MapReduceTestUtil.createJob(createJobConf(), 
              new Path(test, "in"), new Path(test, "out"), numMaps, 0);
  job.setMapperClass(MyMapper.class);
  job.waitForCompletion(true);
  assertTrue("Job failed", job.isSuccessful());
  TaskReport[] reports = job.getTaskReports(TaskType.MAP);
  assertEquals(numMaps, reports.length);
  assertEquals(myStatus, reports[0].getState());

  // test with 1 map and 1 reducer
  // test with default task status
  int numReduces = 1;
  job = MapReduceTestUtil.createJob(createJobConf(), 
          new Path(test, "in"), new Path(test, "out"), numMaps, numReduces);
  job.setMapperClass(DataCopyMapper.class);
  job.setReducerClass(DataCopyReducer.class);
  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(Text.class);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(Text.class);

  // fail early
  job.setMaxMapAttempts(1);
  job.setMaxReduceAttempts(0);

  // run the job and wait for completion
  job.waitForCompletion(true);
  assertTrue("Job failed", job.isSuccessful());

  // check map task reports
  // TODO fix testcase 
  // Disabling checks for now to get builds to run
  /*
  reports = job.getTaskReports(TaskType.MAP);
  assertEquals(numMaps, reports.length);
  assertEquals("map > sort", reports[0].getState());

  // check reduce task reports
  reports = job.getTaskReports(TaskType.REDUCE);
  assertEquals(numReduces, reports.length);
  assertEquals("reduce > reduce", reports[0].getState());
  */
}
项目:hardfs    文件:TestTaskContext.java   
/**
 * Tests context.setStatus method.
 * TODO fix testcase
 * @throws IOException
 * @throws InterruptedException
 * @throws ClassNotFoundException
 */
@Test
@Ignore
public void testContextStatus()
    throws IOException, InterruptedException, ClassNotFoundException {
  Path test = new Path(testRootTempDir, "testContextStatus");

  // test with 1 map and 0 reducers
  // test with custom task status
  int numMaps = 1;
  Job job = MapReduceTestUtil.createJob(createJobConf(), 
              new Path(test, "in"), new Path(test, "out"), numMaps, 0);
  job.setMapperClass(MyMapper.class);
  job.waitForCompletion(true);
  assertTrue("Job failed", job.isSuccessful());
  TaskReport[] reports = job.getTaskReports(TaskType.MAP);
  assertEquals(numMaps, reports.length);
  assertEquals(myStatus, reports[0].getState());

  // test with 1 map and 1 reducer
  // test with default task status
  int numReduces = 1;
  job = MapReduceTestUtil.createJob(createJobConf(), 
          new Path(test, "in"), new Path(test, "out"), numMaps, numReduces);
  job.setMapperClass(DataCopyMapper.class);
  job.setReducerClass(DataCopyReducer.class);
  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(Text.class);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(Text.class);

  // fail early
  job.setMaxMapAttempts(1);
  job.setMaxReduceAttempts(0);

  // run the job and wait for completion
  job.waitForCompletion(true);
  assertTrue("Job failed", job.isSuccessful());

  // check map task reports
  // TODO fix testcase 
  // Disabling checks for now to get builds to run
  /*
  reports = job.getTaskReports(TaskType.MAP);
  assertEquals(numMaps, reports.length);
  assertEquals("map > sort", reports[0].getState());

  // check reduce task reports
  reports = job.getTaskReports(TaskType.REDUCE);
  assertEquals(numReduces, reports.length);
  assertEquals("reduce > reduce", reports[0].getState());
  */
}
项目:hadoop-on-lustre2    文件:TestTaskContext.java   
/**
 * Tests context.setStatus method.
 * TODO fix testcase
 * @throws IOException
 * @throws InterruptedException
 * @throws ClassNotFoundException
 */
@Test
@Ignore
public void testContextStatus()
    throws IOException, InterruptedException, ClassNotFoundException {
  Path test = new Path(testRootTempDir, "testContextStatus");

  // test with 1 map and 0 reducers
  // test with custom task status
  int numMaps = 1;
  Job job = MapReduceTestUtil.createJob(createJobConf(), 
              new Path(test, "in"), new Path(test, "out"), numMaps, 0);
  job.setMapperClass(MyMapper.class);
  job.waitForCompletion(true);
  assertTrue("Job failed", job.isSuccessful());
  TaskReport[] reports = job.getTaskReports(TaskType.MAP);
  assertEquals(numMaps, reports.length);
  assertEquals(myStatus, reports[0].getState());

  // test with 1 map and 1 reducer
  // test with default task status
  int numReduces = 1;
  job = MapReduceTestUtil.createJob(createJobConf(), 
          new Path(test, "in"), new Path(test, "out"), numMaps, numReduces);
  job.setMapperClass(DataCopyMapper.class);
  job.setReducerClass(DataCopyReducer.class);
  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(Text.class);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(Text.class);

  // fail early
  job.setMaxMapAttempts(1);
  job.setMaxReduceAttempts(0);

  // run the job and wait for completion
  job.waitForCompletion(true);
  assertTrue("Job failed", job.isSuccessful());

  // check map task reports
  // TODO fix testcase 
  // Disabling checks for now to get builds to run
  /*
  reports = job.getTaskReports(TaskType.MAP);
  assertEquals(numMaps, reports.length);
  assertEquals("map > sort", reports[0].getState());

  // check reduce task reports
  reports = job.getTaskReports(TaskType.REDUCE);
  assertEquals(numReduces, reports.length);
  assertEquals("reduce > reduce", reports[0].getState());
  */
}
项目:mapreduce-fork    文件:TestTaskContext.java   
/**
 * Tests context.setStatus method.
 * 
 * @throws IOException
 * @throws InterruptedException
 * @throws ClassNotFoundException
 */
@Test
public void testContextStatus()
    throws IOException, InterruptedException, ClassNotFoundException {
  Path test = new Path(testRootTempDir, "testContextStatus");

  // test with 1 map and 0 reducers
  // test with custom task status
  int numMaps = 1;
  Job job = MapReduceTestUtil.createJob(createJobConf(), 
              new Path(test, "in"), new Path(test, "out"), numMaps, 0);
  job.setMapperClass(MyMapper.class);
  job.waitForCompletion(true);
  assertTrue("Job failed", job.isSuccessful());
  TaskReport[] reports = job.getTaskReports(TaskType.MAP);
  assertEquals(numMaps, reports.length);
  assertEquals(myStatus, reports[0].getState());

  // test with 1 map and 1 reducer
  // test with default task status
  int numReduces = 1;
  job = MapReduceTestUtil.createJob(createJobConf(), 
          new Path(test, "in"), new Path(test, "out"), numMaps, numReduces);
  job.setMapperClass(DataCopyMapper.class);
  job.setReducerClass(DataCopyReducer.class);
  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(Text.class);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(Text.class);

  // fail early
  job.setMaxMapAttempts(1);
  job.setMaxReduceAttempts(0);

  // run the job and wait for completion
  job.waitForCompletion(true);
  assertTrue("Job failed", job.isSuccessful());

  // check map task reports
  reports = job.getTaskReports(TaskType.MAP);
  assertEquals(numMaps, reports.length);
  assertEquals("map > sort", reports[0].getState());

  // check reduce task reports
  reports = job.getTaskReports(TaskType.REDUCE);
  assertEquals(numReduces, reports.length);
  assertEquals("reduce > reduce", reports[0].getState());
}