1. 包列表
  2. 类列表
  3. org.apache.hadoop.mapreduce.v2.app.job.Task

Java 类org.apache.hadoop.mapreduce.v2.app.job.Task 实例源码

项目：hadoop    文件：TestHsWebServicesTasks.java    
@Test
public void testTaskId() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);
    for (Task task : jobsMap.get(id).getTasks().values()) {

      String tid = MRApps.toString(task.getID());
      ClientResponse response = r.path("ws").path("v1").path("history")
          .path("mapreduce").path("jobs").path(jobId).path("tasks").path(tid)
          .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
      assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
      JSONObject json = response.getEntity(JSONObject.class);
      assertEquals("incorrect number of elements", 1, json.length());
      JSONObject info = json.getJSONObject("task");
      verifyHsSingleTask(info, task);
    }
  }
}
项目：hadoop    文件：TestAMWebServicesAttempts.java    
@Test
public void testTaskAttemptsSlash() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);
    for (Task task : jobsMap.get(id).getTasks().values()) {

      String tid = MRApps.toString(task.getID());
      ClientResponse response = r.path("ws").path("v1").path("mapreduce")
          .path("jobs").path(jobId).path("tasks").path(tid).path("attempts/")
          .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
      assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
      JSONObject json = response.getEntity(JSONObject.class);
      verifyAMTaskAttempts(json, task);
    }
  }
}
项目：hadoop    文件：TaskAttemptListenerImpl.java    
/**
 * Child checking whether it can commit.
 * 
 * <br>
 * Commit is a two-phased protocol. First the attempt informs the
 * ApplicationMaster that it is
 * {@link #commitPending(TaskAttemptID, TaskStatus)}. Then it repeatedly polls
 * the ApplicationMaster whether it {@link #canCommit(TaskAttemptID)} This is
 * a legacy from the centralized commit protocol handling by the JobTracker.
 */
@Override
public boolean canCommit(TaskAttemptID taskAttemptID) throws IOException {
  LOG.info("Commit go/no-go request from " + taskAttemptID.toString());
  // An attempt is asking if it can commit its output. This can be decided
  // only by the task which is managing the multiple attempts. So redirect the
  // request there.
  org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID =
      TypeConverter.toYarn(taskAttemptID);

  taskHeartbeatHandler.progressing(attemptID);

  // tell task to retry later if AM has not heard from RM within the commit
  // window to help avoid double-committing in a split-brain situation
  long now = context.getClock().getTime();
  if (now - rmHeartbeatHandler.getLastHeartbeatTime() > commitWindowMs) {
    return false;
  }

  Job job = context.getJob(attemptID.getTaskId().getJobId());
  Task task = job.getTask(attemptID.getTaskId());
  return task.canCommit(attemptID);
}
项目：hadoop    文件：StartEndTimesBase.java    
protected DataStatistics dataStatisticsForTask(TaskId taskID) {
  JobId jobID = taskID.getJobId();
  Job job = context.getJob(jobID);

  if (job == null) {
    return null;
  }

  Task task = job.getTask(taskID);

  if (task == null) {
    return null;
  }

  return task.getType() == TaskType.MAP
          ? mapperStatistics.get(job)
          : task.getType() == TaskType.REDUCE
              ? reducerStatistics.get(job)
              : null;
}
项目：hadoop    文件：TaskInfo.java    
public TaskInfo(Task task) {
  TaskType ttype = task.getType();
  this.type = ttype.toString();
  TaskReport report = task.getReport();
  this.startTime = report.getStartTime();
  this.finishTime = report.getFinishTime();
  this.state = report.getTaskState();
  this.elapsedTime = Times.elapsed(this.startTime, this.finishTime,
    this.state == TaskState.RUNNING);
  if (this.elapsedTime == -1) {
    this.elapsedTime = 0;
  }
  this.progress = report.getProgress() * 100;
  this.status =  report.getStatus();
  this.id = MRApps.toString(task.getID());
  this.taskNum = task.getID().getId();
  this.successful = getSuccessfulAttempt(task);
  if (successful != null) {
    this.successfulAttempt = MRApps.toString(successful.getID());
  } else {
    this.successfulAttempt = "";
  }
}
项目：hadoop    文件：TestFail.java    
@Test
//All Task attempts are timed out, leading to Job failure
public void testTimedOutTask() throws Exception {
  MRApp app = new TimeOutTaskMRApp(1, 0);
  Configuration conf = new Configuration();
  int maxAttempts = 2;
  conf.setInt(MRJobConfig.MAP_MAX_ATTEMPTS, maxAttempts);
  // disable uberization (requires entire job to be reattempted, so max for
  // subtask attempts is overridden to 1)
  conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);
  Job job = app.submit(conf);
  app.waitForState(job, JobState.FAILED);
  Map<TaskId,Task> tasks = job.getTasks();
  Assert.assertEquals("Num tasks is not correct", 1, tasks.size());
  Task task = tasks.values().iterator().next();
  Assert.assertEquals("Task state not correct", TaskState.FAILED,
      task.getReport().getTaskState());
  Map<TaskAttemptId, TaskAttempt> attempts =
      tasks.values().iterator().next().getAttempts();
  Assert.assertEquals("Num attempts is not correct", maxAttempts,
      attempts.size());
  for (TaskAttempt attempt : attempts.values()) {
    Assert.assertEquals("Attempt state not correct", TaskAttemptState.FAILED,
        attempt.getReport().getTaskAttemptState());
  }
}
项目：hadoop    文件：TestHsWebServicesTasks.java    
@Test
public void testTaskIdCountersSlash() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);
    for (Task task : jobsMap.get(id).getTasks().values()) {

      String tid = MRApps.toString(task.getID());
      ClientResponse response = r.path("ws").path("v1").path("history")
          .path("mapreduce").path("jobs").path(jobId).path("tasks").path(tid)
          .path("counters/").accept(MediaType.APPLICATION_JSON)
          .get(ClientResponse.class);
      assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
      JSONObject json = response.getEntity(JSONObject.class);
      assertEquals("incorrect number of elements", 1, json.length());
      JSONObject info = json.getJSONObject("jobTaskCounters");
      verifyHsJobTaskCounters(info, task);
    }
  }
}
项目：hadoop    文件：AMWebServices.java    
@GET
@Path("/jobs/{jobid}/tasks/{taskid}/attempts")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public TaskAttemptsInfo getJobTaskAttempts(@Context HttpServletRequest hsr,
    @PathParam("jobid") String jid, @PathParam("taskid") String tid) {

  init();
  TaskAttemptsInfo attempts = new TaskAttemptsInfo();
  Job job = getJobFromJobIdString(jid, appCtx);
  checkAccess(job, hsr);
  Task task = getTaskFromTaskIdString(tid, job);

  for (TaskAttempt ta : task.getAttempts().values()) {
    if (ta != null) {
      if (task.getType() == TaskType.REDUCE) {
        attempts.add(new ReduceTaskAttemptInfo(ta, task.getType()));
      } else {
        attempts.add(new TaskAttemptInfo(ta, task.getType(), true));
      }
    }
  }
  return attempts;
}
项目：hadoop    文件：TestHsWebServicesTasks.java    
public void verifyHsJobTaskCounters(JSONObject info, Task task)
    throws JSONException {

  assertEquals("incorrect number of elements", 2, info.length());

  WebServicesTestUtils.checkStringMatch("id", MRApps.toString(task.getID()),
      info.getString("id"));
  // just do simple verification of fields - not data is correct
  // in the fields
  JSONArray counterGroups = info.getJSONArray("taskCounterGroup");
  for (int i = 0; i < counterGroups.length(); i++) {
    JSONObject counterGroup = counterGroups.getJSONObject(i);
    String name = counterGroup.getString("counterGroupName");
    assertTrue("name not set", (name != null && !name.isEmpty()));
    JSONArray counters = counterGroup.getJSONArray("counter");
    for (int j = 0; j < counters.length(); j++) {
      JSONObject counter = counters.getJSONObject(j);
      String counterName = counter.getString("name");
      assertTrue("name not set",
          (counterName != null && !counterName.isEmpty()));
      long value = counter.getLong("value");
      assertTrue("value  >= 0", value >= 0);
    }
  }
}
项目：hadoop    文件：HsWebServices.java    
@GET
@Path("/mapreduce/jobs/{jobid}/tasks/{taskid}/counters")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public JobTaskCounterInfo getSingleTaskCounters(
    @Context HttpServletRequest hsr, @PathParam("jobid") String jid,
    @PathParam("taskid") String tid) {

  init();
  Job job = AMWebServices.getJobFromJobIdString(jid, ctx);
  checkAccess(job, hsr);
  TaskId taskID = MRApps.toTaskID(tid);
  if (taskID == null) {
    throw new NotFoundException("taskid " + tid + " not found or invalid");
  }
  Task task = job.getTask(taskID);
  if (task == null) {
    throw new NotFoundException("task not found with id " + tid);
  }
  return new JobTaskCounterInfo(task);
}
项目：hadoop    文件：AttemptsPage.java    
@Override
protected Collection<TaskAttempt> getTaskAttempts() {
  List<TaskAttempt> fewTaskAttemps = new ArrayList<TaskAttempt>();
  String taskTypeStr = $(TASK_TYPE);
  TaskType taskType = MRApps.taskType(taskTypeStr);
  String attemptStateStr = $(ATTEMPT_STATE);
  TaskAttemptStateUI neededState = MRApps
      .taskAttemptState(attemptStateStr);
  for (Task task : super.app.getJob().getTasks(taskType).values()) {
    Map<TaskAttemptId, TaskAttempt> attempts = task.getAttempts();
    for (TaskAttempt attempt : attempts.values()) {
      if (neededState.correspondsTo(attempt.getState())) {
        fewTaskAttemps.add(attempt);
      }
    }
  }
  return fewTaskAttemps;
}
项目：hadoop    文件：JobImpl.java    
private void computeProgress() {
  this.readLock.lock();
  try {
    float mapProgress = 0f;
    float reduceProgress = 0f;
    for (Task task : this.tasks.values()) {
      if (task.getType() == TaskType.MAP) {
        mapProgress += (task.isFinished() ? 1f : task.getProgress());
      } else {
        reduceProgress += (task.isFinished() ? 1f : task.getProgress());
      }
    }
    if (this.numMapTasks != 0) {
      mapProgress = mapProgress / this.numMapTasks;
    }
    if (this.numReduceTasks != 0) {
      reduceProgress = reduceProgress / this.numReduceTasks;
    }
    this.mapProgress = mapProgress;
    this.reduceProgress = reduceProgress;
  } finally {
    this.readLock.unlock();
  }
}
项目：hadoop    文件：TestHsWebServicesTasks.java    
@Test
public void testTaskIdDefault() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);
    for (Task task : jobsMap.get(id).getTasks().values()) {

      String tid = MRApps.toString(task.getID());
      ClientResponse response = r.path("ws").path("v1").path("history")
          .path("mapreduce").path("jobs").path(jobId).path("tasks").path(tid)
          .get(ClientResponse.class);
      assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
      JSONObject json = response.getEntity(JSONObject.class);
      assertEquals("incorrect number of elements", 1, json.length());
      JSONObject info = json.getJSONObject("task");
      verifyHsSingleTask(info, task);
    }
  }
}
项目：hadoop    文件：TestHsWebServicesTasks.java    
public void verifyTaskGeneric(Task task, String id, String state,
    String type, String successfulAttempt, long startTime, long finishTime,
    long elapsedTime, float progress) {

  TaskId taskid = task.getID();
  String tid = MRApps.toString(taskid);
  TaskReport report = task.getReport();

  WebServicesTestUtils.checkStringMatch("id", tid, id);
  WebServicesTestUtils.checkStringMatch("type", task.getType().toString(),
      type);
  WebServicesTestUtils.checkStringMatch("state", report.getTaskState()
      .toString(), state);
  // not easily checked without duplicating logic, just make sure its here
  assertNotNull("successfulAttempt null", successfulAttempt);
  assertEquals("startTime wrong", report.getStartTime(), startTime);
  assertEquals("finishTime wrong", report.getFinishTime(), finishTime);
  assertEquals("elapsedTime wrong", finishTime - startTime, elapsedTime);
  assertEquals("progress wrong", report.getProgress() * 100, progress, 1e-3f);
}
项目：hadoop    文件：TestHsWebServicesTasks.java    
public void verifyHsTask(JSONArray arr, Job job, String type)
    throws JSONException {
  for (Task task : job.getTasks().values()) {
    TaskId id = task.getID();
    String tid = MRApps.toString(id);
    Boolean found = false;
    if (type != null && task.getType() == MRApps.taskType(type)) {

      for (int i = 0; i < arr.length(); i++) {
        JSONObject info = arr.getJSONObject(i);
        if (tid.matches(info.getString("id"))) {
          found = true;
          verifyHsSingleTask(info, task);
        }
      }
      assertTrue("task with id: " + tid + " not in web service output", found);
    }
  }
}
项目：hadoop    文件：TestHsWebServicesTasks.java    
@Test
public void testTaskIdCounters() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);
    for (Task task : jobsMap.get(id).getTasks().values()) {

      String tid = MRApps.toString(task.getID());
      ClientResponse response = r.path("ws").path("v1").path("history")
          .path("mapreduce").path("jobs").path(jobId).path("tasks").path(tid)
          .path("counters").accept(MediaType.APPLICATION_JSON)
          .get(ClientResponse.class);
      assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
      JSONObject json = response.getEntity(JSONObject.class);
      assertEquals("incorrect number of elements", 1, json.length());
      JSONObject info = json.getJSONObject("jobTaskCounters");
      verifyHsJobTaskCounters(info, task);
    }
  }
}
项目：hadoop    文件：TestHsWebServicesTasks.java    
@Test
public void testTaskIdCountersDefault() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);
    for (Task task : jobsMap.get(id).getTasks().values()) {

      String tid = MRApps.toString(task.getID());
      ClientResponse response = r.path("ws").path("v1").path("history")
          .path("mapreduce").path("jobs").path(jobId).path("tasks").path(tid)
          .path("counters").get(ClientResponse.class);
      assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
      JSONObject json = response.getEntity(JSONObject.class);
      assertEquals("incorrect number of elements", 1, json.length());
      JSONObject info = json.getJSONObject("jobTaskCounters");
      verifyHsJobTaskCounters(info, task);
    }
  }
}
项目：hadoop    文件：MRClientService.java    
@Override
public GetTaskReportsResponse getTaskReports(
    GetTaskReportsRequest request) throws IOException {
  JobId jobId = request.getJobId();
  TaskType taskType = request.getTaskType();

  GetTaskReportsResponse response = 
    recordFactory.newRecordInstance(GetTaskReportsResponse.class);

  Job job = verifyAndGetJob(jobId, JobACL.VIEW_JOB, true);
  Collection<Task> tasks = job.getTasks(taskType).values();
  LOG.info("Getting task report for " + taskType + "   " + jobId
      + ". Report-size will be " + tasks.size());

  // Take lock to allow only one call, otherwise heap will blow up because
  // of counters in the report when there are multiple callers.
  synchronized (getTaskReportsLock) {
    for (Task task : tasks) {
      response.addTaskReport(task.getReport());
    }
  }

  return response;
}
项目：hadoop    文件：HsWebServices.java    
@GET
@Path("/mapreduce/jobs/{jobid}/tasks/{taskid}/attempts")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public TaskAttemptsInfo getJobTaskAttempts(@Context HttpServletRequest hsr,
    @PathParam("jobid") String jid, @PathParam("taskid") String tid) {

  init();
  TaskAttemptsInfo attempts = new TaskAttemptsInfo();
  Job job = AMWebServices.getJobFromJobIdString(jid, ctx);
  checkAccess(job, hsr);
  Task task = AMWebServices.getTaskFromTaskIdString(tid, job);
  for (TaskAttempt ta : task.getAttempts().values()) {
    if (ta != null) {
      if (task.getType() == TaskType.REDUCE) {
        attempts.add(new ReduceTaskAttemptInfo(ta, task.getType()));
      } else {
        attempts.add(new TaskAttemptInfo(ta, task.getType(), false));
      }
    }
  }
  return attempts;
}
项目：hadoop    文件：TestHsWebServicesAttempts.java    
@Test
public void testTaskAttempts() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);
    for (Task task : jobsMap.get(id).getTasks().values()) {

      String tid = MRApps.toString(task.getID());
      ClientResponse response = r.path("ws").path("v1").path("history")
          .path("mapreduce").path("jobs").path(jobId).path("tasks").path(tid)
          .path("attempts").accept(MediaType.APPLICATION_JSON)
          .get(ClientResponse.class);
      assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
      JSONObject json = response.getEntity(JSONObject.class);
      verifyHsTaskAttempts(json, task);
    }
  }
}
项目：hadoop    文件：TestTaskAttempt.java    
private void testMRAppHistory(MRApp app) throws Exception {
  Configuration conf = new Configuration();
  Job job = app.submit(conf);
  app.waitForState(job, JobState.FAILED);
  Map<TaskId, Task> tasks = job.getTasks();

  Assert.assertEquals("Num tasks is not correct", 1, tasks.size());
  Task task = tasks.values().iterator().next();
  Assert.assertEquals("Task state not correct", TaskState.FAILED, task
      .getReport().getTaskState());
  Map<TaskAttemptId, TaskAttempt> attempts = tasks.values().iterator().next()
      .getAttempts();
  Assert.assertEquals("Num attempts is not correct", 4, attempts.size());

  Iterator<TaskAttempt> it = attempts.values().iterator();
  TaskAttemptReport report = it.next().getReport();
  Assert.assertEquals("Attempt state not correct", TaskAttemptState.FAILED,
      report.getTaskAttemptState());
  Assert.assertEquals("Diagnostic Information is not Correct",
      "Test Diagnostic Event", report.getDiagnosticInfo());
  report = it.next().getReport();
  Assert.assertEquals("Attempt state not correct", TaskAttemptState.FAILED,
      report.getTaskAttemptState());
}
项目：hadoop    文件：TestAMWebServicesTasks.java    
@Test
public void testTaskId() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);
    for (Task task : jobsMap.get(id).getTasks().values()) {

      String tid = MRApps.toString(task.getID());
      ClientResponse response = r.path("ws").path("v1").path("mapreduce")
          .path("jobs").path(jobId).path("tasks").path(tid)
          .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
      assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
      JSONObject json = response.getEntity(JSONObject.class);
      assertEquals("incorrect number of elements", 1, json.length());
      JSONObject info = json.getJSONObject("task");
      verifyAMSingleTask(info, task);
    }
  }
}
项目：hadoop    文件：TestAMWebServicesTasks.java    
public void verifyAMTask(JSONArray arr, Job job, String type)
    throws JSONException {
  for (Task task : job.getTasks().values()) {
    TaskId id = task.getID();
    String tid = MRApps.toString(id);
    Boolean found = false;
    if (type != null && task.getType() == MRApps.taskType(type)) {

      for (int i = 0; i < arr.length(); i++) {
        JSONObject info = arr.getJSONObject(i);
        if (tid.matches(info.getString("id"))) {
          found = true;
          verifyAMSingleTask(info, task);
        }
      }
      assertTrue("task with id: " + tid + " not in web service output", found);
    }
  }
}
项目：hadoop    文件：TestAMWebServicesTasks.java    
public void verifyTaskGeneric(Task task, String id, String state,
    String type, String successfulAttempt, long startTime, long finishTime,
    long elapsedTime, float progress, String status) {

  TaskId taskid = task.getID();
  String tid = MRApps.toString(taskid);
  TaskReport report = task.getReport();

  WebServicesTestUtils.checkStringMatch("id", tid, id);
  WebServicesTestUtils.checkStringMatch("type", task.getType().toString(),
      type);
  WebServicesTestUtils.checkStringMatch("state", report.getTaskState()
      .toString(), state);
  // not easily checked without duplicating logic, just make sure its here
  assertNotNull("successfulAttempt null", successfulAttempt);
  assertEquals("startTime wrong", report.getStartTime(), startTime);
  assertEquals("finishTime wrong", report.getFinishTime(), finishTime);
  assertEquals("elapsedTime wrong", finishTime - startTime, elapsedTime);
  assertEquals("progress wrong", report.getProgress() * 100, progress, 1e-3f);
  assertEquals("status wrong", report.getStatus(), status);
}
项目：hadoop    文件：TestHsWebServicesTasks.java    
@Test
public void testTaskIdSlash() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);
    for (Task task : jobsMap.get(id).getTasks().values()) {

      String tid = MRApps.toString(task.getID());
      ClientResponse response = r.path("ws").path("v1").path("history")
          .path("mapreduce").path("jobs").path(jobId).path("tasks")
          .path(tid + "/").accept(MediaType.APPLICATION_JSON)
          .get(ClientResponse.class);
      assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
      JSONObject json = response.getEntity(JSONObject.class);
      assertEquals("incorrect number of elements", 1, json.length());
      JSONObject info = json.getJSONObject("task");
      verifyHsSingleTask(info, task);
    }
  }
}
项目：hadoop    文件：TestAMWebServicesTasks.java    
@Test
public void testTaskIdCountersSlash() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);
    for (Task task : jobsMap.get(id).getTasks().values()) {

      String tid = MRApps.toString(task.getID());
      ClientResponse response = r.path("ws").path("v1").path("mapreduce")
          .path("jobs").path(jobId).path("tasks").path(tid).path("counters/")
          .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
      assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
      JSONObject json = response.getEntity(JSONObject.class);
      assertEquals("incorrect number of elements", 1, json.length());
      JSONObject info = json.getJSONObject("jobTaskCounters");
      verifyAMJobTaskCounters(info, task);
    }
  }
}
项目：hadoop    文件：TestHsWebServicesAcls.java    
@Before
public void setup() throws IOException {
  this.conf = new JobConf();
  this.conf.set(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
      NullGroupsProvider.class.getName());
  this.conf.setBoolean(MRConfig.MR_ACLS_ENABLED, true);
  Groups.getUserToGroupsMappingService(conf);
  this.ctx = buildHistoryContext(this.conf);
  WebApp webApp = mock(HsWebApp.class);
  when(webApp.name()).thenReturn("hsmockwebapp");
  this.hsWebServices= new HsWebServices(ctx, conf, webApp);
  this.hsWebServices.setResponse(mock(HttpServletResponse.class));

  Job job = ctx.getAllJobs().values().iterator().next();
  this.jobIdStr = job.getID().toString();
  Task task = job.getTasks().values().iterator().next();
  this.taskIdStr = task.getID().toString();
  this.taskAttemptIdStr =
      task.getAttempts().keySet().iterator().next().toString();
}
项目：hadoop    文件：CompletedJob.java    
private void loadAllTasks() {
  if (tasksLoaded.get()) {
    return;
  }
  tasksLock.lock();
  try {
    if (tasksLoaded.get()) {
      return;
    }
    for (Map.Entry<TaskID, TaskInfo> entry : jobInfo.getAllTasks().entrySet()) {
      TaskId yarnTaskID = TypeConverter.toYarn(entry.getKey());
      TaskInfo taskInfo = entry.getValue();
      Task task = new CompletedTask(yarnTaskID, taskInfo);
      tasks.put(yarnTaskID, task);
      if (task.getType() == TaskType.MAP) {
        mapTasks.put(task.getID(), task);
      } else if (task.getType() == TaskType.REDUCE) {
        reduceTasks.put(task.getID(), task);
      }
    }
    tasksLoaded.set(true);
  } finally {
    tasksLock.unlock();
  }
}
项目：hadoop    文件：TestAMWebServicesTasks.java    
public void verifyAMJobTaskCounters(JSONObject info, Task task)
    throws JSONException {

  assertEquals("incorrect number of elements", 2, info.length());

  WebServicesTestUtils.checkStringMatch("id", MRApps.toString(task.getID()),
      info.getString("id"));
  // just do simple verification of fields - not data is correct
  // in the fields
  JSONArray counterGroups = info.getJSONArray("taskCounterGroup");
  for (int i = 0; i < counterGroups.length(); i++) {
    JSONObject counterGroup = counterGroups.getJSONObject(i);
    String name = counterGroup.getString("counterGroupName");
    assertTrue("name not set", (name != null && !name.isEmpty()));
    JSONArray counters = counterGroup.getJSONArray("counter");
    for (int j = 0; j < counters.length(); j++) {
      JSONObject counter = counters.getJSONObject(j);
      String counterName = counter.getString("name");
      assertTrue("name not set",
          (counterName != null && !counterName.isEmpty()));
      long value = counter.getLong("value");
      assertTrue("value  >= 0", value >= 0);
    }
  }
}
项目：hadoop    文件：TestAMWebApp.java    
@Test public void testSingleTaskCounterView() {
  AppContext appContext = new MockAppContext(0, 1, 1, 2);
  Map<String, String> params = getTaskParams(appContext);
  params.put(AMParams.COUNTER_GROUP, 
      "org.apache.hadoop.mapreduce.FileSystemCounter");
  params.put(AMParams.COUNTER_NAME, "HDFS_WRITE_OPS");

  // remove counters from one task attempt
  // to test handling of missing counters
  TaskId taskID = MRApps.toTaskID(params.get(AMParams.TASK_ID));
  Job job = appContext.getJob(taskID.getJobId());
  Task task = job.getTask(taskID);
  TaskAttempt attempt = task.getAttempts().values().iterator().next();
  attempt.getReport().setCounters(null);

  WebAppTests.testPage(SingleCounterPage.class, AppContext.class,
                       appContext, params);
}
项目：hadoop    文件：TestHsWebServicesTasks.java    
@Test
public void testJobTaskCountersXML() throws Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);
    for (Task task : jobsMap.get(id).getTasks().values()) {

      String tid = MRApps.toString(task.getID());
      ClientResponse response = r.path("ws").path("v1").path("history")
          .path("mapreduce").path("jobs").path(jobId).path("tasks").path(tid)
          .path("counters").accept(MediaType.APPLICATION_XML)
          .get(ClientResponse.class);
      assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
      String xml = response.getEntity(String.class);
      DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
      DocumentBuilder db = dbf.newDocumentBuilder();
      InputSource is = new InputSource();
      is.setCharacterStream(new StringReader(xml));
      Document dom = db.parse(is);
      NodeList info = dom.getElementsByTagName("jobTaskCounters");
      verifyHsTaskCountersXML(info, task);
    }
  }
}
项目：hadoop    文件：TestAMWebServicesAttempts.java    
@Test
public void testTaskAttemptsDefault() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);
    for (Task task : jobsMap.get(id).getTasks().values()) {

      String tid = MRApps.toString(task.getID());
      ClientResponse response = r.path("ws").path("v1").path("mapreduce")
          .path("jobs").path(jobId).path("tasks").path(tid).path("attempts")
          .get(ClientResponse.class);
      assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
      JSONObject json = response.getEntity(JSONObject.class);
      verifyAMTaskAttempts(json, task);
    }
  }
}
项目：hadoop    文件：HsWebServices.java    
@GET
@Path("/mapreduce/jobs/{jobid}/tasks")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public TasksInfo getJobTasks(@Context HttpServletRequest hsr,
    @PathParam("jobid") String jid, @QueryParam("type") String type) {

  init();
  Job job = AMWebServices.getJobFromJobIdString(jid, ctx);
  checkAccess(job, hsr);
  TasksInfo allTasks = new TasksInfo();
  for (Task task : job.getTasks().values()) {
    TaskType ttype = null;
    if (type != null && !type.isEmpty()) {
      try {
        ttype = MRApps.taskType(type);
      } catch (YarnRuntimeException e) {
        throw new BadRequestException("tasktype must be either m or r");
      }
    }
    if (ttype != null && task.getType() != ttype) {
      continue;
    }
    allTasks.add(new TaskInfo(task));
  }
  return allTasks;
}
项目：hadoop    文件：TestHsWebServicesAttempts.java    
public void verifyHsTaskAttempts(JSONObject json, Task task)
    throws JSONException {
  assertEquals("incorrect number of elements", 1, json.length());
  JSONObject attempts = json.getJSONObject("taskAttempts");
  assertEquals("incorrect number of elements", 1, json.length());
  JSONArray arr = attempts.getJSONArray("taskAttempt");
  for (TaskAttempt att : task.getAttempts().values()) {
    TaskAttemptId id = att.getID();
    String attid = MRApps.toString(id);
    Boolean found = false;

    for (int i = 0; i < arr.length(); i++) {
      JSONObject info = arr.getJSONObject(i);
      if (attid.matches(info.getString("id"))) {
        found = true;
        verifyHsTaskAttempt(info, att, task.getType());
      }
    }
    assertTrue("task attempt with id: " + attid
        + " not in web service output", found);
  }
}
项目：hadoop    文件：TestBlocks.java    
private Task getTask(long timestamp) {

  JobId jobId = new JobIdPBImpl();
  jobId.setId(0);
  jobId.setAppId(ApplicationIdPBImpl.newInstance(timestamp,1));

  TaskId taskId = new TaskIdPBImpl();
  taskId.setId(0);
  taskId.setTaskType(TaskType.REDUCE);
  taskId.setJobId(jobId);
  Task task = mock(Task.class);
  when(task.getID()).thenReturn(taskId);
  TaskReport report = mock(TaskReport.class);
  when(report.getProgress()).thenReturn(0.7f);
  when(report.getTaskState()).thenReturn(TaskState.SUCCEEDED);
  when(report.getStartTime()).thenReturn(100001L);
  when(report.getFinishTime()).thenReturn(100011L);

  when(task.getReport()).thenReturn(report);
  when(task.getType()).thenReturn(TaskType.REDUCE);
  return task;
}
项目：hadoop    文件：TestAMWebServicesAttempts.java    
public void verifyAMTaskAttempts(JSONObject json, Task task)
    throws JSONException {
  assertEquals("incorrect number of elements", 1, json.length());
  JSONObject attempts = json.getJSONObject("taskAttempts");
  assertEquals("incorrect number of elements", 1, json.length());
  JSONArray arr = attempts.getJSONArray("taskAttempt");
  for (TaskAttempt att : task.getAttempts().values()) {
    TaskAttemptId id = att.getID();
    String attid = MRApps.toString(id);
    Boolean found = false;

    for (int i = 0; i < arr.length(); i++) {
      JSONObject info = arr.getJSONObject(i);
      if (attid.matches(info.getString("id"))) {
        found = true;
        verifyAMTaskAttempt(info, att, task.getType());
      }
    }
    assertTrue("task attempt with id: " + attid
        + " not in web service output", found);
  }
}
项目：hadoop    文件：DefaultSpeculator.java    
/**
 * Absorbs one TaskAttemptStatus
 *
 * @param reportedStatus the status report that we got from a task attempt
 *        that we want to fold into the speculation data for this job
 * @param timestamp the time this status corresponds to.  This matters
 *        because statuses contain progress.
 */
protected void statusUpdate(TaskAttemptStatus reportedStatus, long timestamp) {

  String stateString = reportedStatus.taskState.toString();

  TaskAttemptId attemptID = reportedStatus.id;
  TaskId taskID = attemptID.getTaskId();
  Job job = context.getJob(taskID.getJobId());

  if (job == null) {
    return;
  }

  Task task = job.getTask(taskID);

  if (task == null) {
    return;
  }

  estimator.updateAttempt(reportedStatus, timestamp);

  if (stateString.equals(TaskAttemptState.RUNNING.name())) {
    runningTasks.putIfAbsent(taskID, Boolean.TRUE);
  } else {
    runningTasks.remove(taskID, Boolean.TRUE);
    if (!stateString.equals(TaskAttemptState.STARTING.name())) {
      runningTaskAttemptStatistics.remove(attemptID);
    }
  }
}
项目：hadoop    文件：TaskSpeculationPredicate.java    
boolean canSpeculate(AppContext context, TaskId taskID) {
  // This class rejects speculating any task that already has speculations,
  //  or isn't running.
  //  Subclasses should call TaskSpeculationPredicate.canSpeculate(...) , but
  //  can be even more restrictive.
  JobId jobID = taskID.getJobId();
  Job job = context.getJob(jobID);
  Task task = job.getTask(taskID);
  return task.getAttempts().size() == 1;
}
项目：hadoop    文件：TaskInfo.java    
private TaskAttempt getSuccessfulAttempt(Task task) {
  for (TaskAttempt attempt : task.getAttempts().values()) {
    if (attempt.getState() == TaskAttemptState.SUCCEEDED) {
      return attempt;
    }
  }
  return null;
}
项目：hadoop    文件：AMWebServices.java    
@GET
@Path("/jobs/{jobid}/tasks/{taskid}")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public TaskInfo getJobTask(@Context HttpServletRequest hsr,
    @PathParam("jobid") String jid, @PathParam("taskid") String tid) {

  init();
  Job job = getJobFromJobIdString(jid, appCtx);
  checkAccess(job, hsr);
  Task task = getTaskFromTaskIdString(tid, job);
  return new TaskInfo(task);
}