/** * test some methods of CompletedTaskAttempt */ @Test (timeout=5000) public void testCompletedTaskAttempt(){ TaskAttemptInfo attemptInfo= mock(TaskAttemptInfo.class); when(attemptInfo.getRackname()).thenReturn("Rackname"); when(attemptInfo.getShuffleFinishTime()).thenReturn(11L); when(attemptInfo.getSortFinishTime()).thenReturn(12L); when(attemptInfo.getShufflePort()).thenReturn(10); JobID jobId= new JobID("12345",0); TaskID taskId =new TaskID(jobId,TaskType.REDUCE, 0); TaskAttemptID taskAttemptId= new TaskAttemptID(taskId, 0); when(attemptInfo.getAttemptId()).thenReturn(taskAttemptId); CompletedTaskAttempt taskAttemt= new CompletedTaskAttempt(null,attemptInfo); assertEquals( "Rackname", taskAttemt.getNodeRackName()); assertEquals( Phase.CLEANUP, taskAttemt.getPhase()); assertTrue( taskAttemt.isFinished()); assertEquals( 11L, taskAttemt.getShuffleFinishTime()); assertEquals( 12L, taskAttemt.getSortFinishTime()); assertEquals( 10, taskAttemt.getShufflePort()); }
private TimelineEntity createTaskAttemptEntity(TaskAttemptInfo taskAttemptInfo) { TimelineEntity taskAttempt = new TimelineEntity(); taskAttempt.setEntityType(TASK_ATTEMPT); taskAttempt.setEntityId(taskAttemptInfo.getAttemptId().toString()); taskAttempt.setStartTime(taskAttemptInfo.getStartTime()); taskAttempt.addOtherInfo("START_TIME", taskAttemptInfo.getStartTime()); taskAttempt.addOtherInfo("FINISH_TIME", taskAttemptInfo.getFinishTime()); taskAttempt.addOtherInfo("MAP_FINISH_TIME", taskAttemptInfo.getMapFinishTime()); taskAttempt.addOtherInfo("SHUFFLE_FINISH_TIME", taskAttemptInfo.getShuffleFinishTime()); taskAttempt.addOtherInfo("SORT_FINISH_TIME", taskAttemptInfo.getSortFinishTime()); taskAttempt.addOtherInfo("TASK_STATUS", taskAttemptInfo.getTaskStatus()); taskAttempt.addOtherInfo("STATE", taskAttemptInfo.getState()); taskAttempt.addOtherInfo("ERROR", taskAttemptInfo.getError()); taskAttempt.addOtherInfo("CONTAINER_ID", taskAttemptInfo.getContainerId().toString()); LOG.info("converted task attempt " + taskAttemptInfo.getAttemptId() + " to a timeline entity"); return taskAttempt; }
/** * This method is responsible for populating the setup phase details. * @return TaskOutputDetails contains the details of the set up phase. */ private PhaseDetails prepareSetupDetails(JobInfo jobInfo,Map<TaskAttemptID, TaskAttemptInfo> tasks){ PhaseDetails phaseDetails = new PhaseDetails(); List<TaskOutputDetails> taskOutputDetails = new ArrayList<TaskOutputDetails>(); TaskOutputDetails tod; tod = new TaskOutputDetails(); tod.setTaskType("SETUP"); tod.setTaskID("Setup"); for (Map.Entry<TaskAttemptID, TaskAttemptInfo> task : tasks .entrySet()) { TaskAttemptInfo taskAttemptInfo = (TaskAttemptInfo) (task.getValue()); tod.setLocation(taskAttemptInfo.getHostname()); } long startPoint = jobInfo.getSubmitTime(); tod.setStartPoint(0); long endPoint = (jobInfo.getLaunchTime()-startPoint) / CONVERSION_FACTOR_MILLISECS_TO_SECS; tod.setEndPoint(endPoint); tod.setDataFlowRate(0); taskOutputDetails.add(tod); phaseDetails.setTaskOutputDetails(taskOutputDetails); phaseDetails.setAvgDataFlowRate(0); return phaseDetails; }
/** * This method is responsible for populating the clean up phase details. * @return TaskOutputDetails contains the details of the clean up phase. */ private PhaseDetails prepareCleanupDetails(JobInfo jobInfo, Map<TaskAttemptID, TaskAttemptInfo> tasks){ PhaseDetails phaseDetails = new PhaseDetails(); List<TaskOutputDetails> cleanupTaskOuptputDetails = new ArrayList<TaskOutputDetails>(); TaskOutputDetails taskOutputDetails = new TaskOutputDetails(); taskOutputDetails.setTaskType("CLEANUP"); taskOutputDetails.setTaskID("Cleanup"); for (Map.Entry<TaskAttemptID, TaskAttemptInfo> task : tasks .entrySet()) { TaskAttemptInfo taskAttemptInfo = (TaskAttemptInfo) (task.getValue()); taskOutputDetails.setLocation(taskAttemptInfo.getHostname()); } long startPoint = getMaxReduceTime(tasks,jobInfo.getSubmitTime()); taskOutputDetails.setStartPoint(startPoint); long endPoint = (jobInfo.getFinishTime() - jobInfo.getSubmitTime())/CONVERSION_FACTOR_MILLISECS_TO_SECS; taskOutputDetails.setEndPoint(endPoint); taskOutputDetails.setDataFlowRate(0); cleanupTaskOuptputDetails.add(taskOutputDetails); phaseDetails.setTaskOutputDetails(cleanupTaskOuptputDetails); phaseDetails.setAvgDataFlowRate(0); return phaseDetails; }
public TaskAttemptRecoverEvent(TaskAttemptId id, TaskAttemptInfo taInfo, OutputCommitter committer, boolean recoverOutput) { super(id, TaskAttemptEventType.TA_RECOVER); this.taInfo = taInfo; this.committer = committer; this.recoverAttemptOutput = recoverOutput; }
private TaskAttemptInfo getMockTaskAttemptInfo(TaskAttemptID tai, TaskAttemptState tas) { ContainerId ci = mock(ContainerId.class); Counters counters = mock(Counters.class); TaskType tt = TaskType.MAP; long finishTime = System.currentTimeMillis(); TaskAttemptInfo mockTAinfo = mock(TaskAttemptInfo.class); when(mockTAinfo.getAttemptId()).thenReturn(tai); when(mockTAinfo.getContainerId()).thenReturn(ci); when(mockTAinfo.getCounters()).thenReturn(counters); when(mockTAinfo.getError()).thenReturn(""); when(mockTAinfo.getFinishTime()).thenReturn(finishTime); when(mockTAinfo.getHostname()).thenReturn("localhost"); when(mockTAinfo.getHttpPort()).thenReturn(23); when(mockTAinfo.getMapFinishTime()).thenReturn(finishTime - 1000L); when(mockTAinfo.getPort()).thenReturn(24); when(mockTAinfo.getRackname()).thenReturn("defaultRack"); when(mockTAinfo.getShuffleFinishTime()).thenReturn(finishTime - 2000L); when(mockTAinfo.getShufflePort()).thenReturn(25); when(mockTAinfo.getSortFinishTime()).thenReturn(finishTime - 3000L); when(mockTAinfo.getStartTime()).thenReturn(finishTime -10000); when(mockTAinfo.getState()).thenReturn("task in progress"); when(mockTAinfo.getTaskStatus()).thenReturn(tas.toString()); when(mockTAinfo.getTaskType()).thenReturn(tt); when(mockTAinfo.getTrackerName()).thenReturn("TrackerName"); return mockTAinfo; }
private void loadAllTaskAttempts() { if (taskAttemptsLoaded.get()) { return; } taskAttemptsLock.lock(); try { if (taskAttemptsLoaded.get()) { return; } for (TaskAttemptInfo attemptHistory : taskInfo.getAllTaskAttempts() .values()) { CompletedTaskAttempt attempt = new CompletedTaskAttempt(taskId, attemptHistory); reportDiagnostics.addAll(attempt.getDiagnostics()); attempts.put(attempt.getID(), attempt); if (successfulAttempt == null && attemptHistory.getTaskStatus() != null && attemptHistory.getTaskStatus().equals( TaskState.SUCCEEDED.toString())) { successfulAttempt = TypeConverter.toYarn(attemptHistory.getAttemptId()); } } taskAttemptsLoaded.set(true); } finally { taskAttemptsLock.unlock(); } }
CompletedTaskAttempt(TaskId taskId, TaskAttemptInfo attemptInfo) { this.attemptInfo = attemptInfo; this.attemptId = TypeConverter.toYarn(attemptInfo.getAttemptId()); if (attemptInfo.getTaskStatus() != null) { this.state = TaskAttemptState.valueOf(attemptInfo.getTaskStatus()); } else { this.state = TaskAttemptState.KILLED; localDiagMessage = "Attmpt state missing from History : marked as KILLED"; diagnostics.add(localDiagMessage); } if (attemptInfo.getError() != null) { diagnostics.add(attemptInfo.getError()); } }
private Set<TimelineEntity> createTaskAttemptEntities(TaskInfo taskInfo) { Set<TimelineEntity> taskAttempts = new HashSet<TimelineEntity>(); Map<TaskAttemptID,TaskAttemptInfo> taskAttemptInfoMap = taskInfo.getAllTaskAttempts(); LOG.info("task " + taskInfo.getTaskId() + " has " + taskAttemptInfoMap.size() + " task attempts"); for (TaskAttemptInfo taskAttemptInfo: taskAttemptInfoMap.values()) { TimelineEntity taskAttempt = createTaskAttemptEntity(taskAttemptInfo); taskAttempts.add(taskAttempt); } return taskAttempts; }
@Test (timeout=5000) public void testTaskStartTimes() { TaskId taskId = mock(TaskId.class); TaskInfo taskInfo = mock(TaskInfo.class); Map<TaskAttemptID, TaskAttemptInfo> taskAttempts = new TreeMap<TaskAttemptID, TaskAttemptInfo>(); TaskAttemptID id = new TaskAttemptID("0", 0, TaskType.MAP, 0, 0); TaskAttemptInfo info = mock(TaskAttemptInfo.class); when(info.getAttemptId()).thenReturn(id); when(info.getStartTime()).thenReturn(10l); taskAttempts.put(id, info); id = new TaskAttemptID("1", 0, TaskType.MAP, 1, 1); info = mock(TaskAttemptInfo.class); when(info.getAttemptId()).thenReturn(id); when(info.getStartTime()).thenReturn(20l); taskAttempts.put(id, info); when(taskInfo.getAllTaskAttempts()).thenReturn(taskAttempts); CompletedTask task = new CompletedTask(taskId, taskInfo); TaskReport report = task.getReport(); // Make sure the startTime returned by report is the lesser of the // attempy launch times assertTrue(report.getStartTime() == 10); }
