public void waitForState(Job job, JobState finalState) throws Exception { int timeoutSecs = 0; JobReport report = job.getReport(); while (!finalState.equals(report.getJobState()) && timeoutSecs++ < 20) { System.out.println("Job State is : " + report.getJobState() + " Waiting for state : " + finalState + " map progress : " + report.getMapProgress() + " reduce progress : " + report.getReduceProgress()); report = job.getReport(); Thread.sleep(500); } System.out.println("Job State is : " + report.getJobState()); Assert.assertEquals("Job state is not correct (timedout)", finalState, job.getState()); }
public void verifyCompleted() { for (Job job : getContext().getAllJobs().values()) { JobReport jobReport = job.getReport(); System.out.println("Job start time :" + jobReport.getStartTime()); System.out.println("Job finish time :" + jobReport.getFinishTime()); Assert.assertTrue("Job start time is not less than finish time", jobReport.getStartTime() <= jobReport.getFinishTime()); Assert.assertTrue("Job finish time is in future", jobReport.getFinishTime() <= System.currentTimeMillis()); for (Task task : job.getTasks().values()) { TaskReport taskReport = task.getReport(); System.out.println("Task start time : " + taskReport.getStartTime()); System.out.println("Task finish time : " + taskReport.getFinishTime()); Assert.assertTrue("Task start time is not less than finish time", taskReport.getStartTime() <= taskReport.getFinishTime()); for (TaskAttempt attempt : task.getAttempts().values()) { TaskAttemptReport attemptReport = attempt.getReport(); Assert.assertTrue("Attempt start time is not less than finish time", attemptReport.getStartTime() <= attemptReport.getFinishTime()); } } } }
@Override public GetJobReportResponse getJobReport(GetJobReportRequest request) throws IOException { JobReport jobReport = recordFactory.newRecordInstance(JobReport.class); jobReport.setJobId(request.getJobId()); jobReport.setJobState(jobState); jobReport.setUser(applicationReport.getUser()); jobReport.setStartTime(applicationReport.getStartTime()); jobReport.setDiagnostics(applicationReport.getDiagnostics()); jobReport.setJobName(applicationReport.getName()); jobReport.setTrackingUrl(applicationReport.getTrackingUrl()); jobReport.setFinishTime(applicationReport.getFinishTime()); GetJobReportResponse resp = recordFactory.newRecordInstance(GetJobReportResponse.class); resp.setJobReport(jobReport); return resp; }
public JobStatus getJobStatus(JobID oldJobID) throws IOException { org.apache.hadoop.mapreduce.v2.api.records.JobId jobId = TypeConverter.toYarn(oldJobID); GetJobReportRequest request = recordFactory.newRecordInstance(GetJobReportRequest.class); request.setJobId(jobId); JobReport report = ((GetJobReportResponse) invoke("getJobReport", GetJobReportRequest.class, request)).getJobReport(); JobStatus jobStatus = null; if (report != null) { if (StringUtils.isEmpty(report.getJobFile())) { String jobFile = MRApps.getJobFile(conf, report.getUser(), oldJobID); report.setJobFile(jobFile); } String historyTrackingUrl = report.getTrackingUrl(); String url = StringUtils.isNotEmpty(historyTrackingUrl) ? historyTrackingUrl : trackingUrl; jobStatus = TypeConverter.fromYarn(report, url); } return jobStatus; }
@Override public GetJobReportResponse getJobReport(GetJobReportRequest request) throws IOException { amContact = true; JobReport jobReport = recordFactory.newRecordInstance(JobReport.class); jobReport.setJobId(request.getJobId()); jobReport.setJobState(JobState.RUNNING); jobReport.setJobName("TestClientRedirect-jobname"); jobReport.setUser("TestClientRedirect-user"); jobReport.setStartTime(0L); jobReport.setFinishTime(1L); GetJobReportResponse response = recordFactory .newRecordInstance(GetJobReportResponse.class); response.setJobReport(jobReport); return response; }
public static JobReport newJobReport(JobId jobId, String jobName, String userName, JobState state, long submitTime, long startTime, long finishTime, float setupProgress, float mapProgress, float reduceProgress, float cleanupProgress, String jobFile, List<AMInfo> amInfos, boolean isUber, String diagnostics) { JobReport report = Records.newRecord(JobReport.class); report.setJobId(jobId); report.setJobName(jobName); report.setUser(userName); report.setJobState(state); report.setSubmitTime(submitTime); report.setStartTime(startTime); report.setFinishTime(finishTime); report.setSetupProgress(setupProgress); report.setCleanupProgress(cleanupProgress); report.setMapProgress(mapProgress); report.setReduceProgress(reduceProgress); report.setJobFile(jobFile); report.setAMInfos(amInfos); report.setIsUber(isUber); report.setDiagnostics(diagnostics); return report; }
@Test public void testFromYarnJobReport() throws Exception { int jobStartTime = 612354; int jobFinishTime = 612355; JobState state = JobState.RUNNING; JobId jobId = Records.newRecord(JobId.class); JobReport jobReport = Records.newRecord(JobReport.class); ApplicationId applicationId = ApplicationId.newInstance(0, 0); jobId.setAppId(applicationId); jobId.setId(0); jobReport.setJobId(jobId); jobReport.setJobState(state); jobReport.setStartTime(jobStartTime); jobReport.setFinishTime(jobFinishTime); jobReport.setUser("TestTypeConverter-user"); JobStatus jobStatus = TypeConverter.fromYarn(jobReport, "dummy-jobfile"); Assert.assertEquals(jobStartTime, jobStatus.getStartTime()); Assert.assertEquals(jobFinishTime, jobStatus.getFinishTime()); Assert.assertEquals(state.toString(), jobStatus.getState().toString()); }
public static void verifyHsJobGeneric(Job job, String id, String user, String name, String state, String queue, long startTime, long finishTime, int mapsTotal, int mapsCompleted, int reducesTotal, int reducesCompleted) { JobReport report = job.getReport(); WebServicesTestUtils.checkStringMatch("id", MRApps.toString(job.getID()), id); WebServicesTestUtils.checkStringMatch("user", job.getUserName().toString(), user); WebServicesTestUtils.checkStringMatch("name", job.getName(), name); WebServicesTestUtils.checkStringMatch("state", job.getState().toString(), state); WebServicesTestUtils.checkStringMatch("queue", job.getQueueName(), queue); assertEquals("startTime incorrect", report.getStartTime(), startTime); assertEquals("finishTime incorrect", report.getFinishTime(), finishTime); assertEquals("mapsTotal incorrect", job.getTotalMaps(), mapsTotal); assertEquals("mapsCompleted incorrect", job.getCompletedMaps(), mapsCompleted); assertEquals("reducesTotal incorrect", job.getTotalReduces(), reducesTotal); assertEquals("reducesCompleted incorrect", job.getCompletedReduces(), reducesCompleted); }
private Job getJob() { Job job = mock(Job.class); JobId jobId = new JobIdPBImpl(); ApplicationId appId = ApplicationIdPBImpl.newInstance(System.currentTimeMillis(),4); jobId.setAppId(appId); jobId.setId(1); when(job.getID()).thenReturn(jobId); JobReport report = mock(JobReport.class); when(report.getStartTime()).thenReturn(100010L); when(report.getFinishTime()).thenReturn(100015L); when(job.getReport()).thenReturn(report); when(job.getName()).thenReturn("JobName"); when(job.getUserName()).thenReturn("UserName"); when(job.getQueueName()).thenReturn("QueueName"); when(job.getState()).thenReturn(JobState.SUCCEEDED); when(job.getTotalMaps()).thenReturn(3); when(job.getCompletedMaps()).thenReturn(2); when(job.getTotalReduces()).thenReturn(2); when(job.getCompletedReduces()).thenReturn(1); when(job.getCompletedReduces()).thenReturn(1); return job; }
private static JobsPair split(Map<JobId, Job> mocked) throws IOException { JobsPair ret = new JobsPair(); ret.full = Maps.newHashMap(); ret.partial = Maps.newHashMap(); for(Map.Entry<JobId, Job> entry: mocked.entrySet()) { JobId id = entry.getKey(); Job j = entry.getValue(); MockCompletedJob mockJob = new MockCompletedJob(j); // use MockCompletedJob to set everything below to make sure // consistent with what history server would do ret.full.put(id, mockJob); JobReport report = mockJob.getReport(); JobIndexInfo info = new JobIndexInfo(report.getStartTime(), report.getFinishTime(), mockJob.getUserName(), mockJob.getName(), id, mockJob.getCompletedMaps(), mockJob.getCompletedReduces(), String.valueOf(mockJob.getState())); info.setJobStartTime(report.getStartTime()); info.setQueueName(mockJob.getQueueName()); ret.partial.put(id, new PartialJob(info, id)); } return ret; }
@Test (timeout=100000) public void testCompletedJob() throws Exception { HistoryFileInfo info = mock(HistoryFileInfo.class); when(info.getConfFile()).thenReturn(fullConfPath); //Re-initialize to verify the delayed load. completedJob = new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user", info, jobAclsManager); //Verify tasks loaded based on loadTask parameter. assertEquals(loadTasks, completedJob.tasksLoaded.get()); assertEquals(1, completedJob.getAMInfos().size()); assertEquals(10, completedJob.getCompletedMaps()); assertEquals(1, completedJob.getCompletedReduces()); assertEquals(12, completedJob.getTasks().size()); //Verify tasks loaded at this point. assertEquals(true, completedJob.tasksLoaded.get()); assertEquals(10, completedJob.getTasks(TaskType.MAP).size()); assertEquals(2, completedJob.getTasks(TaskType.REDUCE).size()); assertEquals("user", completedJob.getUserName()); assertEquals(JobState.SUCCEEDED, completedJob.getState()); JobReport jobReport = completedJob.getReport(); assertEquals("user", jobReport.getUser()); assertEquals(JobState.SUCCEEDED, jobReport.getJobState()); }
public static JobReport newJobReport(JobId jobId, String jobName, String userName, JobState state, long submitTime, long startTime, long finishTime, float setupProgress, float mapProgress, float reduceProgress, float cleanupProgress, String jobFile, List<AMInfo> amInfos, boolean isUber, String diagnostics, Priority priority) { JobReport report = Records.newRecord(JobReport.class); report.setJobId(jobId); report.setJobName(jobName); report.setUser(userName); report.setJobState(state); report.setSubmitTime(submitTime); report.setStartTime(startTime); report.setFinishTime(finishTime); report.setSetupProgress(setupProgress); report.setCleanupProgress(cleanupProgress); report.setMapProgress(mapProgress); report.setReduceProgress(reduceProgress); report.setJobFile(jobFile); report.setAMInfos(amInfos); report.setIsUber(isUber); report.setDiagnostics(diagnostics); report.setJobPriority(priority); return report; }
@Test public void testFromYarnJobReport() throws Exception { int jobStartTime = 612354; int jobFinishTime = 612355; JobState state = JobState.RUNNING; JobId jobId = Records.newRecord(JobId.class); JobReport jobReport = Records.newRecord(JobReport.class); ApplicationId applicationId = ApplicationId.newInstance(0, 0); jobId.setAppId(applicationId); jobId.setId(0); jobReport.setJobId(jobId); jobReport.setJobState(state); jobReport.setStartTime(jobStartTime); jobReport.setFinishTime(jobFinishTime); jobReport.setUser("TestTypeConverter-user"); jobReport.setJobPriority(Priority.newInstance(0)); JobStatus jobStatus = TypeConverter.fromYarn(jobReport, "dummy-jobfile"); Assert.assertEquals(jobStartTime, jobStatus.getStartTime()); Assert.assertEquals(jobFinishTime, jobStatus.getFinishTime()); Assert.assertEquals(state.toString(), jobStatus.getState().toString()); Assert.assertEquals(JobPriority.DEFAULT, jobStatus.getPriority()); }
private void constructJobReport() { report = Records.newRecord(JobReport.class); report.setJobId(jobId); report.setJobState(JobState.valueOf(jobInfo.getJobStatus())); report.setSubmitTime(jobInfo.getSubmitTime()); report.setStartTime(jobInfo.getLaunchTime()); report.setFinishTime(jobInfo.getFinishTime()); report.setJobName(jobInfo.getJobname()); report.setUser(jobInfo.getUsername()); report.setMapProgress((float) getCompletedMaps() / getTotalMaps()); report.setReduceProgress((float) getCompletedReduces() / getTotalReduces()); report.setJobFile(getConfFile().toString()); String historyUrl = "N/A"; try { historyUrl = MRWebAppUtil.getApplicationWebURLOnJHSWithoutScheme(conf, jobId.getAppId()); } catch (UnknownHostException e) { //Ignore. } report.setTrackingUrl(historyUrl); report.setAMInfos(getAMInfos()); report.setIsUber(isUber()); }
