@Test(timeout=60000) public void testJobKillTimeout() throws Exception { long timeToWaitBeforeHardKill = 10000 + MRJobConfig.DEFAULT_MR_AM_HARD_KILL_TIMEOUT_MS; conf.setLong(MRJobConfig.MR_AM_HARD_KILL_TIMEOUT_MS, timeToWaitBeforeHardKill); clientDelegate = mock(ClientServiceDelegate.class); doAnswer( new Answer<ClientServiceDelegate>() { @Override public ClientServiceDelegate answer(InvocationOnMock invocation) throws Throwable { return clientDelegate; } } ).when(clientCache).getClient(any(JobID.class)); when(clientDelegate.getJobStatus(any(JobID.class))).thenReturn(new org.apache.hadoop.mapreduce.JobStatus(jobId, 0f, 0f, 0f, 0f, State.RUNNING, JobPriority.HIGH, "tmp", "tmp", "tmp", "tmp")); long startTimeMillis = System.currentTimeMillis(); yarnRunner.killJob(jobId); assertTrue("killJob should have waited at least " + timeToWaitBeforeHardKill + " ms.", System.currentTimeMillis() - startTimeMillis >= timeToWaitBeforeHardKill); }
@Test public void tesAllJobs() throws Exception { final ApplicationClientProtocol applicationsManager = Mockito.mock(ApplicationClientProtocol.class); GetApplicationsResponse allApplicationsResponse = Records .newRecord(GetApplicationsResponse.class); List<ApplicationReport> applications = new ArrayList<ApplicationReport>(); applications.add(getApplicationReport(YarnApplicationState.FINISHED, FinalApplicationStatus.FAILED)); applications.add(getApplicationReport(YarnApplicationState.FINISHED, FinalApplicationStatus.SUCCEEDED)); applications.add(getApplicationReport(YarnApplicationState.FINISHED, FinalApplicationStatus.KILLED)); applications.add(getApplicationReport(YarnApplicationState.FAILED, FinalApplicationStatus.FAILED)); allApplicationsResponse.setApplicationList(applications); Mockito.when( applicationsManager.getApplications(Mockito .any(GetApplicationsRequest.class))).thenReturn( allApplicationsResponse); ResourceMgrDelegate resourceMgrDelegate = new ResourceMgrDelegate( new YarnConfiguration()) { @Override protected void serviceStart() throws Exception { Assert.assertTrue(this.client instanceof YarnClientImpl); ((YarnClientImpl) this.client).setRMClient(applicationsManager); } }; JobStatus[] allJobs = resourceMgrDelegate.getAllJobs(); Assert.assertEquals(State.FAILED, allJobs[0].getState()); Assert.assertEquals(State.SUCCEEDED, allJobs[1].getState()); Assert.assertEquals(State.KILLED, allJobs[2].getState()); Assert.assertEquals(State.FAILED, allJobs[3].getState()); }
public static TaskAttemptState toYarn( org.apache.hadoop.mapred.TaskStatus.State state) { switch (state) { case COMMIT_PENDING: return TaskAttemptState.COMMIT_PENDING; case FAILED: case FAILED_UNCLEAN: return TaskAttemptState.FAILED; case KILLED: case KILLED_UNCLEAN: return TaskAttemptState.KILLED; case RUNNING: return TaskAttemptState.RUNNING; case SUCCEEDED: return TaskAttemptState.SUCCEEDED; case UNASSIGNED: return TaskAttemptState.STARTING; default: throw new YarnRuntimeException("Unrecognized State: " + state); } }
public static State fromYarn(YarnApplicationState yarnApplicationState, FinalApplicationStatus finalApplicationStatus) { switch (yarnApplicationState) { case NEW: case NEW_SAVING: case SUBMITTED: case ACCEPTED: return State.PREP; case RUNNING: return State.RUNNING; case FINISHED: if (finalApplicationStatus == FinalApplicationStatus.SUCCEEDED) { return State.SUCCEEDED; } else if (finalApplicationStatus == FinalApplicationStatus.KILLED) { return State.KILLED; } case FAILED: return State.FAILED; case KILLED: return State.KILLED; } throw new YarnRuntimeException("Unrecognized application state: " + yarnApplicationState); }
@Test public void testEnums() throws Exception { for (YarnApplicationState applicationState : YarnApplicationState.values()) { TypeConverter.fromYarn(applicationState, FinalApplicationStatus.FAILED); } // ad hoc test of NEW_SAVING, which is newly added Assert.assertEquals(State.PREP, TypeConverter.fromYarn( YarnApplicationState.NEW_SAVING, FinalApplicationStatus.FAILED)); for (TaskType taskType : TaskType.values()) { TypeConverter.fromYarn(taskType); } for (JobState jobState : JobState.values()) { TypeConverter.fromYarn(jobState); } for (QueueState queueState : QueueState.values()) { TypeConverter.fromYarn(queueState); } for (TaskState taskState : TaskState.values()) { TypeConverter.fromYarn(taskState); } }
@Test public void testJobToString() throws IOException, InterruptedException { Cluster cluster = mock(Cluster.class); ClientProtocol client = mock(ClientProtocol.class); when(cluster.getClient()).thenReturn(client); JobID jobid = new JobID("1014873536921", 6); JobStatus status = new JobStatus(jobid, 0.0f, 0.0f, 0.0f, 0.0f, State.FAILED, JobPriority.NORMAL, "root", "TestJobToString", "job file", "tracking url"); when(client.getJobStatus(jobid)).thenReturn(status); when(client.getTaskReports(jobid, TaskType.MAP)).thenReturn( new TaskReport[0]); when(client.getTaskReports(jobid, TaskType.REDUCE)).thenReturn( new TaskReport[0]); when(client.getTaskCompletionEvents(jobid, 0, 10)).thenReturn( new TaskCompletionEvent[0]); Job job = Job.getInstance(cluster, status, new JobConf()); Assert.assertNotNull(job.toString()); }
@Test public void testJobToString() throws IOException, InterruptedException { Cluster cluster = mock(Cluster.class); ClientProtocol client = mock(ClientProtocol.class); when(cluster.getClient()).thenReturn(client); JobID jobid = new JobID("1014873536921", 6); JobStatus status = new JobStatus(jobid, 0.0f, 0.0f, 0.0f, 0.0f, State.FAILED, JobPriority.DEFAULT, "root", "TestJobToString", "job file", "tracking url"); when(client.getJobStatus(jobid)).thenReturn(status); when(client.getTaskReports(jobid, TaskType.MAP)).thenReturn( new TaskReport[0]); when(client.getTaskReports(jobid, TaskType.REDUCE)).thenReturn( new TaskReport[0]); when(client.getTaskCompletionEvents(jobid, 0, 10)).thenReturn( new TaskCompletionEvent[0]); Job job = Job.getInstance(cluster, status, new JobConf()); Assert.assertNotNull(job.toString()); }
@Test(timeout=20000) public void testJobKill() throws Exception { clientDelegate = mock(ClientServiceDelegate.class); when(clientDelegate.getJobStatus(any(JobID.class))).thenReturn(new org.apache.hadoop.mapreduce.JobStatus(jobId, 0f, 0f, 0f, 0f, State.PREP, JobPriority.HIGH, "tmp", "tmp", "tmp", "tmp")); when(clientDelegate.killJob(any(JobID.class))).thenReturn(true); doAnswer( new Answer<ClientServiceDelegate>() { @Override public ClientServiceDelegate answer(InvocationOnMock invocation) throws Throwable { return clientDelegate; } } ).when(clientCache).getClient(any(JobID.class)); yarnRunner.killJob(jobId); verify(resourceMgrDelegate).killApplication(appId); when(clientDelegate.getJobStatus(any(JobID.class))).thenReturn(new org.apache.hadoop.mapreduce.JobStatus(jobId, 0f, 0f, 0f, 0f, State.RUNNING, JobPriority.HIGH, "tmp", "tmp", "tmp", "tmp")); yarnRunner.killJob(jobId); verify(clientDelegate).killJob(jobId); }
