public void verifyJobAttemptsGeneric(Job job, String nodeHttpAddress, String nodeId, int id, long startTime, String containerId, String logsLink) { boolean attemptFound = false; for (AMInfo amInfo : job.getAMInfos()) { if (amInfo.getAppAttemptId().getAttemptId() == id) { attemptFound = true; String nmHost = amInfo.getNodeManagerHost(); int nmHttpPort = amInfo.getNodeManagerHttpPort(); int nmPort = amInfo.getNodeManagerPort(); WebServicesTestUtils.checkStringMatch("nodeHttpAddress", nmHost + ":" + nmHttpPort, nodeHttpAddress); WebServicesTestUtils.checkStringMatch("nodeId", NodeId.newInstance(nmHost, nmPort).toString(), nodeId); assertTrue("startime not greater than 0", startTime > 0); WebServicesTestUtils.checkStringMatch("containerId", amInfo .getContainerId().toString(), containerId); String localLogsLink =ujoin("node", "containerlogs", containerId, job.getUserName()); assertTrue("logsLink", logsLink.contains(localLogsLink)); } } assertTrue("attempt: " + id + " was not found", attemptFound); }
public static JobReport newJobReport(JobId jobId, String jobName, String userName, JobState state, long submitTime, long startTime, long finishTime, float setupProgress, float mapProgress, float reduceProgress, float cleanupProgress, String jobFile, List<AMInfo> amInfos, boolean isUber, String diagnostics) { JobReport report = Records.newRecord(JobReport.class); report.setJobId(jobId); report.setJobName(jobName); report.setUser(userName); report.setJobState(state); report.setSubmitTime(submitTime); report.setStartTime(startTime); report.setFinishTime(finishTime); report.setSetupProgress(setupProgress); report.setCleanupProgress(cleanupProgress); report.setMapProgress(mapProgress); report.setReduceProgress(reduceProgress); report.setJobFile(jobFile); report.setAMInfos(amInfos); report.setIsUber(isUber); report.setDiagnostics(diagnostics); return report; }
@GET @Path("/mapreduce/jobs/{jobid}/jobattempts") @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) public AMAttemptsInfo getJobAttempts(@PathParam("jobid") String jid) { init(); Job job = AMWebServices.getJobFromJobIdString(jid, ctx); AMAttemptsInfo amAttempts = new AMAttemptsInfo(); for (AMInfo amInfo : job.getAMInfos()) { AMAttemptInfo attempt = new AMAttemptInfo(amInfo, MRApps.toString(job .getID()), job.getUserName(), uriInfo.getBaseUri().toString(), webapp.name()); amAttempts.add(attempt); } return amAttempts; }
public static JobReport newJobReport(JobId jobId, String jobName, String userName, JobState state, long submitTime, long startTime, long finishTime, float setupProgress, float mapProgress, float reduceProgress, float cleanupProgress, String jobFile, List<AMInfo> amInfos, boolean isUber, String diagnostics, Priority priority) { JobReport report = Records.newRecord(JobReport.class); report.setJobId(jobId); report.setJobName(jobName); report.setUser(userName); report.setJobState(state); report.setSubmitTime(submitTime); report.setStartTime(startTime); report.setFinishTime(finishTime); report.setSetupProgress(setupProgress); report.setCleanupProgress(cleanupProgress); report.setMapProgress(mapProgress); report.setReduceProgress(reduceProgress); report.setJobFile(jobFile); report.setAMInfos(amInfos); report.setIsUber(isUber); report.setDiagnostics(diagnostics); report.setJobPriority(priority); return report; }
@GET @Path("/jobs/{jobid}/jobattempts") @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) public AMAttemptsInfo getJobAttempts(@PathParam("jobid") String jid) { init(); Job job = getJobFromJobIdString(jid, appCtx); AMAttemptsInfo amAttempts = new AMAttemptsInfo(); for (AMInfo amInfo : job.getAMInfos()) { AMAttemptInfo attempt = new AMAttemptInfo(amInfo, MRApps.toString( job.getID()), job.getUserName()); amAttempts.add(attempt); } return amAttempts; }
private void verifyJobReport(JobReport jr) { Assert.assertNotNull("JobReport is null", jr); List<AMInfo> amInfos = jr.getAMInfos(); Assert.assertEquals(1, amInfos.size()); Assert.assertEquals(JobState.RUNNING, jr.getJobState()); AMInfo amInfo = amInfos.get(0); Assert.assertEquals(MRApp.NM_HOST, amInfo.getNodeManagerHost()); Assert.assertEquals(MRApp.NM_PORT, amInfo.getNodeManagerPort()); Assert.assertEquals(MRApp.NM_HTTP_PORT, amInfo.getNodeManagerHttpPort()); Assert.assertEquals(1, amInfo.getAppAttemptId().getAttemptId()); Assert.assertEquals(1, amInfo.getContainerId().getApplicationAttemptId() .getAttemptId()); Assert.assertTrue(amInfo.getStartTime() > 0); Assert.assertEquals(false, jr.isUber()); }
private static AMInfo createAMInfo(int attempt) { ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance( ApplicationId.newInstance(100, 1), attempt); ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1); return MRBuilderUtils.newAMInfo(appAttemptId, System.currentTimeMillis(), containerId, NM_HOST, NM_PORT, NM_HTTP_PORT); }
private void verifyJobReport(JobReport jobReport, JobId jobId) { List<AMInfo> amInfos = jobReport.getAMInfos(); Assert.assertEquals(1, amInfos.size()); AMInfo amInfo = amInfos.get(0); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(jobId.getAppId(), 1); ContainerId amContainerId = ContainerId.newContainerId(appAttemptId, 1); Assert.assertEquals(appAttemptId, amInfo.getAppAttemptId()); Assert.assertEquals(amContainerId, amInfo.getContainerId()); Assert.assertTrue(jobReport.getSubmitTime() > 0); Assert.assertTrue(jobReport.getStartTime() > 0 && jobReport.getStartTime() >= jobReport.getSubmitTime()); Assert.assertTrue(jobReport.getFinishTime() > 0 && jobReport.getFinishTime() >= jobReport.getStartTime()); }
@Override public synchronized void setAMInfos(List<AMInfo> amInfos) { maybeInitBuilder(); if (amInfos == null) { this.builder.clearAmInfos(); this.amInfos = null; return; } initAMInfos(); this.amInfos.clear(); this.amInfos.addAll(amInfos); }
private synchronized void initAMInfos() { if (this.amInfos != null) { return; } JobReportProtoOrBuilder p = viaProto ? proto : builder; List<AMInfoProto> list = p.getAmInfosList(); this.amInfos = new ArrayList<AMInfo>(); for (AMInfoProto amInfoProto : list) { this.amInfos.add(convertFromProtoFormat(amInfoProto)); } }
private synchronized void addAMInfosToProto() { maybeInitBuilder(); builder.clearAmInfos(); if (this.amInfos == null) return; for (AMInfo amInfo : this.amInfos) { builder.addAmInfos(convertToProtoFormat(amInfo)); } }
public static AMInfo newAMInfo(ApplicationAttemptId appAttemptId, long startTime, ContainerId containerId, String nmHost, int nmPort, int nmHttpPort) { AMInfo amInfo = Records.newRecord(AMInfo.class); amInfo.setAppAttemptId(appAttemptId); amInfo.setStartTime(startTime); amInfo.setContainerId(containerId); amInfo.setNodeManagerHost(nmHost); amInfo.setNodeManagerPort(nmPort); amInfo.setNodeManagerHttpPort(nmHttpPort); return amInfo; }
@Override public List<AMInfo> getAMInfos() { List<AMInfo> amInfos = new LinkedList<AMInfo>(); for (org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.AMInfo jhAmInfo : jobInfo .getAMInfos()) { AMInfo amInfo = MRBuilderUtils.newAMInfo(jhAmInfo.getAppAttemptId(), jhAmInfo.getStartTime(), jhAmInfo.getContainerId(), jhAmInfo.getNodeManagerHost(), jhAmInfo.getNodeManagerPort(), jhAmInfo.getNodeManagerHttpPort()); amInfos.add(amInfo); } return amInfos; }
public void verifyHsJobAttemptsGeneric(Job job, String nodeHttpAddress, String nodeId, int id, long startTime, String containerId, String logsLink) { boolean attemptFound = false; for (AMInfo amInfo : job.getAMInfos()) { if (amInfo.getAppAttemptId().getAttemptId() == id) { attemptFound = true; String nmHost = amInfo.getNodeManagerHost(); int nmHttpPort = amInfo.getNodeManagerHttpPort(); int nmPort = amInfo.getNodeManagerPort(); WebServicesTestUtils.checkStringMatch("nodeHttpAddress", nmHost + ":" + nmHttpPort, nodeHttpAddress); WebServicesTestUtils.checkStringMatch("nodeId", NodeId.newInstance(nmHost, nmPort).toString(), nodeId); assertTrue("startime not greater than 0", startTime > 0); WebServicesTestUtils.checkStringMatch("containerId", amInfo .getContainerId().toString(), containerId); String localLogsLink = join( "hsmockwebapp", ujoin("logs", nodeId, containerId, MRApps.toString(job.getID()), job.getUserName())); assertTrue("logsLink", logsLink.contains(localLogsLink)); } } assertTrue("attempt: " + id + " was not found", attemptFound); }
public static JobReport newJobReport(JobId jobId, String jobName, String userName, JobState state, long submitTime, long startTime, long finishTime, float setupProgress, float mapProgress, float reduceProgress, float cleanupProgress, String jobFile, List<AMInfo> amInfos, boolean isUber, String diagnostics) { return newJobReport(jobId, jobName, userName, state, submitTime, startTime, finishTime, setupProgress, mapProgress, reduceProgress, cleanupProgress, jobFile, amInfos, isUber, diagnostics, Priority.newInstance(0)); }
