private static Values getPre21Value(String name) { if (name.equalsIgnoreCase("JOB_CLEANUP")) { return Values.CLEANUP; } if (name.equalsIgnoreCase("JOB_SETUP")) { return Values.SETUP; } // Note that pre-21, the task state of a successful task was logged as // SUCCESS while from 21 onwards, its logged as SUCCEEDED. if (name.equalsIgnoreCase(TaskStatus.State.SUCCEEDED.toString())) { return Values.SUCCESS; } return Values.valueOf(StringUtils.toUpperCase(name)); }
private TaskAttemptInfo getSuccessfulAttemptInfo(TaskType type, int task) { TaskAttemptInfo ret; for (int i = 0; true; ++i) { // Rumen should make up an attempt if it's missing. Or this won't work // at all. It's hard to discern what is happening in there. ret = jobdesc.getTaskAttemptInfo(type, task, i); if (ret.getRunState() == TaskStatus.State.SUCCEEDED) { break; } } if(ret.getRunState() != TaskStatus.State.SUCCEEDED) { LOG.warn("No sucessful attempts tasktype " + type +" task "+ task); } return ret; }
private static Values getPre21Value(String name) { if (name.equalsIgnoreCase("JOB_CLEANUP")) { return Values.CLEANUP; } if (name.equalsIgnoreCase("JOB_SETUP")) { return Values.SETUP; } // Note that pre-21, the task state of a successful task was logged as // SUCCESS while from 21 onwards, its logged as SUCCEEDED. if (name.equalsIgnoreCase(TaskStatus.State.SUCCEEDED.toString())) { return Values.SUCCESS; } return Values.valueOf(name.toUpperCase()); }
public TaskInfoImpl( TaskID taskID, double progress, int runningAttempts, int killedAttempts, int failedAttempts, TaskStatus[] taskStatus, boolean setupOrCleanup, String[] taskTrackers) { this.progress = progress; this.taskID = taskID; this.killedAttempts = killedAttempts; this.failedAttempts = failedAttempts; this.runningAttempts = runningAttempts; if (taskStatus != null) { this.taskStatus = taskStatus; } else { if (taskID.isMap()) { this.taskStatus = new MapTaskStatus[] {}; } else { this.taskStatus = new ReduceTaskStatus[] {}; } } this.setupOrCleanup = setupOrCleanup; this.taskTrackers = taskTrackers; }
@Override public void write(DataOutput out) throws IOException { taskID.write(out); out.writeDouble(progress); out.writeInt(runningAttempts); out.writeInt(killedAttempts); out.writeInt(failedAttempts); out.writeInt(taskStatus.length); for (TaskStatus t : taskStatus) { t.write(out); out.writeUTF(t.getTaskTracker()); } out.writeBoolean(setupOrCleanup); out.writeInt(taskTrackers.length); for(String tt : taskTrackers) { out.writeUTF(tt); } }
/** * It uses to check whether task is started or not. * @param taskInfo task information * @return true if task is running. * @throws IOException if an I/O error occurs. */ public boolean isTaskStarted(TaskInfo taskInfo) throws IOException { JTProtocol wovenClient = getProxy(); int counter = 0; while (counter < 60) { if (taskInfo.getTaskStatus().length > 0) { if (taskInfo.getTaskStatus()[0].getRunState() == TaskStatus.State.RUNNING) { break; } } UtilsForTests.waitFor(1000); taskInfo = wovenClient.getTaskInfo(taskInfo.getTaskID()); counter++; } return (counter != 60)? true : false; }
/** * This methods provides the information on the particular task managed * by a task tracker has stopped or not. * @param TaskID is id of the task to get the status. * @throws IOException if there is an error. * @return true is stopped. */ public boolean isTaskStopped(TaskID tID) throws IOException { int counter = 0; if(tID != null && proxy.getTask(tID) != null) { TaskStatus.State tState= proxy.getTask(tID).getTaskStatus().getRunState(); while ( counter < 60) { if(tState != TaskStatus.State.RUNNING && tState != TaskStatus.State.UNASSIGNED) { break; } UtilsForTests.waitFor(1000); tState= proxy.getTask(tID).getTaskStatus().getRunState(); counter++; } } return (counter != 60)? true : false; }
/** * Metering: Occupied Slots * (Finish - Start) * @param tip {@link TaskInProgress} to be metered which just completed, * cannot be <code>null</code> * @param status {@link TaskStatus} of the completed task, cannot be * <code>null</code> */ @SuppressWarnings("deprecation") private void meterTaskAttemptUnprotected(TaskInProgress tip, TaskStatus status) { Counter slotCounter = (tip.isMapTask()) ? Counter.SLOTS_MILLIS_MAPS : Counter.SLOTS_MILLIS_REDUCES; jobCounters.incrCounter(slotCounter, tip.getNumSlotsRequired() * (status.getFinishTime() - status.getStartTime())); if (!tip.isMapTask()) { jobCounters.incrCounter(Counter.SLOTS_MILLIS_REDUCES_COPY, tip.getNumSlotsRequired() * (status.getShuffleFinishTime() - status.getStartTime())); jobCounters.incrCounter(Counter.SLOTS_MILLIS_REDUCES_SORT, tip.getNumSlotsRequired() * (status.getSortFinishTime() - status.getShuffleFinishTime())); jobCounters.incrCounter(Counter.SLOTS_MILLIS_REDUCES_REDUCE, tip.getNumSlotsRequired() * (status.getFinishTime() - status.getSortFinishTime())); } }
public ShuffleScheduler(JobConf job, TaskStatus status, ExceptionReporter reporter, Progress progress, Counters.Counter shuffledMapsCounter, Counters.Counter reduceShuffleBytes, Counters.Counter failedShuffleCounter) { totalMaps = job.getNumMapTasks(); abortFailureLimit = Math.max(30, totalMaps / 10); remainingMaps = totalMaps; finishedMaps = new boolean[remainingMaps]; this.reporter = reporter; this.status = status; this.progress = progress; this.shuffledMapsCounter = shuffledMapsCounter; this.reduceShuffleBytes = reduceShuffleBytes; this.failedShuffleCounter = failedShuffleCounter; this.startTime = System.currentTimeMillis(); lastProgressTime = startTime; referee.start(); this.maxFailedUniqueFetches = Math.min(totalMaps, this.maxFailedUniqueFetches); this.maxFetchFailuresBeforeReporting = job.getInt( MRJobConfig.SHUFFLE_FETCH_FAILURES, REPORT_FAILURE_LIMIT); this.reportReadErrorImmediately = job.getBoolean( MRJobConfig.SHUFFLE_NOTIFY_READERROR, true); }
public TaskInfoImpl( TaskID taskID, double progress, int runningAttempts, int killedAttempts, int failedAttempts, TaskStatus[] taskStatus, boolean setupOrCleanup, String[] taskTrackers) { this.progress = progress; this.taskID = taskID; this.killedAttempts = killedAttempts; this.failedAttempts = failedAttempts; this.runningAttempts = runningAttempts; if (taskStatus != null) { this.taskStatus = taskStatus; } else { if (taskID.getTaskType() == TaskType.MAP) { this.taskStatus = new MapTaskStatus[] {}; } else { this.taskStatus = new ReduceTaskStatus[] {}; } } this.setupOrCleanup = setupOrCleanup; this.taskTrackers = taskTrackers; }
@Override public void write(DataOutput out) throws IOException { taskID.write(out); out.writeDouble(progress); out.writeInt(runningAttempts); out.writeInt(killedAttempts); out.writeInt(failedAttempts); out.writeInt(taskStatus.length); for (TaskStatus t : taskStatus) { t.write(out); out.writeUTF(t.getTaskTracker()); } out.writeBoolean(setupOrCleanup); out.writeInt(taskTrackers.length); for (String tt : taskTrackers) { out.writeUTF(tt); } }
