private MapTaskCompletionEventsUpdate getMockedCompletionEventsUpdate( int startIdx, int numEvents) { ArrayList<TaskCompletionEvent> tceList = new ArrayList<TaskCompletionEvent>(numEvents); for (int i = 0; i < numEvents; ++i) { int eventIdx = startIdx + i; TaskCompletionEvent tce = new TaskCompletionEvent(eventIdx, new TaskAttemptID("12345", 1, TaskType.MAP, eventIdx, 0), eventIdx, true, TaskCompletionEvent.Status.SUCCEEDED, "http://somehost:8888"); tceList.add(tce); } TaskCompletionEvent[] events = {}; return new MapTaskCompletionEventsUpdate(tceList.toArray(events), false); }
public MapTaskCompletionEventsUpdate getMapTaskCompletionEventsUpdates( int index, JobID jobId, int max) throws IOException { throw new UnsupportedOperationException(); }
/** * Queries the {@link TaskTracker} for a set of map-completion events * from a given event ID. * @throws IOException */ private int getMapCompletionEvents() throws IOException { int numNewMaps = 0; MapTaskCompletionEventsUpdate update = umbilical.getMapCompletionEvents((org.apache.hadoop.mapred.JobID) reduce.getJobID(), fromEventId, MAX_EVENTS_TO_FETCH, (org.apache.hadoop.mapred.TaskAttemptID) reduce); TaskCompletionEvent events[] = update.getMapTaskCompletionEvents(); LOG.debug("Got " + events.length + " map completion events from " + fromEventId); // Check if the reset is required. // Since there is no ordering of the task completion events at the // reducer, the only option to sync with the new jobtracker is to reset // the events index if (update.shouldReset()) { fromEventId = 0; scheduler.resetKnownMaps(); } // Update the last seen event ID fromEventId += events.length; // Process the TaskCompletionEvents: // 1. Save the SUCCEEDED maps in knownOutputs to fetch the outputs. // 2. Save the OBSOLETE/FAILED/KILLED maps in obsoleteOutputs to stop // fetching from those maps. // 3. Remove TIPFAILED maps from neededOutputs since we don't need their // outputs at all. for (TaskCompletionEvent event : events) { switch (event.getTaskStatus()) { case SUCCEEDED: URI u = getBaseURI(event.getTaskTrackerHttp()); scheduler.addKnownMapOutput(u.getHost() + ":" + u.getPort(), u.toString(), event.getTaskAttemptId()); numNewMaps ++; int duration = event.getTaskRunTime(); if (duration > maxMapRuntime) { maxMapRuntime = duration; scheduler.informMaxMapRunTime(maxMapRuntime); } break; case FAILED: case KILLED: case OBSOLETE: scheduler.obsoleteMapOutput(event.getTaskAttemptId()); LOG.info("Ignoring obsolete output of " + event.getTaskStatus() + " map-task: '" + event.getTaskAttemptId() + "'"); break; case TIPFAILED: scheduler.tipFailed(event.getTaskAttemptId().getTaskID()); LOG.info("Ignoring output of failed map TIP: '" + event.getTaskAttemptId() + "'"); break; } } return numNewMaps; }
