private void initializeTracker() throws IOException { tracker.setIndexCache(new IndexCache(trackerFConf)); tracker.setTaskMemoryManagerEnabledFlag(); // for test case system FS is the local FS tracker.systemFS = FileSystem.getLocal(trackerFConf); tracker.systemDirectory = new Path(TEST_ROOT_DIR.getAbsolutePath()); tracker.setLocalFileSystem(tracker.systemFS); tracker.runningTasks = new LinkedHashMap<TaskAttemptID, TaskInProgress>(); tracker.runningJobs = new TreeMap<JobID, RunningJob>(); trackerFConf.deleteLocalFiles(TaskTracker.SUBDIR); // setup task controller taskController = getTaskController(); taskController.setConf(trackerFConf); taskController.setup(lDirAlloc, new LocalStorage(trackerFConf.getLocalDirs())); tracker.setTaskController(taskController); tracker.setLocalizer(new Localizer(tracker.getLocalFileSystem(),localDirs)); }
private void startTT(JobConf conf) throws IOException, InterruptedException { myClock = new FakeClock(); // clock is reset. String localdirs = TEST_ROOT_DIR + "/userlogs/local/0," + TEST_ROOT_DIR + "/userlogs/local/1"; conf.set(JobConf.MAPRED_LOCAL_DIR_PROPERTY, localdirs); tt = new TaskTracker(); tt.setConf(new JobConf(conf)); LocalDirAllocator localDirAllocator = new LocalDirAllocator("mapred.local.dir"); tt.setLocalDirAllocator(localDirAllocator); LocalStorage localStorage = new LocalStorage(conf.getLocalDirs()); LocalFileSystem localFs = FileSystem.getLocal(conf); localStorage.checkDirs(localFs, true); tt.setLocalStorage(localStorage); localizer = new Localizer(FileSystem.get(conf), conf .getTrimmedStrings(JobConf.MAPRED_LOCAL_DIR_PROPERTY)); tt.setLocalizer(localizer); userLogManager = new UtilsForTests.InLineUserLogManager(conf); TaskController taskController = userLogManager.getTaskController(); taskController.setup(localDirAllocator, localStorage); tt.setTaskController(taskController); userLogCleaner = userLogManager.getUserLogCleaner(); userLogCleaner.setClock(myClock); tt.setUserLogManager(userLogManager); userLogManager.clearOldUserLogs(conf); }
private void startTT(JobConf conf) throws IOException, InterruptedException { myClock = new FakeClock(); // clock is reset. String localdirs = TEST_ROOT_DIR + "/userlogs/local/0," + TEST_ROOT_DIR + "/userlogs/local/1"; conf.set(JobConf.MAPRED_LOCAL_DIR_PROPERTY, localdirs); tt = new TaskTracker(); tt.setConf(new JobConf(conf)); LocalDirAllocator localDirAllocator = new LocalDirAllocator("mapred.local.dir"); localizer = new Localizer(FileSystem.get(conf), conf .getStrings(JobConf.MAPRED_LOCAL_DIR_PROPERTY)); tt.setLocalizer(localizer); userLogManager = new UtilsForTests.InLineUserLogManager(conf); TaskController taskController = userLogManager.getTaskController(); taskController.setup(localDirAllocator, new LocalStorage(conf.getStrings(JobConf.MAPRED_LOCAL_DIR_PROPERTY))); tt.setTaskController(taskController); userLogCleaner = userLogManager.getUserLogCleaner(); userLogCleaner.setClock(myClock); tt.setUserLogManager(userLogManager); userLogManager.clearOldUserLogs(conf); }
private void initializeTracker() throws IOException { tracker.setIndexCache(new IndexCache(trackerFConf)); tracker.setTaskMemoryManagerEnabledFlag(); // for test case system FS is the local FS tracker.systemFS = FileSystem.getLocal(trackerFConf); tracker.setLocalFileSystem(tracker.systemFS); tracker.systemDirectory = new Path(TEST_ROOT_DIR.getAbsolutePath()); tracker.runningTasks = new LinkedHashMap<TaskAttemptID, TaskInProgress>(); tracker.runningJobs = new TreeMap<JobID, RunningJob>(); tracker.setAsyncDiskService(new MRAsyncDiskService(trackerFConf)); tracker.getAsyncDiskService().cleanupAllVolumes(); // Set up TaskTracker instrumentation tracker.setTaskTrackerInstrumentation( TaskTracker.createInstrumentation(tracker, trackerFConf)); // setup task controller taskController = createTaskController(); taskController.setConf(trackerFConf); taskController.setup(); tracker.setTaskController(taskController); tracker.setLocalizer(new Localizer(tracker.getLocalFileSystem(), localDirs, taskController)); }
public TestUserLogCleanup() throws IOException { Configuration conf = new Configuration(); localizer = new Localizer(FileSystem.get(conf), conf .getStrings(MRConfig.LOCAL_DIR), new DefaultTaskController()); taskLogCleanupThread = new UserLogCleaner(conf); taskLogCleanupThread.setClock(myClock); tt = new TaskTracker(); tt.setConf(new JobConf(conf)); tt.setLocalizer(localizer); tt.setTaskLogCleanupThread(taskLogCleanupThread); }
/** * Create all of the directories for the task and launches the child jvm. * @param user the user name * @param attemptId the attempt id * @throws IOException */ @Override public int launchTask(String user, String jobId, String attemptId, List<String> setup, List<String> jvmArguments, File currentWorkDirectory, String stdout, String stderr) throws IOException { ShellCommandExecutor shExec = null; try { FileSystem localFs = FileSystem.getLocal(getConf()); //create the attempt dirs new Localizer(localFs, getConf().getStrings(JobConf.MAPRED_LOCAL_DIR_PROPERTY)). initializeAttemptDirs(user, jobId, attemptId); // create the working-directory of the task if (!currentWorkDirectory.mkdir()) { throw new IOException("Mkdirs failed to create " + currentWorkDirectory.toString()); } //mkdir the loglocation String logLocation = TaskLog.getAttemptDir(jobId, attemptId).toString(); if (!localFs.mkdirs(new Path(logLocation))) { throw new IOException("Mkdirs failed to create " + logLocation); } //read the configuration for the job FileSystem rawFs = FileSystem.getLocal(getConf()).getRaw(); long logSize = 0; //TODO MAPREDUCE-1100 // get the JVM command line. String cmdLine = TaskLog.buildCommandLine(setup, jvmArguments, new File(stdout), new File(stderr), logSize, true); // write the command to a file in the // task specific cache directory // TODO copy to user dir Path p = new Path(allocator.getLocalPathForWrite( TaskTracker.getPrivateDirTaskScriptLocation(user, jobId, attemptId), getConf()), COMMAND_FILE); String commandFile = writeCommand(cmdLine, rawFs, p); rawFs.setPermission(p, TaskController.TASK_LAUNCH_SCRIPT_PERMISSION); shExec = new ShellCommandExecutor(new String[]{ "bash", "-c", commandFile}, currentWorkDirectory); shExec.execute(); } catch (Exception e) { if (shExec == null) { return -1; } int exitCode = shExec.getExitCode(); LOG.warn("Exit code from task is : " + exitCode); LOG.info("Output from DefaultTaskController's launchTask follows:"); logOutput(shExec.getOutput()); return exitCode; } return 0; }
Localizer getLocalizer() { return localizer; }
void setLocalizer(Localizer l) { localizer = l; }
