@Override public void setup(LocalDirAllocator allocator, LocalStorage localStorage) throws IOException { // Check the permissions of the task-controller binary by running // it plainly. If permissions are correct, it returns an error // code 1, else it returns 24 or something else if some other bugs // are also present. String[] taskControllerCmd = new String[] { taskControllerExe }; ShellCommandExecutor shExec = new ShellCommandExecutor(taskControllerCmd); try { shExec.execute(); } catch (ExitCodeException e) { int exitCode = shExec.getExitCode(); if (exitCode != 1) { LOG.warn("Exit code from checking binary permissions is : " + exitCode); logOutput(shExec.getOutput()); throw new IOException("Task controller setup failed because of invalid" + "permissions/ownership with exit code " + exitCode, e); } } this.allocator = allocator; this.localStorage = localStorage; }
private void initializeTracker() throws IOException { tracker.setIndexCache(new IndexCache(trackerFConf)); tracker.setTaskMemoryManagerEnabledFlag(); // for test case system FS is the local FS tracker.systemFS = FileSystem.getLocal(trackerFConf); tracker.systemDirectory = new Path(TEST_ROOT_DIR.getAbsolutePath()); tracker.setLocalFileSystem(tracker.systemFS); tracker.runningTasks = new LinkedHashMap<TaskAttemptID, TaskInProgress>(); tracker.runningJobs = new TreeMap<JobID, RunningJob>(); trackerFConf.deleteLocalFiles(TaskTracker.SUBDIR); // setup task controller taskController = getTaskController(); taskController.setConf(trackerFConf); taskController.setup(lDirAlloc, new LocalStorage(trackerFConf.getLocalDirs())); tracker.setTaskController(taskController); tracker.setLocalizer(new Localizer(tracker.getLocalFileSystem(),localDirs)); }
private TaskTracker setupTaskTracker(Configuration conf) throws Exception { JobConf ttConf = new JobConf(conf); // Doesn't matter what we give here - we won't actually // connect to it. TaskTracker tt = new TaskTracker(); tt.setConf(ttConf); tt.setTaskController(Mockito.mock(TaskController.class)); LocalDirAllocator localDirAllocator = new LocalDirAllocator("mapred.local.dir"); tt.setLocalDirAllocator(localDirAllocator); LocalFileSystem localFs = FileSystem.getLocal(conf); LocalStorage localStorage = new LocalStorage(ttConf.getLocalDirs()); localStorage.checkDirs(localFs, true); tt.setLocalStorage(localStorage); tt.setLocalFileSystem(localFs); tt.initializeDirectories(); return tt; }
public TestJvmManager() throws Exception { user = UserGroupInformation.getCurrentUser().getShortUserName(); tt = new TaskTracker(); ttConf = new JobConf(); ttConf.setLong("mapred.tasktracker.tasks.sleeptime-before-sigkill", 2000); tt.setConf(ttConf); tt.setMaxMapSlots(MAP_SLOTS); tt.setMaxReduceSlots(REDUCE_SLOTS); TaskController dtc; tt.setTaskController((dtc = new DefaultTaskController())); Configuration conf = new Configuration(); dtc.setConf(conf); LocalDirAllocator ldirAlloc = new LocalDirAllocator(JobConf.MAPRED_LOCAL_DIR_PROPERTY); tt.getTaskController().setup(ldirAlloc, new LocalStorage(ttConf.getLocalDirs())); JobID jobId = new JobID("test", 0); jvmManager = new JvmManager(tt); tt.setJvmManagerInstance(jvmManager); tt.setUserLogManager(new UserLogManager(ttConf)); tt.setCleanupThread(new InlineCleanupQueue()); }
private void startTT(JobConf conf) throws IOException, InterruptedException { myClock = new FakeClock(); // clock is reset. String localdirs = TEST_ROOT_DIR + "/userlogs/local/0," + TEST_ROOT_DIR + "/userlogs/local/1"; conf.set(JobConf.MAPRED_LOCAL_DIR_PROPERTY, localdirs); tt = new TaskTracker(); tt.setConf(new JobConf(conf)); LocalDirAllocator localDirAllocator = new LocalDirAllocator("mapred.local.dir"); tt.setLocalDirAllocator(localDirAllocator); LocalStorage localStorage = new LocalStorage(conf.getLocalDirs()); LocalFileSystem localFs = FileSystem.getLocal(conf); localStorage.checkDirs(localFs, true); tt.setLocalStorage(localStorage); localizer = new Localizer(FileSystem.get(conf), conf .getTrimmedStrings(JobConf.MAPRED_LOCAL_DIR_PROPERTY)); tt.setLocalizer(localizer); userLogManager = new UtilsForTests.InLineUserLogManager(conf); TaskController taskController = userLogManager.getTaskController(); taskController.setup(localDirAllocator, localStorage); tt.setTaskController(taskController); userLogCleaner = userLogManager.getUserLogCleaner(); userLogCleaner.setClock(myClock); tt.setUserLogManager(userLogManager); userLogManager.clearOldUserLogs(conf); }
@Override public void setup(LocalDirAllocator allocator, LocalStorage localStorage) throws IOException { // Check the permissions of the task-controller binary by running it plainly. // If permissions are correct, it returns an error code 1, else it returns // 24 or something else if some other bugs are also present. String[] taskControllerCmd = new String[] { taskControllerExe }; ShellCommandExecutor shExec = new ShellCommandExecutor(taskControllerCmd); try { shExec.execute(); } catch (ExitCodeException e) { int exitCode = shExec.getExitCode(); if (exitCode != 1) { LOG.warn("Exit code from checking binary permissions is : " + exitCode); logOutput(shExec.getOutput()); throw new IOException("Task controller setup failed because of invalid" + "permissions/ownership with exit code " + exitCode, e); } } this.allocator = allocator; this.localStorage = localStorage; }
protected void initMyTest() throws Exception { testDir.mkdirs(); mapredLocal.mkdirs(); createFakeTCScript(); conf.set(JobConf.MAPRED_LOCAL_DIR_PROPERTY, mapredLocal.toString()); // Set the task-controller binary path. conf.set("mapreduce.tasktracker.task-controller.exe", fakeTaskController.toString()); ltc = new LinuxTaskController(); ltc.setConf(conf); // LinuxTaskController runs task-controller in setup() with no // argument and expects 1 in return try { ltc.setup(new LocalDirAllocator(mapredLocal.toString()), new LocalStorage(new String[]{mapredLocal.toString()})); } catch (IOException ie) { fail("Error running task-controller from setup()."); } initialized = true; }
private void validateTaskControllerSetup(TaskController controller, boolean shouldFail) throws IOException { if (shouldFail) { // task controller setup should fail validating permissions. Throwable th = null; try { controller.setup( new LocalDirAllocator(JobConf.MAPRED_LOCAL_DIR_PROPERTY), new LocalStorage(controller.getConf().getStrings( JobConf.MAPRED_LOCAL_DIR_PROPERTY))); } catch (IOException ie) { th = ie; } assertNotNull("No exception during setup", th); assertTrue("Exception message does not contain exit code" + INVALID_TASKCONTROLLER_PERMISSIONS, th.getMessage().contains( "with exit code " + INVALID_TASKCONTROLLER_PERMISSIONS)); } else { controller.setup(new LocalDirAllocator(JobConf.MAPRED_LOCAL_DIR_PROPERTY), new LocalStorage(controller.getConf().getStrings( JobConf.MAPRED_LOCAL_DIR_PROPERTY))); } }
private void startTT(JobConf conf) throws IOException, InterruptedException { myClock = new FakeClock(); // clock is reset. String localdirs = TEST_ROOT_DIR + "/userlogs/local/0," + TEST_ROOT_DIR + "/userlogs/local/1"; conf.set(JobConf.MAPRED_LOCAL_DIR_PROPERTY, localdirs); tt = new TaskTracker(); tt.setConf(new JobConf(conf)); LocalDirAllocator localDirAllocator = new LocalDirAllocator("mapred.local.dir"); localizer = new Localizer(FileSystem.get(conf), conf .getStrings(JobConf.MAPRED_LOCAL_DIR_PROPERTY)); tt.setLocalizer(localizer); userLogManager = new UtilsForTests.InLineUserLogManager(conf); TaskController taskController = userLogManager.getTaskController(); taskController.setup(localDirAllocator, new LocalStorage(conf.getStrings(JobConf.MAPRED_LOCAL_DIR_PROPERTY))); tt.setTaskController(taskController); userLogCleaner = userLogManager.getUserLogCleaner(); userLogCleaner.setClock(myClock); tt.setUserLogManager(userLogManager); userLogManager.clearOldUserLogs(conf); }
private void validateTaskControllerSetup(Configuration conf, TaskController controller, boolean shouldFail, int exitCode) throws IOException { File confFile = ClusterWithLinuxTaskController .createTaskControllerConf(taskControllerPath, conf); execCommand(confFile, "sudo", Shell.SET_OWNER_COMMAND, "root"); if (shouldFail) { // task controller setup should fail validating permissions. Throwable th = null; try { controller.setup( new LocalDirAllocator(JobConf.MAPRED_LOCAL_DIR_PROPERTY), new LocalStorage(controller.getConf().getStrings( JobConf.MAPRED_LOCAL_DIR_PROPERTY))); } catch (IOException ie) { th = ie; } assertNotNull("No exception during setup", th); assertTrue("Exception message does not contain exit code " + exitCode, th.getMessage().contains( "with exit code " + exitCode)); } else { controller.setup(new LocalDirAllocator(JobConf.MAPRED_LOCAL_DIR_PROPERTY), new LocalStorage(controller.getConf().getStrings( JobConf.MAPRED_LOCAL_DIR_PROPERTY))); } execCommand(confFile, "sudo", "rm"); }
@Override public void setup(LocalDirAllocator allocator, LocalStorage localStorage) throws IOException { // get the current ugi and set the task controller group owner getConf().set(TT_GROUP, taskTrackerSpecialGroup); // write configuration file configurationFile = createTaskControllerConf(System .getProperty(TASKCONTROLLER_PATH), getConf()); super.setup(allocator, localStorage); }
@Override public void setup(LocalDirAllocator allocator, LocalStorage localStorage) { this.allocator = allocator; this.localStorage = localStorage; }
