private void initializeTracker() throws IOException { tracker.setIndexCache(new IndexCache(trackerFConf)); tracker.setTaskMemoryManagerEnabledFlag(); // for test case system FS is the local FS tracker.systemFS = FileSystem.getLocal(trackerFConf); tracker.systemDirectory = new Path(TEST_ROOT_DIR.getAbsolutePath()); tracker.setLocalFileSystem(tracker.systemFS); tracker.runningTasks = new LinkedHashMap<TaskAttemptID, TaskInProgress>(); tracker.runningJobs = new TreeMap<JobID, RunningJob>(); trackerFConf.deleteLocalFiles(TaskTracker.SUBDIR); // setup task controller taskController = getTaskController(); taskController.setConf(trackerFConf); taskController.setup(lDirAlloc, new LocalStorage(trackerFConf.getLocalDirs())); tracker.setTaskController(taskController); tracker.setLocalizer(new Localizer(tracker.getLocalFileSystem(),localDirs)); }
/** * Validates if task cleanup is done properly */ private void testTaskFilesRemoval(boolean needCleanup, boolean jvmReuse) throws Exception { // Localize job and localize task. TaskTracker.RunningJob rjob = tracker.localizeJob(tip); localizedJobConf = rjob.getJobConf(); if (jvmReuse) { localizedJobConf.setNumTasksToExecutePerJvm(2); } initializeTask(); // TODO: Let the task run and create files. // create files and set permissions 555. Verify if task controller sets // the permissions for TT to delete the task dir or work dir properly validateRemoveTaskFiles(needCleanup, jvmReuse, tip); }
/** * Localizes a cleanup task and validates permissions. * * @throws InterruptedException * @throws IOException */ public void testCleanupTaskLocalization() throws IOException, InterruptedException { if (!canRun()) { return; } task.setTaskCleanupTask(); // register task tip = tracker.new TaskInProgress(task, trackerFConf); // localize the job again. RunningJob rjob = tracker.localizeJob(tip); localizedJobConf = rjob.getJobConf(); checkJobLocalization(); // localize task cleanup attempt initializeTask(); checkTaskLocalization(); }
@Override RunningJob localizeJob(TaskInProgress tip) throws IOException, InterruptedException { if (firstJobStarted == false) { firstJobStarted = true; while (quitWaiting == false) { Thread.sleep(100); } firstJobFinished = true; } // mock out a RunningJob RunningJob rjob = mock(RunningJob.class); when(rjob.getLocalizedJobConf()).thenReturn(new Path("testing")); when(rjob.getJobConf()).thenReturn(new JobConf()); jobLocalizedCount++; return rjob; }
private void initializeTracker() throws IOException { tracker.setIndexCache(new IndexCache(trackerFConf)); tracker.setTaskMemoryManagerEnabledFlag(); // for test case system FS is the local FS tracker.systemFS = FileSystem.getLocal(trackerFConf); tracker.setLocalFileSystem(tracker.systemFS); tracker.systemDirectory = new Path(TEST_ROOT_DIR.getAbsolutePath()); tracker.runningTasks = new LinkedHashMap<TaskAttemptID, TaskInProgress>(); tracker.runningJobs = new TreeMap<JobID, RunningJob>(); tracker.setAsyncDiskService(new MRAsyncDiskService(trackerFConf)); tracker.getAsyncDiskService().cleanupAllVolumes(); // Set up TaskTracker instrumentation tracker.setTaskTrackerInstrumentation( TaskTracker.createInstrumentation(tracker, trackerFConf)); // setup task controller taskController = createTaskController(); taskController.setConf(trackerFConf); taskController.setup(); tracker.setTaskController(taskController); tracker.setLocalizer(new Localizer(tracker.getLocalFileSystem(), localDirs, taskController)); }
/** * Localizes a cleanup task and validates permissions. * * @throws InterruptedException * @throws IOException */ public void testCleanupTaskLocalization() throws IOException, InterruptedException { if (!canRun()) { return; } task.setTaskCleanupTask(); // register task tip = tracker.new TaskInProgress(task, trackerFConf); // localize the job. RunningJob rjob = tracker.localizeJob(tip); localizedJobConf = rjob.getJobConf(); checkJobLocalization(); // localize task cleanup attempt initializeTask(); checkTaskLocalization(); }
/** * Test job localization on a TT. Tests localization of job.xml, job.jar and * corresponding setting of configuration. Also test * {@link TaskController#initializeJob(JobInitializationContext)} * * @throws IOException */ public void testJobLocalization() throws Exception { if (!canRun()) { return; } TaskTracker.RunningJob rjob = tracker.localizeJob(tip); localizedJobConf = rjob.getJobConf(); checkJobLocalization(); }
/** * Test task localization on a TT. * * @throws IOException */ public void testTaskLocalization() throws Exception { if (!canRun()) { return; } TaskTracker.RunningJob rjob = tracker.localizeJob(tip); localizedJobConf = rjob.getJobConf(); initializeTask(); checkTaskLocalization(); }
private void initializeTask() throws IOException { tip.setJobConf(localizedJobConf); // ////////// The central method being tested tip.localizeTask(task); // ////////// // check the functionality of localizeTask for (String dir : trackerFConf.getStrings("mapred.local.dir")) { File attemptDir = new File(dir, TaskTracker.getLocalTaskDir(task.getUser(), jobId .toString(), taskId.toString(), task.isTaskCleanupTask())); assertTrue("attempt-dir " + attemptDir + " in localDir " + dir + " is not created!!", attemptDir.exists()); } attemptWorkDir = lDirAlloc.getLocalPathToRead(TaskTracker.getTaskWorkDir( task.getUser(), task.getJobID().toString(), task.getTaskID() .toString(), task.isTaskCleanupTask()), trackerFConf); assertTrue("atttempt work dir for " + taskId.toString() + " is not created in any of the configured dirs!!", attemptWorkDir != null); RunningJob rjob = new RunningJob(jobId); TaskController taskController = new DefaultTaskController(); taskController.setConf(trackerFConf); rjob.distCacheMgr = new TrackerDistributedCacheManager(trackerFConf, taskController). newTaskDistributedCacheManager(jobId, trackerFConf); TaskRunner runner = task.createRunner(tracker, tip, rjob); tip.setTaskRunner(runner); // /////// Few more methods being tested runner.setupChildTaskConfiguration(lDirAlloc); TaskRunner.createChildTmpDir(new File(attemptWorkDir.toUri().getPath()), localizedJobConf, true); attemptLogFiles = runner.prepareLogFiles(task.getTaskID(), task.isTaskCleanupTask()); // Make sure the task-conf file is created Path localTaskFile = lDirAlloc.getLocalPathToRead(TaskTracker.getTaskConfFile(task .getUser(), task.getJobID().toString(), task.getTaskID() .toString(), task.isTaskCleanupTask()), trackerFConf); assertTrue("Task conf file " + localTaskFile.toString() + " is not created!!", new File(localTaskFile.toUri().getPath()) .exists()); // /////// One more method being tested. This happens in child space. localizedTaskConf = new JobConf(localTaskFile); TaskRunner.setupChildMapredLocalDirs(task, localizedTaskConf); // /////// }
@Override protected void launchTaskForJob(TaskInProgress tip, JobConf jobConf, RunningJob rjob) throws IOException { jobLaunchCount++; }
@Test public void testTaskEnv() throws Throwable { ttConf.set("mapreduce.admin.user.shell", "/bin/testshell"); ttConf.set("mapreduce.admin.user.env", "key1=value1,key2=value2"); ttConf.set("mapred.child.env", "ROOT=$HOME"); final Map<String, String> env = new HashMap<String, String>(); String user = "test"; JobConf taskConf = new JobConf(ttConf); TaskAttemptID attemptID = new TaskAttemptID("test", 0, true, 0, 0); Task task = new MapTask(null, attemptID, 0, null, MAP_SLOTS); task.setConf(taskConf); TaskInProgress tip = tt.new TaskInProgress(task, taskConf); RunningJob rjob = new RunningJob(attemptID.getJobID()); TaskController taskController = new DefaultTaskController(); taskController.setConf(ttConf); rjob.distCacheMgr = new TrackerDistributedCacheManager(ttConf, taskController). newTaskDistributedCacheManager(attemptID.getJobID(), taskConf); final TaskRunner taskRunner = task.createRunner(tt, tip, rjob); String errorInfo = "Child error"; String mapredChildEnv = taskRunner.getChildEnv(taskConf); taskRunner.updateUserLoginEnv(errorInfo, user, taskConf, env); taskRunner.setEnvFromInputString(errorInfo, env, mapredChildEnv); final Vector<String> vargs = new Vector<String>(1); File pidFile = new File(TEST_DIR, "pid"); vargs.add(writeScript("ENV", "/bin/env ", pidFile).getAbsolutePath()); final File workDir = new File(TEST_DIR, "work"); workDir.mkdir(); final File stdout = new File(TEST_DIR, "stdout"); final File stderr = new File(TEST_DIR, "stderr"); Map<String, String> jvmenvmap = env; String javaOpts = taskRunner.getChildJavaOpts(ttConf, JobConf.MAPRED_MAP_TASK_JAVA_OPTS); assertTrue(jvmenvmap.containsKey("SHELL")); assertTrue(jvmenvmap.containsValue("/bin/testshell")); assertTrue(jvmenvmap.containsKey("key2")); assertTrue(jvmenvmap.containsValue("value2")); assertTrue(javaOpts, javaOpts.contains("Xmx")); assertTrue(javaOpts, javaOpts.contains("IPv4")); String root = jvmenvmap.get("ROOT"); assertTrue(root, root.contentEquals(System.getenv("HOME"))); }
