@Test(timeout = 50000) public void testScanningOldDirs() throws Exception { LOG.info("STARTING testScanningOldDirs"); try { Configuration conf = new Configuration(); conf.setClass( NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, MyResolver.class, DNSToSwitchMapping.class); RackResolver.init(conf); MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(), true); app.submit(conf); Job job = app.getContext().getAllJobs().values().iterator().next(); JobId jobId = job.getID(); LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString()); app.waitForState(job, JobState.SUCCEEDED); // make sure all events are flushed app.waitForState(Service.STATE.STOPPED); HistoryFileManagerForTest hfm = new HistoryFileManagerForTest(); hfm.init(conf); HistoryFileInfo fileInfo = hfm.getFileInfo(jobId); Assert.assertNotNull("Unable to locate job history", fileInfo); // force the manager to "forget" the job hfm.deleteJobFromJobListCache(fileInfo); final int msecPerSleep = 10; int msecToSleep = 10 * 1000; while (fileInfo.isMovePending() && msecToSleep > 0) { Assert.assertTrue(!fileInfo.didMoveFail()); msecToSleep -= msecPerSleep; Thread.sleep(msecPerSleep); } Assert.assertTrue("Timeout waiting for history move", msecToSleep > 0); fileInfo = hfm.getFileInfo(jobId); hfm.stop(); Assert.assertNotNull("Unable to locate old job history", fileInfo); Assert.assertTrue("HistoryFileManager not shutdown properly", hfm.moveToDoneExecutor.isTerminated()); } finally { LOG.info("FINISHED testScanningOldDirs"); } }
/** * test clean old history files. Files should be deleted after 1 week by * default. */ @Test(timeout = 15000) public void testDeleteFileInfo() throws Exception { LOG.info("STARTING testDeleteFileInfo"); try { Configuration conf = new Configuration(); conf.setClass( NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, MyResolver.class, DNSToSwitchMapping.class); RackResolver.init(conf); MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(), true); app.submit(conf); Job job = app.getContext().getAllJobs().values().iterator().next(); JobId jobId = job.getID(); app.waitForState(job, JobState.SUCCEEDED); // make sure all events are flushed app.waitForState(Service.STATE.STOPPED); HistoryFileManager hfm = new HistoryFileManager(); hfm.init(conf); HistoryFileInfo fileInfo = hfm.getFileInfo(jobId); hfm.initExisting(); // wait for move files form the done_intermediate directory to the gone // directory while (fileInfo.isMovePending()) { Thread.sleep(300); } Assert.assertNotNull(hfm.jobListCache.values()); // try to remove fileInfo hfm.clean(); // check that fileInfo does not deleted Assert.assertFalse(fileInfo.isDeleted()); // correct live time hfm.setMaxHistoryAge(-1); hfm.clean(); hfm.stop(); Assert.assertTrue("Thread pool shutdown", hfm.moveToDoneExecutor.isTerminated()); // should be deleted ! Assert.assertTrue("file should be deleted ", fileInfo.isDeleted()); } finally { LOG.info("FINISHED testDeleteFileInfo"); } }
/** * Simple test some methods of JobHistory */ @Test(timeout = 20000) public void testJobHistoryMethods() throws Exception { LOG.info("STARTING testJobHistoryMethods"); try { Configuration configuration = new Configuration(); configuration .setClass( NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, MyResolver.class, DNSToSwitchMapping.class); RackResolver.init(configuration); MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(), true); app.submit(configuration); Job job = app.getContext().getAllJobs().values().iterator().next(); app.waitForState(job, JobState.SUCCEEDED); JobHistory jobHistory = new JobHistory(); jobHistory.init(configuration); // Method getAllJobs Assert.assertEquals(1, jobHistory.getAllJobs().size()); // and with ApplicationId Assert.assertEquals(1, jobHistory.getAllJobs(app.getAppID()).size()); JobsInfo jobsinfo = jobHistory.getPartialJobs(0L, 10L, null, "default", 0L, System.currentTimeMillis() + 1, 0L, System.currentTimeMillis() + 1, JobState.SUCCEEDED); Assert.assertEquals(1, jobsinfo.getJobs().size()); Assert.assertNotNull(jobHistory.getApplicationAttemptId()); // test Application Id Assert.assertEquals("application_0_0000", jobHistory.getApplicationID() .toString()); Assert .assertEquals("Job History Server", jobHistory.getApplicationName()); // method does not work Assert.assertNull(jobHistory.getEventHandler()); // method does not work Assert.assertNull(jobHistory.getClock()); // method does not work Assert.assertNull(jobHistory.getClusterInfo()); } finally { LOG.info("FINISHED testJobHistoryMethods"); } }
/** * Test clean old history files. Files should be deleted after 1 week by * default. */ @Test(timeout = 15000) public void testDeleteFileInfo() throws Exception { LOG.info("STARTING testDeleteFileInfo"); try { Configuration conf = new Configuration(); conf.setClass( NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, MyResolver.class, DNSToSwitchMapping.class); RackResolver.init(conf); MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(), true); app.submit(conf); Job job = app.getContext().getAllJobs().values().iterator().next(); JobId jobId = job.getID(); app.waitForState(job, JobState.SUCCEEDED); // make sure all events are flushed app.waitForState(Service.STATE.STOPPED); HistoryFileManager hfm = new HistoryFileManager(); hfm.init(conf); HistoryFileInfo fileInfo = hfm.getFileInfo(jobId); hfm.initExisting(); // wait for move files form the done_intermediate directory to the gone // directory while (fileInfo.isMovePending()) { Thread.sleep(300); } Assert.assertNotNull(hfm.jobListCache.values()); // try to remove fileInfo hfm.clean(); // check that fileInfo does not deleted Assert.assertFalse(fileInfo.isDeleted()); // correct live time hfm.setMaxHistoryAge(-1); hfm.clean(); hfm.stop(); Assert.assertTrue("Thread pool shutdown", hfm.moveToDoneExecutor.isTerminated()); // should be deleted ! Assert.assertTrue("file should be deleted ", fileInfo.isDeleted()); } finally { LOG.info("FINISHED testDeleteFileInfo"); } }
/** * Simple test some methods of JobHistory */ @Test(timeout = 20000) public void testJobHistoryMethods() throws Exception { LOG.info("STARTING testJobHistoryMethods"); try { Configuration configuration = new Configuration(); configuration .setClass( NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, MyResolver.class, DNSToSwitchMapping.class); RackResolver.init(configuration); MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(), true); app.submit(configuration); Job job = app.getContext().getAllJobs().values().iterator().next(); JobId jobId = job.getID(); LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString()); app.waitForState(job, JobState.SUCCEEDED); JobHistory jobHistory = new JobHistory(); jobHistory.init(configuration); // Method getAllJobs Assert.assertEquals(1, jobHistory.getAllJobs().size()); // and with ApplicationId Assert.assertEquals(1, jobHistory.getAllJobs(app.getAppID()).size()); JobsInfo jobsinfo = jobHistory.getPartialJobs(0L, 10L, null, "default", 0L, System.currentTimeMillis() + 1, 0L, System.currentTimeMillis() + 1, JobState.SUCCEEDED); Assert.assertEquals(1, jobsinfo.getJobs().size()); Assert.assertNotNull(jobHistory.getApplicationAttemptId()); // test Application Id Assert.assertEquals("application_0_0000", jobHistory.getApplicationID() .toString()); Assert .assertEquals("Job History Server", jobHistory.getApplicationName()); // method does not work Assert.assertNull(jobHistory.getEventHandler()); // method does not work Assert.assertNull(jobHistory.getClock()); // method does not work Assert.assertNull(jobHistory.getClusterInfo()); } finally { LOG.info("FINISHED testJobHistoryMethods"); } }
@Test(timeout = 50000) public void testScanningOldDirs() throws Exception { LOG.info("STARTING testScanningOldDirs"); try { Configuration conf = new Configuration(); conf.setClass( CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, MyResolver.class, DNSToSwitchMapping.class); RackResolver.init(conf); MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(), true); app.submit(conf); Job job = app.getContext().getAllJobs().values().iterator().next(); JobId jobId = job.getID(); LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString()); app.waitForState(job, JobState.SUCCEEDED); // make sure all events are flushed app.waitForState(Service.STATE.STOPPED); HistoryFileManagerForTest hfm = new HistoryFileManagerForTest(); hfm.init(conf); HistoryFileInfo fileInfo = hfm.getFileInfo(jobId); Assert.assertNotNull("Unable to locate job history", fileInfo); // force the manager to "forget" the job hfm.deleteJobFromJobListCache(fileInfo); final int msecPerSleep = 10; int msecToSleep = 10 * 1000; while (fileInfo.isMovePending() && msecToSleep > 0) { Assert.assertTrue(!fileInfo.didMoveFail()); msecToSleep -= msecPerSleep; Thread.sleep(msecPerSleep); } Assert.assertTrue("Timeout waiting for history move", msecToSleep > 0); fileInfo = hfm.getFileInfo(jobId); Assert.assertNotNull("Unable to locate old job history", fileInfo); } finally { LOG.info("FINISHED testScanningOldDirs"); } }
/** * test clean old history files. Files should be deleted after 1 week by * default. */ @Test(timeout = 15000) public void testDeleteFileInfo() throws Exception { LOG.info("STARTING testDeleteFileInfo"); try { Configuration conf = new Configuration(); conf.setClass( CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, MyResolver.class, DNSToSwitchMapping.class); RackResolver.init(conf); MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(), true); app.submit(conf); Job job = app.getContext().getAllJobs().values().iterator().next(); JobId jobId = job.getID(); app.waitForState(job, JobState.SUCCEEDED); // make sure all events are flushed app.waitForState(Service.STATE.STOPPED); HistoryFileManager hfm = new HistoryFileManager(); hfm.init(conf); HistoryFileInfo fileInfo = hfm.getFileInfo(jobId); hfm.initExisting(); // wait for move files form the done_intermediate directory to the gone // directory while (fileInfo.isMovePending()) { Thread.sleep(300); } Assert.assertNotNull(hfm.jobListCache.values()); // try to remove fileInfo hfm.clean(); // check that fileInfo does not deleted Assert.assertFalse(fileInfo.isDeleted()); // correct live time hfm.setMaxHistoryAge(-1); hfm.clean(); // should be deleted ! Assert.assertTrue("file should be deleted ", fileInfo.isDeleted()); } finally { LOG.info("FINISHED testDeleteFileInfo"); } }
/** * Simple test some methods of JobHistory */ @Test(timeout = 20000) public void testJobHistoryMethods() throws Exception { LOG.info("STARTING testJobHistoryMethods"); try { Configuration configuration = new Configuration(); configuration .setClass( CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, MyResolver.class, DNSToSwitchMapping.class); RackResolver.init(configuration); MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(), true); app.submit(configuration); Job job = app.getContext().getAllJobs().values().iterator().next(); app.waitForState(job, JobState.SUCCEEDED); JobHistory jobHistory = new JobHistory(); jobHistory.init(configuration); // Method getAllJobs Assert.assertEquals(1, jobHistory.getAllJobs().size()); // and with ApplicationId Assert.assertEquals(1, jobHistory.getAllJobs(app.getAppID()).size()); JobsInfo jobsinfo = jobHistory.getPartialJobs(0L, 10L, null, "default", 0L, System.currentTimeMillis() + 1, 0L, System.currentTimeMillis() + 1, JobState.SUCCEEDED); Assert.assertEquals(1, jobsinfo.getJobs().size()); Assert.assertNotNull(jobHistory.getApplicationAttemptId()); // test Application Id Assert.assertEquals("application_0_0000", jobHistory.getApplicationID() .toString()); Assert .assertEquals("Job History Server", jobHistory.getApplicationName()); // method does not work Assert.assertNull(jobHistory.getEventHandler()); // method does not work Assert.assertNull(jobHistory.getClock()); // method does not work Assert.assertNull(jobHistory.getClusterInfo()); } finally { LOG.info("FINISHED testJobHistoryMethods"); } }
@Test(timeout = 50000) public void testScanningOldDirs() throws Exception { LOG.info("STARTING testScanningOldDirs"); try { Configuration conf = new Configuration(); conf.setClass( CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, MyResolver.class, DNSToSwitchMapping.class); RackResolver.init(conf); MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(), true); app.submit(conf); Job job = app.getContext().getAllJobs().values().iterator().next(); JobId jobId = job.getID(); LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString()); app.waitForState(job, JobState.SUCCEEDED); // make sure all events are flushed app.waitForState(Service.STATE.STOPPED); HistoryFileManagerForTest hfm = new HistoryFileManagerForTest(); hfm.init(conf); HistoryFileInfo fileInfo = hfm.getFileInfo(jobId); Assert.assertNotNull("Unable to locate job history", fileInfo); // force the manager to "forget" the job hfm.deleteJobFromJobListCache(fileInfo); final int msecPerSleep = 10; int msecToSleep = 10 * 1000; while (fileInfo.isMovePending() && msecToSleep > 0) { Assert.assertTrue(!fileInfo.didMoveFail()); msecToSleep -= msecPerSleep; Thread.sleep(msecPerSleep); } Assert.assertTrue("Timeout waiting for history move", msecToSleep > 0); fileInfo = hfm.getFileInfo(jobId); hfm.stop(); Assert.assertNotNull("Unable to locate old job history", fileInfo); Assert.assertTrue("HistoryFileManager not shutdown properly", hfm.moveToDoneExecutor.isTerminated()); } finally { LOG.info("FINISHED testScanningOldDirs"); } }
/** * test clean old history files. Files should be deleted after 1 week by * default. */ @Test(timeout = 15000) public void testDeleteFileInfo() throws Exception { LOG.info("STARTING testDeleteFileInfo"); try { Configuration conf = new Configuration(); conf.setClass( CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, MyResolver.class, DNSToSwitchMapping.class); RackResolver.init(conf); MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(), true); app.submit(conf); Job job = app.getContext().getAllJobs().values().iterator().next(); JobId jobId = job.getID(); app.waitForState(job, JobState.SUCCEEDED); // make sure all events are flushed app.waitForState(Service.STATE.STOPPED); HistoryFileManager hfm = new HistoryFileManager(); hfm.init(conf); HistoryFileInfo fileInfo = hfm.getFileInfo(jobId); hfm.initExisting(); // wait for move files form the done_intermediate directory to the gone // directory while (fileInfo.isMovePending()) { Thread.sleep(300); } Assert.assertNotNull(hfm.jobListCache.values()); // try to remove fileInfo hfm.clean(); // check that fileInfo does not deleted Assert.assertFalse(fileInfo.isDeleted()); // correct live time hfm.setMaxHistoryAge(-1); hfm.clean(); hfm.stop(); Assert.assertTrue("Thread pool shutdown", hfm.moveToDoneExecutor.isTerminated()); // should be deleted ! Assert.assertTrue("file should be deleted ", fileInfo.isDeleted()); } finally { LOG.info("FINISHED testDeleteFileInfo"); } }
