/** Test that volumes specified as relative paths are handled properly * by MRAsyncDiskService (MAPREDUCE-1887). */ public void testVolumeNormalization() throws Throwable { LOG.info("TEST_ROOT_DIR is " + TEST_ROOT_DIR); String relativeTestRoot = relativeToWorking(TEST_ROOT_DIR); FileSystem localFileSystem = FileSystem.getLocal(new Configuration()); String [] vols = new String[] { relativeTestRoot + "/0", relativeTestRoot + "/1" }; // Put a file in one of the volumes to be cleared on startup. Path delDir = new Path(vols[0], MRAsyncDiskService.TOBEDELETED); localFileSystem.mkdirs(delDir); localFileSystem.create(new Path(delDir, "foo")).close(); MRAsyncDiskService service = new MRAsyncDiskService( localFileSystem, vols); makeSureCleanedUp(vols, service); }
private void makeSureCleanedUp(String[] vols, MRAsyncDiskService service) throws Throwable { // Sleep at most 5 seconds to make sure the deleted items are all gone. service.shutdown(); if (!service.awaitTermination(5000)) { fail("MRAsyncDiskService is still not shutdown in 5 seconds!"); } // All contents should be gone by now. for (int i = 0; i < vols.length; i++) { File subDir = new File(vols[i]); String[] subDirContent = subDir.list(); // 0.20 version of MRAsyncDiskService deletes toBeDeleted/ after // shutdown, so we should not see any children in any volume. assertEquals("Volume should contain no children: " + MRAsyncDiskService.TOBEDELETED, 0, subDirContent.length); } }
/** * Creates a TrackerDistributedCacheManager with a MRAsyncDiskService. * @param asyncDiskService Provides a set of ThreadPools for async disk * operations. */ public TrackerDistributedCacheManager(Configuration conf, TaskController taskController, MRAsyncDiskService asyncDiskService) throws IOException { this(conf, taskController); this.asyncDiskService = asyncDiskService; }
/** * This test creates some directories inside the volume roots, and then * call asyncDiskService.cleanupAllVolumes. * We should be able to delete all files/dirs inside the volumes except * the toBeDeleted directory. */ public void testMRAsyncDiskServiceCleanupAllVolumes() throws Throwable { FileSystem localFileSystem = FileSystem.getLocal(new Configuration()); String[] vols = new String[]{TEST_ROOT_DIR + "/0", TEST_ROOT_DIR + "/1"}; MRAsyncDiskService service = new MRAsyncDiskService( localFileSystem, vols); String a = "a"; String b = "b"; String c = "b/c"; String d = "d"; File fa = new File(vols[0], a); File fb = new File(vols[1], b); File fc = new File(vols[1], c); File fd = new File(vols[1], d); // Create the directories fa.mkdirs(); fb.mkdirs(); fc.mkdirs(); fd.mkdirs(); assertTrue(fa.exists()); assertTrue(fb.exists()); assertTrue(fc.exists()); assertTrue(fd.exists()); // Delete all of them service.cleanupAllVolumes(); assertFalse(fa.exists()); assertFalse(fb.exists()); assertFalse(fc.exists()); assertFalse(fd.exists()); // Make sure everything is cleaned up makeSureCleanedUp(vols, service); }
/** * This test creates some directories inside the volume roots, and then call * asyncDiskService.cleanupAllVolumes. We should be able to delete only those * files/dirs that have been specified. */ public void testCleanupDirsInAllVolumes() throws Throwable { FileSystem localFileSystem = FileSystem.getLocal(new Configuration()); String[] vols = new String[] { TEST_ROOT_DIR + "/0", TEST_ROOT_DIR + "/1" }; MRAsyncDiskService service = new MRAsyncDiskService(localFileSystem, vols); String a = "a"; String b = "b"; File fa0 = new File(vols[0], a); File fa1 = new File(vols[1], a); File fb0 = new File(vols[0], b); File fb1 = new File(vols[1], b); // Create the directories fa0.mkdirs(); fa1.mkdirs(); fb0.mkdirs(); fb1.mkdirs(); assertTrue(fa0.exists()); assertTrue(fa1.exists()); assertTrue(fb0.exists()); assertTrue(fb1.exists()); // Delete all of them service.cleanupDirsInAllVolumes(new String[] { a }); assertFalse(fa0.exists()); assertFalse(fa1.exists()); assertTrue(fb0.exists()); assertTrue(fb1.exists()); }
/** * This test creates some directories inside the toBeDeleted directory and * then start the asyncDiskService. * AsyncDiskService will create tasks to delete the content inside the * toBeDeleted directories. */ public void testMRAsyncDiskServiceStartupCleaning() throws Throwable { FileSystem localFileSystem = FileSystem.getLocal(new Configuration()); String[] vols = new String[]{TEST_ROOT_DIR + "/0", TEST_ROOT_DIR + "/1"}; String a = "a"; String b = "b"; String c = "b/c"; String d = "d"; // Create directories inside SUBDIR String suffix = Path.SEPARATOR_CHAR + MRAsyncDiskService.TOBEDELETED; File fa = new File(vols[0] + suffix, a); File fb = new File(vols[1] + suffix, b); File fc = new File(vols[1] + suffix, c); File fd = new File(vols[1] + suffix, d); // Create the directories fa.mkdirs(); fb.mkdirs(); fc.mkdirs(); fd.mkdirs(); assertTrue(fa.exists()); assertTrue(fb.exists()); assertTrue(fc.exists()); assertTrue(fd.exists()); // Create the asyncDiskService which will delete all contents inside SUBDIR MRAsyncDiskService service = new MRAsyncDiskService( localFileSystem, vols); // Make sure everything is cleaned up makeSureCleanedUp(vols, service); }
public void testToleratesSomeUnwritableVolumes() throws Throwable { FileSystem localFileSystem = FileSystem.getLocal(new Configuration()); String[] vols = new String[]{TEST_ROOT_DIR + "/0", TEST_ROOT_DIR + "/1"}; assertTrue(new File(vols[0]).mkdirs()); assertEquals(0, FileUtil.chmod(vols[0], "400")); // read only try { new MRAsyncDiskService(localFileSystem, vols); } finally { FileUtil.chmod(vols[0], "755"); // make writable again } }
/** * This test creates some directories inside the volume roots, and then * call asyncDiskService.MoveAndDeleteAllVolumes. * We should be able to delete all files/dirs inside the volumes except * the toBeDeleted directory. */ public void testMRAsyncDiskServiceMoveAndDeleteAllVolumes() throws Throwable { FileSystem localFileSystem = FileSystem.getLocal(new Configuration()); String[] vols = new String[]{TEST_ROOT_DIR + "/0", TEST_ROOT_DIR + "/1"}; MRAsyncDiskService service = new MRAsyncDiskService( localFileSystem, vols); String a = "a"; String b = "b"; String c = "b/c"; String d = "d"; File fa = new File(vols[0], a); File fb = new File(vols[1], b); File fc = new File(vols[1], c); File fd = new File(vols[1], d); // Create the directories fa.mkdirs(); fb.mkdirs(); fc.mkdirs(); fd.mkdirs(); assertTrue(fa.exists()); assertTrue(fb.exists()); assertTrue(fc.exists()); assertTrue(fd.exists()); // Delete all of them service.cleanupAllVolumes(); assertFalse(fa.exists()); assertFalse(fb.exists()); assertFalse(fc.exists()); assertFalse(fd.exists()); // Make sure everything is cleaned up makeSureCleanedUp(vols, service); }
/** * This test creates some directories inside the toBeDeleted directory and * then start the asyncDiskService. * AsyncDiskService will create tasks to delete the content inside the * toBeDeleted directories. */ public void testMRAsyncDiskServiceStartupCleaning() throws Throwable { FileSystem localFileSystem = FileSystem.getLocal(new Configuration()); String[] vols = new String[]{TEST_ROOT_DIR + "/0", TEST_ROOT_DIR + "/1"}; String a = "a"; String b = "b"; String c = "b/c"; String d = "d"; // Create directories inside SUBDIR File fa = new File(vols[0] + Path.SEPARATOR_CHAR + MRAsyncDiskService.TOBEDELETED, a); File fb = new File(vols[1] + Path.SEPARATOR_CHAR + MRAsyncDiskService.TOBEDELETED, b); File fc = new File(vols[1] + Path.SEPARATOR_CHAR + MRAsyncDiskService.TOBEDELETED, c); File fd = new File(vols[1] + Path.SEPARATOR_CHAR + MRAsyncDiskService.TOBEDELETED, d); // Create the directories fa.mkdirs(); fb.mkdirs(); fc.mkdirs(); fd.mkdirs(); assertTrue(fa.exists()); assertTrue(fb.exists()); assertTrue(fc.exists()); assertTrue(fd.exists()); // Create the asyncDiskService which will delete all contents inside SUBDIR MRAsyncDiskService service = new MRAsyncDiskService( localFileSystem, vols); // Make sure everything is cleaned up makeSureCleanedUp(vols, service); }
private void makeSureCleanedUp(String[] vols, MRAsyncDiskService service) throws Throwable { // Sleep at most 5 seconds to make sure the deleted items are all gone. service.shutdown(); if (!service.awaitTermination(5000)) { fail("MRAsyncDiskService is still not shutdown in 5 seconds!"); } // All contents should be gone by now. for (int i = 0; i < vols.length; i++) { File subDir = new File(vols[0]); String[] subDirContent = subDir.list(); assertEquals("Volume should be cleaned up.", 0, subDirContent.length); } }
/** * This test creates some directories and then removes them through * MRAsyncDiskService. */ public void testMRAsyncDiskService() throws Throwable { FileSystem localFileSystem = FileSystem.getLocal(new Configuration()); String[] vols = new String[]{TEST_ROOT_DIR + "/0", TEST_ROOT_DIR + "/1"}; MRAsyncDiskService service = new MRAsyncDiskService( localFileSystem, vols); String a = "a"; String b = "b"; String c = "b/c"; String d = "d"; File fa = new File(vols[0], a); File fb = new File(vols[1], b); File fc = new File(vols[1], c); File fd = new File(vols[1], d); // Create the directories fa.mkdirs(); fb.mkdirs(); fc.mkdirs(); fd.mkdirs(); assertTrue(fa.exists()); assertTrue(fb.exists()); assertTrue(fc.exists()); assertTrue(fd.exists()); // Move and delete them service.moveAndDeleteRelativePath(vols[0], a); assertFalse(fa.exists()); service.moveAndDeleteRelativePath(vols[1], b); assertFalse(fb.exists()); assertFalse(fc.exists()); assertFalse(service.moveAndDeleteRelativePath(vols[1], "not_exists")); // asyncDiskService is NOT able to delete files outside all volumes. IOException ee = null; try { service.moveAndDeleteAbsolutePath(TEST_ROOT_DIR + "/2"); } catch (IOException e) { ee = e; } assertNotNull("asyncDiskService should not be able to delete files " + "outside all volumes", ee); // asyncDiskService is able to automatically find the file in one // of the volumes. assertTrue(service.moveAndDeleteAbsolutePath(vols[1] + Path.SEPARATOR_CHAR + d)); // Make sure everything is cleaned up makeSureCleanedUp(vols, service); }
MapOutputFile(JobID jobId, MRAsyncDiskService asyncDiskService) { this.jobId = jobId; this.asyncDiskService = asyncDiskService; }
/** * Returns the MRAsyncDiskService object for async deletions. */ public MRAsyncDiskService getAsyncDiskService() { return asyncDiskService; }
