private Path createPath(String path, boolean checkWrite) throws IOException { Path file = new Path(new Path(localDirs[dirNumLastAccessed]), path); if (checkWrite) { //check whether we are able to create a directory here. If the disk //happens to be RDONLY we will fail try { DiskChecker.checkDir(new File(file.getParent().toUri().getPath())); return file; } catch (DiskErrorException d) { LOG.warn("Disk Error Exception: ", d); return null; } } return file; }
/** Get a path from the local FS for reading. We search through all the * configured dirs for the file's existence and return the complete * path to the file when we find one */ public synchronized Path getLocalPathToRead(String pathStr, Configuration conf) throws IOException { confChanged(conf); int numDirs = localDirs.length; int numDirsSearched = 0; //remove the leading slash from the path (to make sure that the uri //resolution results in a valid path on the dir being checked) if (pathStr.startsWith("/")) { pathStr = pathStr.substring(1); } while (numDirsSearched < numDirs) { Path file = new Path(localDirs[numDirsSearched], pathStr); if (localFS.exists(file)) { return file; } numDirsSearched++; } //no path found throw new DiskErrorException ("Could not find " + pathStr +" in any of" + " the configured local directories"); }
private void _mkdirs(boolean exists, FsPermission before, FsPermission after) throws Throwable { File localDir = make(stub(File.class).returning(exists).from.exists()); when(localDir.mkdir()).thenReturn(true); Path dir = mock(Path.class); // use default stubs LocalFileSystem fs = make(stub(LocalFileSystem.class) .returning(localDir).from.pathToFile(dir)); FileStatus stat = make(stub(FileStatus.class) .returning(after).from.getPermission()); when(fs.getFileStatus(dir)).thenReturn(stat); try { DiskChecker.mkdirsWithExistsAndPermissionCheck(fs, dir, before); if (!exists) verify(fs).setPermission(dir, before); else { verify(fs).getFileStatus(dir); verify(stat).getPermission(); } } catch (DiskErrorException e) { if (before != after) assertTrue(e.getMessage().startsWith("Incorrect permission")); } }
private void _checkDirs(boolean isDir, FsPermission perm, boolean success) throws Throwable { File localDir = File.createTempFile("test", "tmp"); if (isDir) { localDir.delete(); localDir.mkdir(); } Shell.execCommand(Shell.getSetPermissionCommand(String.format("%04o", perm.toShort()), false, localDir.getAbsolutePath())); try { DiskChecker.checkDir(FileSystem.getLocal(new Configuration()), new Path(localDir.getAbsolutePath()), perm); assertTrue("checkDir success", success); } catch (DiskErrorException e) { assertFalse("checkDir success", success); } localDir.delete(); }
private void _checkDirs(boolean isDir, String perm, boolean success) throws Throwable { File localDir = File.createTempFile("test", "tmp"); if (isDir) { localDir.delete(); localDir.mkdir(); } Shell.execCommand(Shell.getSetPermissionCommand(perm, false, localDir.getAbsolutePath())); try { DiskChecker.checkDir(localDir); assertTrue("checkDir success", success); } catch (DiskErrorException e) { e.printStackTrace(); assertFalse("checkDir success", success); } localDir.delete(); System.out.println("checkDir success: " + success); }
@Test (timeout = 30000) public void testCheckDirsIOException() throws Throwable { Path path = new Path("target", TestDiskChecker.class.getSimpleName()); File localDir = new File(path.toUri().getRawPath()); localDir.mkdir(); File localFile = new File(localDir, "test"); localFile.createNewFile(); File spyLocalDir = spy(localDir); doReturn(localFile.toPath()).when(spyLocalDir).toPath(); try { DiskChecker.checkDirs(spyLocalDir); fail("Expected exception for I/O error"); } catch (DiskErrorException e) { GenericTestUtils.assertExceptionContains("I/O error", e); assertTrue(e.getCause() instanceof IOException); } finally { localFile.delete(); localDir.delete(); } }
/** * Check if the given local directories * (and parent directories, if necessary) can be created. * @param localDirs where the new TaskTracker should keep its local files. * @throws DiskErrorException if all local directories are not writable */ private static void checkLocalDirs(String[] localDirs) throws DiskErrorException { boolean writable = false; if (localDirs != null) { for (int i = 0; i < localDirs.length; i++) { try { DiskChecker.checkDir(new File(localDirs[i])); writable = true; } catch(DiskErrorException e) { LOG.warn("Task Tracker local " + e.getMessage()); } } } if (!writable) throw new DiskErrorException( "all local directories are not writable"); }
public static AvatarDataNode makeInstance(String[] dataDirs, Configuration conf) throws IOException { ArrayList<File> dirs = new ArrayList<File>(); for (int i = 0; i < dataDirs.length; i++) { File data = new File(dataDirs[i]); try { DiskChecker.checkDir(data); dirs.add(data); } catch(DiskErrorException e) { LOG.warn("Invalid directory in dfs.data.dir: " + e.getMessage()); } } if (dirs.size() > 0) { String dnThreadName = "AvatarDataNode: [" + StringUtils.arrayToString(dataDirs) + "]"; return new AvatarDataNode(conf, dirs, dnThreadName); } LOG.error("All directories in dfs.data.dir are invalid."); return null; }
/** * Make an instance of DataNode after ensuring that at least one of the * given data directories (and their parent directories, if necessary) * can be created. * @param dataDirs List of directories, where the new DataNode instance should * keep its files. * @param conf Configuration instance to use. * @return DataNode instance for given list of data dirs and conf, or null if * no directory from this directory list can be created. * @throws IOException */ public static DataNode makeInstance(String[] dataDirs, Configuration conf) throws IOException { ArrayList<File> dirs = new ArrayList<File>(); for (int i = 0; i < dataDirs.length; i++) { File data = new File(dataDirs[i]); try { DiskChecker.checkDir(data); dirs.add(data); } catch(DiskErrorException e) { LOG.warn("Invalid directory in dfs.data.dir: " + e.getMessage()); } } if (dirs.size() > 0) return new DataNode(conf, dirs); LOG.error("All directories in dfs.data.dir are invalid."); return null; }
/** * Check the health of current set of local directories, updating the list * of valid directories if necessary. * @return <em>true</em> if there is a new disk-failure identified in * this checking. <em>false</em> otherwise. */ synchronized boolean checkDirs() { int oldNumFailures = numFailures; for (final String dir : localDirs) { try { DiskChecker.checkDir(new File(dir)); } catch (DiskErrorException de) { LOG.warn("Directory " + dir + " error " + de.getMessage() + ", removing from the list of valid directories."); localDirs.remove(dir); failedDirs.add(dir); numFailures++; } } return numFailures > oldNumFailures; }
private Path createPath(Path dir, String path, boolean checkWrite) throws IOException { Path file = new Path(dir, path); if (checkWrite) { //check whether we are able to create a directory here. If the disk //happens to be RDONLY we will fail try { DiskChecker.checkDir(new File(file.getParent().toUri().getPath())); return file; } catch (DiskErrorException d) { LOG.warn("Disk Error Exception: ", d); return null; } } return file; }
/** Get a path from the local FS for reading. We search through all the * configured dirs for the file's existence and return the complete * path to the file when we find one */ public Path getLocalPathToRead(String pathStr, Configuration conf) throws IOException { Context ctx = confChanged(conf); int numDirs = ctx.localDirs.length; int numDirsSearched = 0; //remove the leading slash from the path (to make sure that the uri //resolution results in a valid path on the dir being checked) if (pathStr.startsWith("/")) { pathStr = pathStr.substring(1); } while (numDirsSearched < numDirs) { Path file = new Path(ctx.localDirs[numDirsSearched], pathStr); if (ctx.localFS.exists(file)) { return file; } numDirsSearched++; } //no path found throw new DiskErrorException ("Could not find " + pathStr +" in any of" + " the configured local directories"); }
