private static void verifyFsckHealth(String expected) throws Exception { // Fsck health has error code 0. // Make sure filesystem is in healthy state String outStr = runFsck(conf, 0, true, "/"); LOG.info(outStr); Assert.assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); if (!expected.equals("")) { Assert.assertTrue(outStr.contains(expected)); } }
/** * Test running fsck on a system with encryption zones. */ @Test(timeout = 60000) public void testFsckOnEncryptionZones() throws Exception { final int len = 8196; final Path zoneParent = new Path("/zones"); final Path zone1 = new Path(zoneParent, "zone1"); final Path zone1File = new Path(zone1, "file"); fsWrapper.mkdir(zone1, FsPermission.getDirDefault(), true); dfsAdmin.createEncryptionZone(zone1, TEST_KEY); DFSTestUtil.createFile(fs, zone1File, len, (short) 1, 0xFEED); ByteArrayOutputStream bStream = new ByteArrayOutputStream(); PrintStream out = new PrintStream(bStream, true); int errCode = ToolRunner.run(new DFSck(conf, out), new String[]{ "/" }); assertEquals("Fsck ran with non-zero error code", 0, errCode); String result = bStream.toString(); assertTrue("Fsck did not return HEALTHY status", result.contains(NamenodeFsck.HEALTHY_STATUS)); // Run fsck directly on the encryption zone instead of root errCode = ToolRunner.run(new DFSck(conf, out), new String[]{ zoneParent.toString() }); assertEquals("Fsck ran with non-zero error code", 0, errCode); result = bStream.toString(); assertTrue("Fsck did not return HEALTHY status", result.contains(NamenodeFsck.HEALTHY_STATUS)); }
private static void verifyFsckBlockCorrupted() throws Exception { String outStr = runFsck(conf, 1, true, "/"); LOG.info(outStr); Assert.assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS)); }
private int doWork(final String[] args) throws IOException { final StringBuilder url = new StringBuilder(); URI namenodeAddress = getCurrentNamenodeAddress(); if (namenodeAddress == null) { //Error message already output in {@link #getCurrentNamenodeAddress()} System.err.println("DFSck exiting."); return 0; } url.append(namenodeAddress.toString()); System.err.println("Connecting to namenode via " + url.toString()); url.append("/fsck?ugi=").append(ugi.getShortUserName()); String dir = null; boolean doListCorruptFileBlocks = false; for (int idx = 0; idx < args.length; idx++) { if (args[idx].equals("-move")) { url.append("&move=1"); } else if (args[idx].equals("-delete")) { url.append("&delete=1"); } else if (args[idx].equals("-files")) { url.append("&files=1"); } else if (args[idx].equals("-openforwrite")) { url.append("&openforwrite=1"); } else if (args[idx].equals("-blocks")) { url.append("&blocks=1"); } else if (args[idx].equals("-locations")) { url.append("&locations=1"); } else if (args[idx].equals("-racks")) { url.append("&racks=1"); } else if (args[idx].equals("-list-corruptfileblocks")) { url.append("&listcorruptfileblocks=1"); doListCorruptFileBlocks = true; } else if (args[idx].equals("-includeSnapshots")) { url.append("&includeSnapshots=1"); } else if (!args[idx].startsWith("-")) { if (null == dir) { dir = args[idx]; } else { System.err.println("fsck: can only operate on one path at a time '" + args[idx] + "'"); printUsage(System.err); return -1; } } else { System.err.println("fsck: Illegal option '" + args[idx] + "'"); printUsage(System.err); return -1; } } if (null == dir) { dir = "/"; } url.append("&path=").append(URLEncoder.encode(dir, "UTF-8")); if (doListCorruptFileBlocks) { return listCorruptFileBlocks(dir, url.toString()); } URL path = new URL(url.toString()); URLConnection connection; try { connection = connectionFactory.openConnection(path, isSpnegoEnabled); } catch (AuthenticationException e) { throw new IOException(e); } InputStream stream = connection.getInputStream(); BufferedReader input = new BufferedReader(new InputStreamReader( stream, "UTF-8")); String line = null; String lastLine = null; int errCode = -1; try { while ((line = input.readLine()) != null) { out.println(line); lastLine = line; } } finally { input.close(); } if (lastLine.endsWith(NamenodeFsck.HEALTHY_STATUS)) { errCode = 0; } else if (lastLine.endsWith(NamenodeFsck.CORRUPT_STATUS)) { errCode = 1; } else if (lastLine.endsWith(NamenodeFsck.NONEXISTENT_STATUS)) { errCode = 0; } return errCode; }
/** * @param args */ public int run(String[] args) throws Exception { try { args = DFSUtil.setGenericConf(args, getConf()); } catch (IllegalArgumentException e) { System.err.println(e.getMessage()); printUsage(); return -1; } String fsName = getInfoServer(); if (args.length == 0) { printUsage(); return -1; } StringBuffer url = new StringBuffer("http://"+fsName+"/fsck?path="); String dir = "/"; int limit = 500; // limit output. // find top-level dir first for (int idx = 0; idx < args.length; ) { if (args[idx].equals("-limit")) { idx++; // Skip over limit value } else if (!args[idx].startsWith("-")) { dir = args[idx]; break; } idx++; } url.append(URLEncoder.encode(dir, "UTF-8")); boolean doListCorruptFileBlocks = false; for (int idx = 0; idx < args.length; ) { if (args[idx].equals("-move")) { url.append("&move=1"); } else if (args[idx].equals("-delete")) { url.append("&delete=1"); } else if (args[idx].equals("-files")) { url.append("&files=1"); } else if (args[idx].equals("-openforwrite")) { url.append("&openforwrite=1"); } else if (args[idx].equals("-blocks")) { url.append("&blocks=1"); } else if (args[idx].equals("-locations")) { url.append("&locations=1"); } else if (args[idx].equals("-racks")) { url.append("&racks=1"); } else if (args[idx].equals("-list-corruptfileblocks")) { url.append("&listcorruptfileblocks=1"); doListCorruptFileBlocks = true; } else if (args[idx].equals("-limit")) { idx++; limit = Integer.parseInt(args[idx]); } else if (args[idx].equals("-list-decommissioningblocks")) { url.append("&decommissioning=1"); } idx++; } if (doListCorruptFileBlocks) { return listCorruptFileBlocks(dir, limit, url.toString()); } URL path = new URL(url.toString()); System.err.println("Connecting to : " + path); URLConnection connection = path.openConnection(); InputStream stream = connection.getInputStream(); BufferedReader input = new BufferedReader(new InputStreamReader( stream, "UTF-8")); String line = null; String lastLine = null; int errCode = -1; try { while ((line = input.readLine()) != null) { out.println(line); lastLine = line; } } finally { input.close(); } if (lastLine.endsWith(NamenodeFsck.HEALTHY_STATUS)) { errCode = 0; } else if (lastLine.endsWith(NamenodeFsck.CORRUPT_STATUS)) { errCode = 1; } else if (lastLine.endsWith(NamenodeFsck.NONEXISTENT_STATUS)) { errCode = 0; } return errCode; }
private Integer listCorruptFileBlocks(String dir, String baseUrl) throws IOException { int errCode = -1; int numCorrupt = 0; int cookie = 0; final String noCorruptLine = "has no CORRUPT files"; final String noMoreCorruptLine = "has no more CORRUPT files"; final String cookiePrefix = "Cookie:"; boolean allDone = false; while (!allDone) { final StringBuffer url = new StringBuffer(baseUrl); if (cookie > 0) { url.append("&startblockafter=").append(String.valueOf(cookie)); } URL path = new URL(url.toString()); URLConnection connection = SecurityUtil.openSecureHttpConnection(path); InputStream stream = connection.getInputStream(); BufferedReader input = new BufferedReader(new InputStreamReader( stream, "UTF-8")); try { String line = null; while ((line = input.readLine()) != null) { if (line.startsWith(cookiePrefix)){ try{ cookie = Integer.parseInt(line.split("\t")[1]); } catch (Exception e){ allDone = true; break; } continue; } if ((line.endsWith(noCorruptLine)) || (line.endsWith(noMoreCorruptLine)) || (line.endsWith(NamenodeFsck.NONEXISTENT_STATUS))) { allDone = true; break; } if ((line.isEmpty()) || (line.startsWith("FSCK started by")) || (line.startsWith("The filesystem under path"))) continue; numCorrupt++; if (numCorrupt == 1) { out.println("The list of corrupt files under path '" + dir + "' are:"); } out.println(line); } } finally { input.close(); } } out.println("The filesystem under path '" + dir + "' has " + numCorrupt + " CORRUPT files"); if (numCorrupt == 0) errCode = 0; return errCode; }
private int doWork(final String[] args) throws IOException { final StringBuilder url = new StringBuilder(HttpConfig.getSchemePrefix()); String namenodeAddress = getCurrentNamenodeAddress(); if (namenodeAddress == null) { //Error message already output in {@link #getCurrentNamenodeAddress()} System.err.println("DFSck exiting."); return 0; } url.append(namenodeAddress); System.err.println("Connecting to namenode via " + url.toString()); url.append("/fsck?ugi=").append(ugi.getShortUserName()); String dir = null; boolean doListCorruptFileBlocks = false; for (int idx = 0; idx < args.length; idx++) { if (args[idx].equals("-move")) { url.append("&move=1"); } else if (args[idx].equals("-delete")) { url.append("&delete=1"); } else if (args[idx].equals("-files")) { url.append("&files=1"); } else if (args[idx].equals("-openforwrite")) { url.append("&openforwrite=1"); } else if (args[idx].equals("-blocks")) { url.append("&blocks=1"); } else if (args[idx].equals("-locations")) { url.append("&locations=1"); } else if (args[idx].equals("-racks")) { url.append("&racks=1"); } else if (args[idx].equals("-list-corruptfileblocks")) { url.append("&listcorruptfileblocks=1"); doListCorruptFileBlocks = true; } else if (!args[idx].startsWith("-")) { if (null == dir) { dir = args[idx]; } else { System.err.println("fsck: can only operate on one path at a time '" + args[idx] + "'"); printUsage(System.err); return -1; } } else { System.err.println("fsck: Illegal option '" + args[idx] + "'"); printUsage(System.err); return -1; } } if (null == dir) { dir = "/"; } url.append("&path=").append(URLEncoder.encode(dir, "UTF-8")); if (doListCorruptFileBlocks) { return listCorruptFileBlocks(dir, url.toString()); } URL path = new URL(url.toString()); URLConnection connection = SecurityUtil.openSecureHttpConnection(path); InputStream stream = connection.getInputStream(); BufferedReader input = new BufferedReader(new InputStreamReader( stream, "UTF-8")); String line = null; String lastLine = null; int errCode = -1; try { while ((line = input.readLine()) != null) { out.println(line); lastLine = line; } } finally { input.close(); } if (lastLine.endsWith(NamenodeFsck.HEALTHY_STATUS)) { errCode = 0; } else if (lastLine.endsWith(NamenodeFsck.CORRUPT_STATUS)) { errCode = 1; } else if (lastLine.endsWith(NamenodeFsck.NONEXISTENT_STATUS)) { errCode = 0; } return errCode; }
private Integer listCorruptFileBlocks(String dir, String baseUrl) throws IOException { int errCode = -1; int numCorrupt = 0; int cookie = 0; final String noCorruptLine = "has no CORRUPT files"; final String noMoreCorruptLine = "has no more CORRUPT files"; final String cookiePrefix = "Cookie:"; boolean allDone = false; while (!allDone) { final StringBuffer url = new StringBuffer(baseUrl); if (cookie > 0) { url.append("&startblockafter=").append(String.valueOf(cookie)); } URL path = new URL(url.toString()); URLConnection connection = SecurityUtil2.openSecureHttpConnection(path); InputStream stream = connection.getInputStream(); BufferedReader input = new BufferedReader(new InputStreamReader(stream, "UTF-8")); try { String line = null; while ((line = input.readLine()) != null) { if (line.startsWith(cookiePrefix)) { try { cookie = Integer.parseInt(line.split("\t")[1]); } catch (Exception e) { allDone = true; break; } continue; } if ((line.endsWith(noCorruptLine)) || (line.endsWith(noMoreCorruptLine)) || (line.endsWith(NamenodeFsck.NONEXISTENT_STATUS))) { allDone = true; break; } if ((line.isEmpty()) || (line.startsWith("FSCK started by")) || (line.startsWith("The filesystem under path"))) { continue; } numCorrupt++; if (numCorrupt == 1) { out.println( "The list of corrupt files under path '" + dir + "' are:"); } out.println(line); } } finally { input.close(); } } out.println("The filesystem under path '" + dir + "' has " + numCorrupt + " CORRUPT files"); if (numCorrupt == 0) { errCode = 0; } return errCode; }
private int doWork(final String[] args) throws IOException { final StringBuilder url = new StringBuilder(HttpConfig2.getSchemePrefix()); String namenodeAddress = getCurrentNamenodeAddress(); if (namenodeAddress == null) { //Error message already output in {@link #getCurrentNamenodeAddress()} System.err.println("DFSck exiting."); return 0; } url.append(namenodeAddress); System.err.println("Connecting to namenode via " + url.toString()); url.append("/fsck?ugi=").append(ugi.getShortUserName()); String dir = null; boolean doListCorruptFileBlocks = false; for (String arg : args) { if (arg.equals("-move")) { url.append("&move=1"); } else if (arg.equals("-delete")) { url.append("&delete=1"); } else if (arg.equals("-files")) { url.append("&files=1"); } else if (arg.equals("-openforwrite")) { url.append("&openforwrite=1"); } else if (arg.equals("-blocks")) { url.append("&blocks=1"); } else if (arg.equals("-locations")) { url.append("&locations=1"); } else if (arg.equals("-racks")) { url.append("&racks=1"); } else if (arg.equals("-list-corruptfileblocks")) { url.append("&listcorruptfileblocks=1"); doListCorruptFileBlocks = true; } else if (!arg.startsWith("-")) { if (null == dir) { dir = arg; } else { System.err.println( "fsck: can only operate on one path at a time '" + arg + "'"); printUsage(System.err); return -1; } } else { System.err.println("fsck: Illegal option '" + arg + "'"); printUsage(System.err); return -1; } } if (null == dir) { dir = "/"; } url.append("&path=").append(URLEncoder.encode(dir, "UTF-8")); if (doListCorruptFileBlocks) { return listCorruptFileBlocks(dir, url.toString()); } URL path = new URL(url.toString()); URLConnection connection = SecurityUtil2.openSecureHttpConnection(path); InputStream stream = connection.getInputStream(); BufferedReader input = new BufferedReader(new InputStreamReader(stream, "UTF-8")); String line = null; String lastLine = null; int errCode = -1; try { while ((line = input.readLine()) != null) { out.println(line); lastLine = line; } } finally { input.close(); } if (lastLine.endsWith(NamenodeFsck.HEALTHY_STATUS)) { errCode = 0; } else if (lastLine.endsWith(NamenodeFsck.CORRUPT_STATUS)) { errCode = 1; } else if (lastLine.endsWith(NamenodeFsck.NONEXISTENT_STATUS)) { errCode = 0; } return errCode; }
/** * @param args */ public int run(final String[] args) throws IOException { if (args.length == 0) { printUsage(); return -1; } try { return UserGroupInformation.getCurrentUser().doAs(new PrivilegedExceptionAction<Integer>() { @Override public Integer run() throws Exception { final StringBuffer url = new StringBuffer("http://"); url.append(NameNode.getInfoServer(getConf())).append("/fsck?ugi=").append(ugi.getShortUserName()).append("&path="); String dir = "/"; // find top-level dir first for (int idx = 0; idx < args.length; idx++) { if (!args[idx].startsWith("-")) { dir = args[idx]; break; } } url.append(URLEncoder.encode(dir, "UTF-8")); for (int idx = 0; idx < args.length; idx++) { if (args[idx].equals("-move")) { url.append("&move=1"); } else if (args[idx].equals("-delete")) { url.append("&delete=1"); } else if (args[idx].equals("-files")) { url.append("&files=1"); } else if (args[idx].equals("-openforwrite")) { url.append("&openforwrite=1"); } else if (args[idx].equals("-blocks")) { url.append("&blocks=1"); } else if (args[idx].equals("-locations")) { url.append("&locations=1"); } else if (args[idx].equals("-racks")) { url.append("&racks=1"); } } URL path = new URL(url.toString()); URLConnection connection = SecurityUtil.openSecureHttpConnection(path); InputStream stream = connection.getInputStream(); BufferedReader input = new BufferedReader(new InputStreamReader( stream, "UTF-8")); String line = null; String lastLine = null; int errCode = -1; try { while ((line = input.readLine()) != null) { System.out.println(line); lastLine = line; } } finally { input.close(); } if (lastLine.endsWith(NamenodeFsck.HEALTHY_STATUS)) { errCode = 0; } else if (lastLine.endsWith(NamenodeFsck.CORRUPT_STATUS)) { errCode = 1; } else if (lastLine.endsWith(NamenodeFsck.NONEXISTENT_STATUS)) { errCode = 0; } return errCode; } }); } catch (InterruptedException e) { throw new IOException(e); } }
private Integer listCorruptFileBlocks(String dir, String baseUrl) throws IOException { int errCode = -1; int numCorrupt = 0; String lastBlock = null; final String noCorruptLine = "has no CORRUPT files"; final String noMoreCorruptLine = "has no more CORRUPT files"; boolean allDone = false; while (!allDone) { final StringBuffer url = new StringBuffer(baseUrl); if (lastBlock != null) { url.append("&startblockafter=").append(lastBlock); } URL path = new URL(url.toString()); SecurityUtil.fetchServiceTicket(path); URLConnection connection = path.openConnection(); InputStream stream = connection.getInputStream(); BufferedReader input = new BufferedReader(new InputStreamReader( stream, "UTF-8")); try { String line = null; while ((line = input.readLine()) != null) { if ((line.endsWith(noCorruptLine)) || (line.endsWith(noMoreCorruptLine)) || (line.endsWith(NamenodeFsck.NONEXISTENT_STATUS))) { allDone = true; break; } if ((line.isEmpty()) || (line.startsWith("FSCK started by")) || (line.startsWith("The filesystem under path"))) continue; numCorrupt++; if (numCorrupt == 1) { out.println("The list of corrupt files under path '" + dir + "' are:"); } out.println(line); try { // Get the block # that we need to send in next call lastBlock = line.split("\t")[0]; } catch (Exception e) { allDone = true; break; } } } finally { input.close(); } } out.println("The filesystem under path '" + dir + "' has " + numCorrupt + " CORRUPT files"); if (numCorrupt == 0) errCode = 0; return errCode; }
private int doWork(final String[] args) throws IOException { String proto = "http://"; if (UserGroupInformation.isSecurityEnabled()) { System.setProperty("https.cipherSuites", Krb5AndCertsSslSocketConnector.KRB5_CIPHER_SUITES.get(0)); proto = "https://"; } final StringBuilder url = new StringBuilder(proto); url.append(NameNode.getInfoServer(getConf())); url.append("/fsck?ugi=").append(ugi.getShortUserName()).append("&path="); String dir = "/"; // find top-level dir first for (int idx = 0; idx < args.length; idx++) { if (!args[idx].startsWith("-")) { dir = args[idx]; break; } } url.append(URLEncoder.encode(dir, "UTF-8")); boolean doListCorruptFileBlocks = false; for (int idx = 0; idx < args.length; idx++) { if (args[idx].equals("-move")) { url.append("&move=1"); } else if (args[idx].equals("-delete")) { url.append("&delete=1"); } else if (args[idx].equals("-files")) { url.append("&files=1"); } else if (args[idx].equals("-openforwrite")) { url.append("&openforwrite=1"); } else if (args[idx].equals("-blocks")) { url.append("&blocks=1"); } else if (args[idx].equals("-locations")) { url.append("&locations=1"); } else if (args[idx].equals("-racks")) { url.append("&racks=1"); } else if (args[idx].equals("-list-corruptfileblocks")) { url.append("&listcorruptfileblocks=1"); doListCorruptFileBlocks = true; } } if (doListCorruptFileBlocks) { return listCorruptFileBlocks(dir, url.toString()); } URL path = new URL(url.toString()); SecurityUtil.fetchServiceTicket(path); URLConnection connection = path.openConnection(); InputStream stream = connection.getInputStream(); BufferedReader input = new BufferedReader(new InputStreamReader( stream, "UTF-8")); String line = null; String lastLine = null; int errCode = -1; try { while ((line = input.readLine()) != null) { out.println(line); lastLine = line; } } finally { input.close(); } if (lastLine.endsWith(NamenodeFsck.HEALTHY_STATUS)) { errCode = 0; } else if (lastLine.endsWith(NamenodeFsck.CORRUPT_STATUS)) { errCode = 1; } else if (lastLine.endsWith(NamenodeFsck.NONEXISTENT_STATUS)) { errCode = 0; } return errCode; }