/** * Tests that the # of missing block replicas and expected replicas is correct * @throws IOException */ @Test public void testFsckMissingReplicas() throws IOException { // Desired replication factor // Set this higher than NUM_REPLICAS so it's under-replicated final short REPL_FACTOR = 2; // Number of replicas to actually start final short NUM_REPLICAS = 1; // Number of blocks to write final short NUM_BLOCKS = 3; // Set a small-ish blocksize final long blockSize = 512; Configuration conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); MiniDFSCluster cluster = null; DistributedFileSystem dfs = null; try { // Startup a minicluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_REPLICAS).build(); assertNotNull("Failed Cluster Creation", cluster); cluster.waitClusterUp(); dfs = cluster.getFileSystem(); assertNotNull("Failed to get FileSystem", dfs); // Create a file that will be intentionally under-replicated final String pathString = new String("/testfile"); final Path path = new Path(pathString); long fileLen = blockSize * NUM_BLOCKS; DFSTestUtil.createFile(dfs, path, fileLen, REPL_FACTOR, 1); // Create an under-replicated file NameNode namenode = cluster.getNameNode(); NetworkTopology nettop = cluster.getNamesystem().getBlockManager() .getDatanodeManager().getNetworkTopology(); Map<String,String[]> pmap = new HashMap<String, String[]>(); Writer result = new StringWriter(); PrintWriter out = new PrintWriter(result, true); InetAddress remoteAddress = InetAddress.getLocalHost(); NamenodeFsck fsck = new NamenodeFsck(conf, namenode, nettop, pmap, out, NUM_REPLICAS, remoteAddress); // Run the fsck and check the Result final HdfsFileStatus file = namenode.getRpcServer().getFileInfo(pathString); assertNotNull(file); Result res = new Result(conf); fsck.check(pathString, file, res); // Also print the output from the fsck, for ex post facto sanity checks System.out.println(result.toString()); assertEquals(res.missingReplicas, (NUM_BLOCKS*REPL_FACTOR) - (NUM_BLOCKS*NUM_REPLICAS)); assertEquals(res.numExpectedReplicas, NUM_BLOCKS*REPL_FACTOR); } finally { if(dfs != null) { dfs.close(); } if(cluster != null) { cluster.shutdown(); } } }
/** * Tests that the # of misreplaced replicas is correct * @throws IOException */ @Test public void testFsckMisPlacedReplicas() throws IOException { // Desired replication factor final short REPL_FACTOR = 2; // Number of replicas to actually start short NUM_DN = 2; // Number of blocks to write final short NUM_BLOCKS = 3; // Set a small-ish blocksize final long blockSize = 512; String [] racks = {"/rack1", "/rack1"}; String [] hosts = {"host1", "host2"}; Configuration conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); MiniDFSCluster cluster = null; DistributedFileSystem dfs = null; try { // Startup a minicluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DN).hosts(hosts) .racks(racks).build(); assertNotNull("Failed Cluster Creation", cluster); cluster.waitClusterUp(); dfs = cluster.getFileSystem(); assertNotNull("Failed to get FileSystem", dfs); // Create a file that will be intentionally under-replicated final String pathString = new String("/testfile"); final Path path = new Path(pathString); long fileLen = blockSize * NUM_BLOCKS; DFSTestUtil.createFile(dfs, path, fileLen, REPL_FACTOR, 1); // Create an under-replicated file NameNode namenode = cluster.getNameNode(); NetworkTopology nettop = cluster.getNamesystem().getBlockManager() .getDatanodeManager().getNetworkTopology(); // Add a new node on different rack, so previous blocks' replicas // are considered to be misplaced nettop.add(DFSTestUtil.getDatanodeDescriptor("/rack2", "/host3")); NUM_DN++; Map<String,String[]> pmap = new HashMap<String, String[]>(); Writer result = new StringWriter(); PrintWriter out = new PrintWriter(result, true); InetAddress remoteAddress = InetAddress.getLocalHost(); NamenodeFsck fsck = new NamenodeFsck(conf, namenode, nettop, pmap, out, NUM_DN, remoteAddress); // Run the fsck and check the Result final HdfsFileStatus file = namenode.getRpcServer().getFileInfo(pathString); assertNotNull(file); Result res = new Result(conf); fsck.check(pathString, file, res); // check misReplicatedBlock number. assertEquals(res.numMisReplicatedBlocks, NUM_BLOCKS); } finally { if(dfs != null) { dfs.close(); } if(cluster != null) { cluster.shutdown(); } } }
/** Test fsck with FileNotFound */ @Test public void testFsckFileNotFound() throws Exception { // Number of replicas to actually start final short NUM_REPLICAS = 1; Configuration conf = new Configuration(); NameNode namenode = mock(NameNode.class); NetworkTopology nettop = mock(NetworkTopology.class); Map<String,String[]> pmap = new HashMap<String, String[]>(); Writer result = new StringWriter(); PrintWriter out = new PrintWriter(result, true); InetAddress remoteAddress = InetAddress.getLocalHost(); FSNamesystem fsName = mock(FSNamesystem.class); BlockManager blockManager = mock(BlockManager.class); DatanodeManager dnManager = mock(DatanodeManager.class); when(namenode.getNamesystem()).thenReturn(fsName); when(fsName.getBlockLocations(any(FSPermissionChecker.class), anyString(), anyLong(), anyLong(), anyBoolean(), anyBoolean())) .thenThrow(new FileNotFoundException()); when(fsName.getBlockManager()).thenReturn(blockManager); when(blockManager.getDatanodeManager()).thenReturn(dnManager); NamenodeFsck fsck = new NamenodeFsck(conf, namenode, nettop, pmap, out, NUM_REPLICAS, remoteAddress); String pathString = "/tmp/testFile"; long length = 123L; boolean isDir = false; int blockReplication = 1; long blockSize = 128 *1024L; long modTime = 123123123L; long accessTime = 123123120L; FsPermission perms = FsPermission.getDefault(); String owner = "foo"; String group = "bar"; byte [] symlink = null; byte [] path = new byte[128]; path = DFSUtil.string2Bytes(pathString); long fileId = 312321L; int numChildren = 1; byte storagePolicy = 0; HdfsFileStatus file = new HdfsFileStatus(length, isDir, blockReplication, blockSize, modTime, accessTime, perms, owner, group, symlink, path, fileId, numChildren, null, storagePolicy); Result res = new Result(conf); try { fsck.check(pathString, file, res); } catch (Exception e) { fail("Unexpected exception "+ e.getMessage()); } assertTrue(res.toString().contains("HEALTHY")); }
/** * Tests that the # of missing block replicas and expected replicas is correct * @throws IOException */ @Test public void testFsckMissingReplicas() throws IOException { // Desired replication factor // Set this higher than NUM_REPLICAS so it's under-replicated final short REPL_FACTOR = 2; // Number of replicas to actually start final short NUM_REPLICAS = 1; // Number of blocks to write final short NUM_BLOCKS = 3; // Set a small-ish blocksize final long blockSize = 512; Configuration conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); MiniDFSCluster cluster = null; DistributedFileSystem dfs = null; try { // Startup a minicluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_REPLICAS).build(); assertNotNull("Failed Cluster Creation", cluster); cluster.waitClusterUp(); dfs = cluster.getFileSystem(); assertNotNull("Failed to get FileSystem", dfs); // Create a file that will be intentionally under-replicated final String pathString = new String("/testfile"); final Path path = new Path(pathString); long fileLen = blockSize * NUM_BLOCKS; DFSTestUtil.createFile(dfs, path, fileLen, REPL_FACTOR, 1); // Create an under-replicated file NameNode namenode = cluster.getNameNode(); NetworkTopology nettop = cluster.getNamesystem().getBlockManager() .getDatanodeManager().getNetworkTopology(); Map<String,String[]> pmap = new HashMap<String, String[]>(); Writer result = new StringWriter(); PrintWriter out = new PrintWriter(result, true); InetAddress remoteAddress = InetAddress.getLocalHost(); NamenodeFsck fsck = new NamenodeFsck(conf, namenode, nettop, pmap, out, NUM_REPLICAS, remoteAddress); // Run the fsck and check the Result final HdfsFileStatus file = namenode.getRpcServer().getFileInfo(pathString); assertNotNull(file); Result replRes = new ReplicationResult(conf); Result ecRes = new ErasureCodingResult(conf); fsck.check(pathString, file, replRes, ecRes); // Also print the output from the fsck, for ex post facto sanity checks System.out.println(result.toString()); assertEquals(replRes.missingReplicas, (NUM_BLOCKS*REPL_FACTOR) - (NUM_BLOCKS*NUM_REPLICAS)); assertEquals(replRes.numExpectedReplicas, NUM_BLOCKS*REPL_FACTOR); } finally { if(dfs != null) { dfs.close(); } if(cluster != null) { cluster.shutdown(); } } }
/** * Tests that the # of misreplaced replicas is correct * @throws IOException */ @Test public void testFsckMisPlacedReplicas() throws IOException { // Desired replication factor final short REPL_FACTOR = 2; // Number of replicas to actually start short NUM_DN = 2; // Number of blocks to write final short NUM_BLOCKS = 3; // Set a small-ish blocksize final long blockSize = 512; String [] racks = {"/rack1", "/rack1"}; String [] hosts = {"host1", "host2"}; Configuration conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); MiniDFSCluster cluster = null; DistributedFileSystem dfs = null; try { // Startup a minicluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DN).hosts(hosts) .racks(racks).build(); assertNotNull("Failed Cluster Creation", cluster); cluster.waitClusterUp(); dfs = cluster.getFileSystem(); assertNotNull("Failed to get FileSystem", dfs); // Create a file that will be intentionally under-replicated final String pathString = new String("/testfile"); final Path path = new Path(pathString); long fileLen = blockSize * NUM_BLOCKS; DFSTestUtil.createFile(dfs, path, fileLen, REPL_FACTOR, 1); // Create an under-replicated file NameNode namenode = cluster.getNameNode(); NetworkTopology nettop = cluster.getNamesystem().getBlockManager() .getDatanodeManager().getNetworkTopology(); // Add a new node on different rack, so previous blocks' replicas // are considered to be misplaced nettop.add(DFSTestUtil.getDatanodeDescriptor("/rack2", "/host3")); NUM_DN++; Map<String,String[]> pmap = new HashMap<String, String[]>(); Writer result = new StringWriter(); PrintWriter out = new PrintWriter(result, true); InetAddress remoteAddress = InetAddress.getLocalHost(); NamenodeFsck fsck = new NamenodeFsck(conf, namenode, nettop, pmap, out, NUM_DN, remoteAddress); // Run the fsck and check the Result final HdfsFileStatus file = namenode.getRpcServer().getFileInfo(pathString); assertNotNull(file); Result replRes = new ReplicationResult(conf); Result ecRes = new ErasureCodingResult(conf); fsck.check(pathString, file, replRes, ecRes); // check misReplicatedBlock number. assertEquals(replRes.numMisReplicatedBlocks, NUM_BLOCKS); } finally { if(dfs != null) { dfs.close(); } if(cluster != null) { cluster.shutdown(); } } }
/** Test fsck with FileNotFound */ @Test public void testFsckFileNotFound() throws Exception { // Number of replicas to actually start final short NUM_REPLICAS = 1; Configuration conf = new Configuration(); NameNode namenode = mock(NameNode.class); NetworkTopology nettop = mock(NetworkTopology.class); Map<String,String[]> pmap = new HashMap<>(); Writer result = new StringWriter(); PrintWriter out = new PrintWriter(result, true); InetAddress remoteAddress = InetAddress.getLocalHost(); FSNamesystem fsName = mock(FSNamesystem.class); FSDirectory fsd = mock(FSDirectory.class); BlockManager blockManager = mock(BlockManager.class); DatanodeManager dnManager = mock(DatanodeManager.class); INodesInPath iip = mock(INodesInPath.class); when(namenode.getNamesystem()).thenReturn(fsName); when(fsName.getBlockManager()).thenReturn(blockManager); when(fsName.getFSDirectory()).thenReturn(fsd); when(fsd.getFSNamesystem()).thenReturn(fsName); when(fsd.getINodesInPath(anyString(), anyBoolean())).thenReturn(iip); when(blockManager.getDatanodeManager()).thenReturn(dnManager); NamenodeFsck fsck = new NamenodeFsck(conf, namenode, nettop, pmap, out, NUM_REPLICAS, remoteAddress); String pathString = "/tmp/testFile"; long length = 123L; boolean isDir = false; int blockReplication = 1; long blockSize = 128 *1024L; long modTime = 123123123L; long accessTime = 123123120L; FsPermission perms = FsPermission.getDefault(); String owner = "foo"; String group = "bar"; byte [] symlink = null; byte [] path = DFSUtil.string2Bytes(pathString); long fileId = 312321L; int numChildren = 1; byte storagePolicy = 0; HdfsFileStatus file = new HdfsFileStatus(length, isDir, blockReplication, blockSize, modTime, accessTime, perms, owner, group, symlink, path, fileId, numChildren, null, storagePolicy, null); Result replRes = new ReplicationResult(conf); Result ecRes = new ErasureCodingResult(conf); try { fsck.check(pathString, file, replRes, ecRes); } catch (Exception e) { fail("Unexpected exception " + e.getMessage()); } assertTrue(replRes.isHealthy()); }
/** * Tests that the # of missing block replicas and expected replicas is correct * @throws IOException */ @Test public void testFsckMissingReplicas() throws IOException { // Desired replication factor // Set this higher than NUM_REPLICAS so it's under-replicated final short REPL_FACTOR = 2; // Number of replicas to actually start final short NUM_REPLICAS = 1; // Number of blocks to write final short NUM_BLOCKS = 3; // Set a small-ish blocksize final long blockSize = 512; Configuration conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); MiniDFSCluster cluster = null; DistributedFileSystem dfs = null; try { // Startup a minicluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_REPLICAS).build(); assertNotNull("Failed Cluster Creation", cluster); cluster.waitClusterUp(); dfs = cluster.getFileSystem(); assertNotNull("Failed to get FileSystem", dfs); // Create a file that will be intentionally under-replicated final String pathString = new String("/testfile"); final Path path = new Path(pathString); long fileLen = blockSize * NUM_BLOCKS; DFSTestUtil.createFile(dfs, path, fileLen, REPL_FACTOR, 1); // Create an under-replicated file NameNode namenode = cluster.getNameNode(); NetworkTopology nettop = cluster.getNamesystem().getBlockManager() .getDatanodeManager().getNetworkTopology(); Map<String,String[]> pmap = new HashMap<String, String[]>(); Writer result = new StringWriter(); PrintWriter out = new PrintWriter(result, true); InetAddress remoteAddress = InetAddress.getLocalHost(); NamenodeFsck fsck = new NamenodeFsck(conf, namenode, nettop, pmap, out, NUM_REPLICAS, (short)1, remoteAddress); // Run the fsck and check the Result final HdfsFileStatus file = namenode.getRpcServer().getFileInfo(pathString); assertNotNull(file); Result res = new Result(conf); fsck.check(pathString, file, res); // Also print the output from the fsck, for ex post facto sanity checks System.out.println(result.toString()); assertEquals(res.missingReplicas, (NUM_BLOCKS*REPL_FACTOR) - (NUM_BLOCKS*NUM_REPLICAS)); assertEquals(res.numExpectedReplicas, NUM_BLOCKS*REPL_FACTOR); } finally { if(dfs != null) { dfs.close(); } if(cluster != null) { cluster.shutdown(); } } }
/** * Tests that the # of misreplaced replicas is correct * @throws IOException */ @Test public void testFsckMisPlacedReplicas() throws IOException { // Desired replication factor final short REPL_FACTOR = 2; // Number of replicas to actually start short NUM_DN = 2; // Number of blocks to write final short NUM_BLOCKS = 3; // Set a small-ish blocksize final long blockSize = 512; String [] racks = {"/rack1", "/rack1"}; String [] hosts = {"host1", "host2"}; Configuration conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); MiniDFSCluster cluster = null; DistributedFileSystem dfs = null; try { // Startup a minicluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DN).hosts(hosts) .racks(racks).build(); assertNotNull("Failed Cluster Creation", cluster); cluster.waitClusterUp(); dfs = cluster.getFileSystem(); assertNotNull("Failed to get FileSystem", dfs); // Create a file that will be intentionally under-replicated final String pathString = new String("/testfile"); final Path path = new Path(pathString); long fileLen = blockSize * NUM_BLOCKS; DFSTestUtil.createFile(dfs, path, fileLen, REPL_FACTOR, 1); // Create an under-replicated file NameNode namenode = cluster.getNameNode(); NetworkTopology nettop = cluster.getNamesystem().getBlockManager() .getDatanodeManager().getNetworkTopology(); // Add a new node on different rack, so previous blocks' replicas // are considered to be misplaced nettop.add(DFSTestUtil.getDatanodeDescriptor("/rack2", "/host3")); NUM_DN++; Map<String,String[]> pmap = new HashMap<String, String[]>(); Writer result = new StringWriter(); PrintWriter out = new PrintWriter(result, true); InetAddress remoteAddress = InetAddress.getLocalHost(); NamenodeFsck fsck = new NamenodeFsck(conf, namenode, nettop, pmap, out, NUM_DN, REPL_FACTOR, remoteAddress); // Run the fsck and check the Result final HdfsFileStatus file = namenode.getRpcServer().getFileInfo(pathString); assertNotNull(file); Result res = new Result(conf); fsck.check(pathString, file, res); // check misReplicatedBlock number. assertEquals(res.numMisReplicatedBlocks, NUM_BLOCKS); } finally { if(dfs != null) { dfs.close(); } if(cluster != null) { cluster.shutdown(); } } }
/** Test fsck with FileNotFound */ @Test public void testFsckFileNotFound() throws Exception { // Number of replicas to actually start final short NUM_REPLICAS = 1; Configuration conf = new Configuration(); NameNode namenode = mock(NameNode.class); NetworkTopology nettop = mock(NetworkTopology.class); Map<String,String[]> pmap = new HashMap<String, String[]>(); Writer result = new StringWriter(); PrintWriter out = new PrintWriter(result, true); InetAddress remoteAddress = InetAddress.getLocalHost(); FSNamesystem fsName = mock(FSNamesystem.class); BlockManager blockManager = mock(BlockManager.class); DatanodeManager dnManager = mock(DatanodeManager.class); when(namenode.getNamesystem()).thenReturn(fsName); when(fsName.getBlockLocations(anyString(), anyLong(), anyLong(), anyBoolean(), anyBoolean(), anyBoolean())). thenThrow(new FileNotFoundException()) ; when(fsName.getBlockManager()).thenReturn(blockManager); when(blockManager.getDatanodeManager()).thenReturn(dnManager); NamenodeFsck fsck = new NamenodeFsck(conf, namenode, nettop, pmap, out, NUM_REPLICAS, (short)1, remoteAddress); String pathString = "/tmp/testFile"; long length = 123L; boolean isDir = false; int blockReplication = 1; long blockSize = 128 *1024L; long modTime = 123123123L; long accessTime = 123123120L; FsPermission perms = FsPermission.getDefault(); String owner = "foo"; String group = "bar"; byte [] symlink = null; byte [] path = new byte[128]; path = DFSUtil.string2Bytes(pathString); long fileId = 312321L; int numChildren = 1; byte storagePolicy = 0; HdfsFileStatus file = new HdfsFileStatus(length, isDir, blockReplication, blockSize, modTime, accessTime, perms, owner, group, symlink, path, fileId, numChildren, null, storagePolicy); Result res = new Result(conf); try { fsck.check(pathString, file, res); } catch (Exception e) { fail("Unexpected exception "+ e.getMessage()); } assertTrue(res.toString().contains("HEALTHY")); }
/** * Tests that the # of missing block replicas and expected replicas is correct * @throws IOException */ @Test public void testFsckMissingReplicas() throws IOException { // Desired replication factor // Set this higher than NUM_REPLICAS so it's under-replicated final short REPL_FACTOR = 2; // Number of replicas to actually start final short NUM_REPLICAS = 1; // Number of blocks to write final short NUM_BLOCKS = 3; // Set a small-ish blocksize final long blockSize = 512; Configuration conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); MiniDFSCluster cluster = null; DistributedFileSystem dfs = null; try { // Startup a minicluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_REPLICAS).build(); assertNotNull("Failed Cluster Creation", cluster); cluster.waitClusterUp(); dfs = (DistributedFileSystem) cluster.getFileSystem(); assertNotNull("Failed to get FileSystem", dfs); // Create a file that will be intentionally under-replicated final String pathString = new String("/testfile"); final Path path = new Path(pathString); long fileLen = blockSize * NUM_BLOCKS; DFSTestUtil.createFile(dfs, path, fileLen, REPL_FACTOR, 1); // Create an under-replicated file NameNode namenode = cluster.getNameNode(); NetworkTopology nettop = cluster.getNamesystem().getBlockManager() .getDatanodeManager().getNetworkTopology(); Map<String,String[]> pmap = new HashMap<String, String[]>(); Writer result = new StringWriter(); PrintWriter out = new PrintWriter(result, true); InetAddress remoteAddress = InetAddress.getLocalHost(); NamenodeFsck fsck = new NamenodeFsck(conf, namenode, nettop, pmap, out, NUM_REPLICAS, (short)1, remoteAddress); // Run the fsck and check the Result final HdfsFileStatus file = namenode.getRpcServer().getFileInfo(pathString); assertNotNull(file); Result res = new Result(conf); fsck.check(pathString, file, res); // Also print the output from the fsck, for ex post facto sanity checks System.out.println(result.toString()); assertEquals(res.missingReplicas, (NUM_BLOCKS*REPL_FACTOR) - (NUM_BLOCKS*NUM_REPLICAS)); assertEquals(res.numExpectedReplicas, NUM_BLOCKS*REPL_FACTOR); } finally { if(dfs != null) { dfs.close(); } if(cluster != null) { cluster.shutdown(); } } }
/** * Tests that the # of missing block replicas and expected replicas is * correct * * @throws IOException */ @Test public void testFsckMissingReplicas() throws IOException { // Desired replication factor // Set this higher than NUM_REPLICAS so it's under-replicated final short REPL_FACTOR = 2; // Number of replicas to actually start final short NUM_REPLICAS = 1; // Number of blocks to write final short NUM_BLOCKS = 3; // Set a small-ish blocksize final long blockSize = 512; Configuration conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); MiniDFSCluster cluster = null; DistributedFileSystem dfs = null; try { // Startup a minicluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_REPLICAS).build(); assertNotNull("Failed Cluster Creation", cluster); cluster.waitClusterUp(); dfs = (DistributedFileSystem) cluster.getFileSystem(); assertNotNull("Failed to get FileSystem", dfs); // Create a file that will be intentionally under-replicated final String pathString = new String("/testfile"); final Path path = new Path(pathString); long fileLen = blockSize * NUM_BLOCKS; DFSTestUtil.createFile(dfs, path, fileLen, REPL_FACTOR, 1); // Create an under-replicated file NameNode namenode = cluster.getNameNode(); NetworkTopology nettop = cluster.getNamesystem().getBlockManager().getDatanodeManager() .getNetworkTopology(); Map<String, String[]> pmap = new HashMap<>(); Writer result = new StringWriter(); PrintWriter out = new PrintWriter(result, true); InetAddress remoteAddress = InetAddress.getLocalHost(); NamenodeFsck fsck = new NamenodeFsck(conf, namenode, nettop, pmap, out, NUM_REPLICAS, (short) 1, remoteAddress); // Run the fsck and check the Result final HdfsFileStatus file = namenode.getRpcServer().getFileInfo(pathString); assertNotNull(file); Result res = new Result(conf); fsck.check(pathString, file, res); // Also print the output from the fsck, for ex post facto sanity checks System.out.println(result.toString()); assertEquals(res.missingReplicas, (NUM_BLOCKS * REPL_FACTOR) - (NUM_BLOCKS * NUM_REPLICAS)); assertEquals(res.numExpectedReplicas, NUM_BLOCKS * REPL_FACTOR); } finally { if (dfs != null) { dfs.close(); } if (cluster != null) { cluster.shutdown(); } } }
/** Test fsck with FileNotFound */ @Test public void testFsckFileNotFound() throws Exception { // Number of replicas to actually start final short NUM_REPLICAS = 1; Configuration conf = new Configuration(); NameNode namenode = mock(NameNode.class); NetworkTopology nettop = mock(NetworkTopology.class); Map<String,String[]> pmap = new HashMap<String, String[]>(); Writer result = new StringWriter(); PrintWriter out = new PrintWriter(result, true); InetAddress remoteAddress = InetAddress.getLocalHost(); FSNamesystem fsName = mock(FSNamesystem.class); when(namenode.getNamesystem()).thenReturn(fsName); when(fsName.getBlockLocations(anyString(), anyLong(), anyLong(), anyBoolean(), anyBoolean(), anyBoolean())). thenThrow(new FileNotFoundException()) ; NamenodeFsck fsck = new NamenodeFsck(conf, namenode, nettop, pmap, out, NUM_REPLICAS, (short)1, remoteAddress); String pathString = "/tmp/testFile"; long length = 123L; boolean isDir = false; int blockReplication = 1; long blockSize = 128 *1024L; long modTime = 123123123L; long accessTime = 123123120L; FsPermission perms = FsPermission.getDefault(); String owner = "foo"; String group = "bar"; byte [] symlink = null; byte [] path = new byte[128]; path = DFSUtil.string2Bytes(pathString); long fileId = 312321L; int numChildren = 1; HdfsFileStatus file = new HdfsFileStatus(length, isDir, blockReplication, blockSize, modTime, accessTime, perms, owner, group, symlink, path, fileId, numChildren); Result res = new Result(conf); try { fsck.check(pathString, file, res); } catch (Exception e) { fail("Unexpected exception "+ e.getMessage()); } assertTrue(res.toString().contains("HEALTHY")); }
/** * Tests that the # of misreplaced replicas is correct * @throws IOException */ @Test public void testFsckMisPlacedReplicas() throws IOException { // Desired replication factor final short REPL_FACTOR = 2; // Number of replicas to actually start short NUM_DN = 2; // Number of blocks to write final short NUM_BLOCKS = 3; // Set a small-ish blocksize final long blockSize = 512; String [] racks = {"/rack1", "/rack1"}; String [] hosts = {"host1", "host2"}; Configuration conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); MiniDFSCluster cluster = null; DistributedFileSystem dfs = null; try { // Startup a minicluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DN).hosts(hosts) .racks(racks).build(); assertNotNull("Failed Cluster Creation", cluster); cluster.waitClusterUp(); dfs = (DistributedFileSystem) cluster.getFileSystem(); assertNotNull("Failed to get FileSystem", dfs); // Create a file that will be intentionally under-replicated final String pathString = new String("/testfile"); final Path path = new Path(pathString); long fileLen = blockSize * NUM_BLOCKS; DFSTestUtil.createFile(dfs, path, fileLen, REPL_FACTOR, 1); // Create an under-replicated file NameNode namenode = cluster.getNameNode(); NetworkTopology nettop = cluster.getNamesystem().getBlockManager() .getDatanodeManager().getNetworkTopology(); // Add a new node on different rack, so previous blocks' replicas // are considered to be misplaced nettop.add(DFSTestUtil.getDatanodeDescriptor("/rack2", "/host3")); NUM_DN++; Map<String,String[]> pmap = new HashMap<String, String[]>(); Writer result = new StringWriter(); PrintWriter out = new PrintWriter(result, true); InetAddress remoteAddress = InetAddress.getLocalHost(); NamenodeFsck fsck = new NamenodeFsck(conf, namenode, nettop, pmap, out, NUM_DN, (short)REPL_FACTOR, remoteAddress); // Run the fsck and check the Result final HdfsFileStatus file = namenode.getRpcServer().getFileInfo(pathString); assertNotNull(file); Result res = new Result(conf); fsck.check(pathString, file, res); // check misReplicatedBlock number. assertEquals(res.numMisReplicatedBlocks, NUM_BLOCKS); } finally { if(dfs != null) { dfs.close(); } if(cluster != null) { cluster.shutdown(); } } }
