/** * A server-side method to respond to a getfile http request * Copies the contents of the local file into the output stream. */ static void getFileServer(OutputStream outstream, File localfile) throws IOException { byte buf[] = new byte[BUFFER_SIZE]; FileInputStream infile = null; try { infile = new FileInputStream(localfile); if (ErrorSimulator.getErrorSimulation(2) && localfile.getAbsolutePath().contains("secondary")) { // throw exception only when the secondary sends its image throw new IOException("If this exception is not caught by the " + "name-node fs image will be truncated."); } int num = 1; while (num > 0) { num = infile.read(buf); if (num <= 0) { break; } outstream.write(buf, 0, num); } } finally { if (infile != null) { infile.close(); } } }
/** * A server-side method to respond to a getfile http request * Copies the contents of the local file into the output stream. */ static void getFileServer(OutputStream outstream, File localfile) throws IOException { byte buf[] = new byte[BUFFER_SIZE]; FileInputStream infile = null; try { infile = new FileInputStream(localfile); if (ErrorSimulator.getErrorSimulation(2) && localfile.getAbsolutePath().contains("secondary")) { // throw exception only when the secondary sends its image throw new IOException("If this exception is not caught by the " + "name-node fs image will be truncated."); } if (ErrorSimulator.getErrorSimulation(3) && localfile.getAbsolutePath().contains("fsimage")) { // Test sending image shorter than localfile long len = localfile.length(); buf = new byte[(int)Math.min(len/2, BUFFER_SIZE)]; // This will read at most half of the image // and the rest of the image will be sent over the wire infile.read(buf); } int num = 1; while (num > 0) { num = infile.read(buf); if (num <= 0) { break; } outstream.write(buf, 0, num); } } finally { if (infile != null) { infile.close(); } } }
/** * Simulate a secondary node failure to transfer image * back to the name-node. * Used to truncate primary fsimage file. */ void testSecondaryFailsToReturnImage(Configuration conf) throws IOException { System.out.println("Starting testSecondaryFailsToReturnImage"); Path file1 = new Path("checkpointRI.dat"); MiniDFSCluster cluster = new MiniDFSCluster(conf, numDatanodes, false, null); cluster.waitActive(); FileSystem fileSys = cluster.getFileSystem(); FSImage image = cluster.getNameNode().getFSImage(); try { assertTrue(!fileSys.exists(file1)); StorageDirectory sd = null; for (Iterator<StorageDirectory> it = image.dirIterator(NameNodeDirType.IMAGE); it.hasNext();) sd = it.next(); assertTrue(sd != null); long fsimageLength = FSImage.getImageFile(sd, NameNodeFile.IMAGE).length(); // // Make the checkpoint // SecondaryNameNode secondary = startSecondaryNameNode(conf); ErrorSimulator.setErrorSimulation(2); try { secondary.doCheckpoint(); // this should fail assertTrue(false); } catch (IOException e) { System.out.println("testSecondaryFailsToReturnImage: doCheckpoint() " + "failed predictably - " + e); } ErrorSimulator.clearErrorSimulation(2); // Verify that image file sizes did not change. for (Iterator<StorageDirectory> it = image.dirIterator(NameNodeDirType.IMAGE); it.hasNext();) { assertTrue(FSImage.getImageFile(it.next(), NameNodeFile.IMAGE).length() == fsimageLength); } secondary.shutdown(); } finally { fileSys.close(); cluster.shutdown(); } }
/** * Simulate namenode failing to send the whole file * secondary namenode sometimes assumed it received all of it */ @SuppressWarnings("deprecation") void testNameNodeImageSendFail(Configuration conf) throws IOException { System.out.println("Starting testNameNodeImageSendFail"); Path file1 = new Path("checkpointww.dat"); MiniDFSCluster cluster = new MiniDFSCluster(conf, numDatanodes, false, null); cluster.waitActive(); FileSystem fileSys = cluster.getFileSystem(); try { assertTrue(!fileSys.exists(file1)); // // Make the checkpoint fail after rolling the edit log. // SecondaryNameNode secondary = startSecondaryNameNode(conf); ErrorSimulator.setErrorSimulation(3); try { secondary.doCheckpoint(); // this should fail fail("Did not get expected exception"); } catch (IOException e) { // We only sent part of the image. Have to trigger this exception assertTrue(e.getMessage().contains("is not of the advertised size")); } ErrorSimulator.clearErrorSimulation(3); secondary.shutdown(); // secondary namenode crash! // start new instance of secondary and verify that // a new rollEditLog suceedes inspite of the fact that // edits.new already exists. // secondary = startSecondaryNameNode(conf); secondary.doCheckpoint(); // this should work correctly secondary.shutdown(); // // Create a new file // writeFile(fileSys, file1, replication); checkFile(fileSys, file1, replication); } finally { fileSys.close(); cluster.shutdown(); } }
/** * A server-side method to respond to a getfile http request * Copies the contents of the local file into the output stream. */ static void getFileServer(OutputStream outstream, File localfile, DataTransferThrottler throttler) throws IOException { byte buf[] = new byte[BUFFER_SIZE]; FileInputStream infile = null; long totalReads = 0, totalSends = 0; try { infile = new FileInputStream(localfile); if (ErrorSimulator.getErrorSimulation(2) && localfile.getAbsolutePath().contains("secondary")) { // throw exception only when the secondary sends its image throw new IOException("If this exception is not caught by the " + "name-node fs image will be truncated."); } if (ErrorSimulator.getErrorSimulation(3) && localfile.getAbsolutePath().contains("fsimage")) { // Test sending image shorter than localfile long len = localfile.length(); buf = new byte[(int)Math.min(len/2, BUFFER_SIZE)]; // This will read at most half of the image // and the rest of the image will be sent over the wire infile.read(buf); } int num = 1; while (num > 0) { long startRead = System.currentTimeMillis(); num = infile.read(buf); if (num <= 0) { break; } outstream.write(buf, 0, num); if (throttler != null) { throttler.throttle(num); } } } finally { if (infile != null) { infile.close(); } } }
