public static MD5Hash downloadImageToStorage(URL fsName, long imageTxId, Storage dstStorage, boolean needDigest) throws IOException { String fileid = ImageServlet.getParamStringForImage(null, imageTxId, dstStorage); String fileName = NNStorage.getCheckpointImageFileName(imageTxId); List<File> dstFiles = dstStorage.getFiles( NameNodeDirType.IMAGE, fileName); if (dstFiles.isEmpty()) { throw new IOException("No targets in destination storage!"); } MD5Hash hash = getFileClient(fsName, fileid, dstFiles, dstStorage, needDigest); LOG.info("Downloaded file " + dstFiles.get(0).getName() + " size " + dstFiles.get(0).length() + " bytes."); return hash; }
static MD5Hash handleUploadImageRequest(HttpServletRequest request, long imageTxId, Storage dstStorage, InputStream stream, long advertisedSize, DataTransferThrottler throttler) throws IOException { String fileName = NNStorage.getCheckpointImageFileName(imageTxId); List<File> dstFiles = dstStorage.getFiles(NameNodeDirType.IMAGE, fileName); if (dstFiles.isEmpty()) { throw new IOException("No targets in destination storage!"); } MD5Hash advertisedDigest = parseMD5Header(request); MD5Hash hash = receiveFile(fileName, dstFiles, dstStorage, true, advertisedSize, advertisedDigest, fileName, stream, throttler); LOG.info("Downloaded file " + dstFiles.get(0).getName() + " size " + dstFiles.get(0).length() + " bytes."); return hash; }
/** * Rename FSImage with the specific txid */ private void renameCheckpoint(long txid, NameNodeFile fromNnf, NameNodeFile toNnf, boolean renameMD5) throws IOException { ArrayList<StorageDirectory> al = null; for (StorageDirectory sd : storage.dirIterable(NameNodeDirType.IMAGE)) { try { renameImageFileInDir(sd, fromNnf, toNnf, txid, renameMD5); } catch (IOException ioe) { LOG.warn("Unable to rename checkpoint in " + sd, ioe); if (al == null) { al = Lists.newArrayList(); } al.add(sd); } } if(al != null) storage.reportErrorsOnDirectories(al); }
/** * Confirm that FSImage files in all StorageDirectory are the same, * and non-empty, and there are the expected number of them. * @param fsn - the FSNamesystem being checked. * @param numImageDirs - the configured number of StorageDirectory of type IMAGE. * @return - the md5 hash of the most recent FSImage files, which must all be the same. * @throws AssertionError if image files are empty or different, * if less than two StorageDirectory are provided, or if the * actual number of StorageDirectory is less than configured. */ public static String checkImages( FSNamesystem fsn, int numImageDirs) throws Exception { NNStorage stg = fsn.getFSImage().getStorage(); //any failed StorageDirectory is removed from the storageDirs list assertEquals("Some StorageDirectories failed Upgrade", numImageDirs, stg.getNumStorageDirs(NameNodeDirType.IMAGE)); assertTrue("Not enough fsimage copies in MiniDFSCluster " + "to test parallel write", numImageDirs > 1); // List of "current/" directory from each SD List<File> dirs = FSImageTestUtil.getCurrentDirs(stg, NameNodeDirType.IMAGE); // across directories, all files with same names should be identical hashes FSImageTestUtil.assertParallelFilesAreIdentical( dirs, Collections.<String>emptySet()); FSImageTestUtil.assertSameNewestImage(dirs); // Return the hash of the newest image file StorageDirectory firstSd = stg.dirIterator(NameNodeDirType.IMAGE).next(); File latestImage = FSImageTestUtil.findLatestImageFile(firstSd); String md5 = FSImageTestUtil.getImageFileMD5IgnoringTxId(latestImage); System.err.println("md5 of " + latestImage + ": " + md5); return md5; }
/** * Test the "easy case" where we have more images in the * directory than we need to keep. Should purge the * old ones. */ @Test public void testPurgeEasyCase() throws IOException { TestCaseDescription tc = new TestCaseDescription(); tc.addRoot("/foo1", NameNodeDirType.IMAGE_AND_EDITS); tc.addImage("/foo1/current/" + getImageFileName(100), true); tc.addImage("/foo1/current/" + getImageFileName(200), true); tc.addImage("/foo1/current/" + getImageFileName(300), false); tc.addImage("/foo1/current/" + getImageFileName(400), false); tc.addLog("/foo1/current/" + getFinalizedEditsFileName(101,200), true); tc.addLog("/foo1/current/" + getFinalizedEditsFileName(201,300), true); tc.addLog("/foo1/current/" + getFinalizedEditsFileName(301,400), false); tc.addLog("/foo1/current/" + getInProgressEditsFileName(401), false); // Test that other files don't get purged tc.addLog("/foo1/current/VERSION", false); runTest(tc); }
/** * Same as above, but across multiple directories */ @Test public void testPurgeMultipleDirs() throws IOException { TestCaseDescription tc = new TestCaseDescription(); tc.addRoot("/foo1", NameNodeDirType.IMAGE_AND_EDITS); tc.addRoot("/foo2", NameNodeDirType.IMAGE_AND_EDITS); tc.addImage("/foo1/current/" + getImageFileName(100), true); tc.addImage("/foo1/current/" + getImageFileName(200), true); tc.addImage("/foo2/current/" + getImageFileName(200), true); tc.addImage("/foo1/current/" + getImageFileName(300), false); tc.addImage("/foo1/current/" + getImageFileName(400), false); tc.addLog("/foo1/current/" + getFinalizedEditsFileName(101, 200), true); tc.addLog("/foo1/current/" + getFinalizedEditsFileName(201, 300), true); tc.addLog("/foo2/current/" + getFinalizedEditsFileName(201, 300), true); tc.addLog("/foo1/current/" + getFinalizedEditsFileName(301, 400), false); tc.addLog("/foo2/current/" + getFinalizedEditsFileName(301, 400), false); tc.addLog("/foo1/current/" + getInProgressEditsFileName(401), false); runTest(tc); }
@Test public void testSeparateEditDirs() throws IOException { TestCaseDescription tc = new TestCaseDescription(); tc.addRoot("/foo1", NameNodeDirType.IMAGE); tc.addRoot("/foo2", NameNodeDirType.EDITS); tc.addImage("/foo1/current/" + getImageFileName(100), true); tc.addImage("/foo1/current/" + getImageFileName(200), true); tc.addImage("/foo1/current/" + getImageFileName(300), false); tc.addImage("/foo1/current/" + getImageFileName(400), false); tc.addLog("/foo2/current/" + getFinalizedEditsFileName(101, 200), true); tc.addLog("/foo2/current/" + getFinalizedEditsFileName(201, 300), true); tc.addLog("/foo2/current/" + getFinalizedEditsFileName(301, 400), false); tc.addLog("/foo2/current/" + getInProgressEditsFileName(401), false); runTest(tc); }
@Test public void testRetainExtraLogs() throws IOException { conf.setLong(DFSConfigKeys.DFS_NAMENODE_NUM_EXTRA_EDITS_RETAINED_KEY, 50); TestCaseDescription tc = new TestCaseDescription(); tc.addRoot("/foo1", NameNodeDirType.IMAGE); tc.addRoot("/foo2", NameNodeDirType.EDITS); tc.addImage("/foo1/current/" + getImageFileName(100), true); tc.addImage("/foo1/current/" + getImageFileName(200), true); tc.addImage("/foo1/current/" + getImageFileName(300), false); tc.addImage("/foo1/current/" + getImageFileName(400), false); tc.addLog("/foo2/current/" + getFinalizedEditsFileName(101, 200), true); // Since we need 50 extra edits, *do* retain the 201-300 segment tc.addLog("/foo2/current/" + getFinalizedEditsFileName(201, 300), false); tc.addLog("/foo2/current/" + getFinalizedEditsFileName(301, 400), false); tc.addLog("/foo2/current/" + getInProgressEditsFileName(401), false); runTest(tc); }
/** * Simple test with image, edits, and inprogress edits */ @Test public void testCurrentStorageInspector() throws IOException { FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector(); StorageDirectory mockDir = FSImageTestUtil.mockStorageDirectory( NameNodeDirType.IMAGE_AND_EDITS, false, "/foo/current/" + getImageFileName(123), "/foo/current/" + getFinalizedEditsFileName(123, 456), "/foo/current/" + getImageFileName(456), "/foo/current/" + getInProgressEditsFileName(457)); inspector.inspectDirectory(mockDir); assertEquals(2, inspector.foundImages.size()); FSImageFile latestImage = inspector.getLatestImages().get(0); assertEquals(456, latestImage.txId); assertSame(mockDir, latestImage.sd); assertTrue(inspector.isUpgradeFinalized()); assertEquals(new File("/foo/current/"+getImageFileName(456)), latestImage.getFile()); }
/** * Test the normal operation of loading transactions from * file journal manager. 3 edits directories are setup without any * failures. Test that we read in the expected number of transactions. */ @Test public void testNormalOperation() throws IOException { File f1 = new File(TestEditLog.TEST_DIR + "/normtest0"); File f2 = new File(TestEditLog.TEST_DIR + "/normtest1"); File f3 = new File(TestEditLog.TEST_DIR + "/normtest2"); List<URI> editUris = ImmutableList.of(f1.toURI(), f2.toURI(), f3.toURI()); NNStorage storage = setupEdits(editUris, 5); long numJournals = 0; for (StorageDirectory sd : storage.dirIterable(NameNodeDirType.EDITS)) { FileJournalManager jm = new FileJournalManager(conf, sd, storage); assertEquals(6*TXNS_PER_ROLL, getNumberOfTransactions(jm, 1, true, false)); numJournals++; } assertEquals(3, numJournals); }
/** * Test a mixture of inprogress files and finalised. Set up 3 edits * directories and fail the second on the last roll. Verify that reading * the transactions, reads from the finalised directories. */ @Test public void testInprogressRecoveryMixed() throws IOException { File f1 = new File(TestEditLog.TEST_DIR + "/mixtest0"); File f2 = new File(TestEditLog.TEST_DIR + "/mixtest1"); File f3 = new File(TestEditLog.TEST_DIR + "/mixtest2"); List<URI> editUris = ImmutableList.of(f1.toURI(), f2.toURI(), f3.toURI()); // abort after the 5th roll NNStorage storage = setupEdits(editUris, 5, new AbortSpec(5, 1)); Iterator<StorageDirectory> dirs = storage.dirIterator(NameNodeDirType.EDITS); StorageDirectory sd = dirs.next(); FileJournalManager jm = new FileJournalManager(conf, sd, storage); assertEquals(6*TXNS_PER_ROLL, getNumberOfTransactions(jm, 1, true, false)); sd = dirs.next(); jm = new FileJournalManager(conf, sd, storage); assertEquals(5*TXNS_PER_ROLL + TXNS_PER_FAIL, getNumberOfTransactions(jm, 1, true, false)); sd = dirs.next(); jm = new FileJournalManager(conf, sd, storage); assertEquals(6*TXNS_PER_ROLL, getNumberOfTransactions(jm, 1, true, false)); }
@Test(expected=IllegalStateException.class) public void testFinalizeErrorReportedToNNStorage() throws IOException, InterruptedException { File f = new File(TestEditLog.TEST_DIR + "/filejournaltestError"); // abort after 10th roll NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()), 10, new AbortSpec(10, 0)); StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next(); FileJournalManager jm = new FileJournalManager(conf, sd, storage); String sdRootPath = sd.getRoot().getAbsolutePath(); FileUtil.chmod(sdRootPath, "-w", true); try { jm.finalizeLogSegment(0, 1); } finally { FileUtil.chmod(sdRootPath, "+w", true); assertTrue(storage.getRemovedStorageDirs().contains(sd)); } }
/** * Test that we can read from a stream created by FileJournalManager. * Create a single edits directory, failing it on the final roll. * Then try loading from the point of the 3rd roll. Verify that we read * the correct number of transactions from this point. */ @Test public void testReadFromStream() throws IOException { File f = new File(TestEditLog.TEST_DIR + "/readfromstream"); // abort after 10th roll NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()), 10, new AbortSpec(10, 0)); StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next(); FileJournalManager jm = new FileJournalManager(conf, sd, storage); long expectedTotalTxnCount = TXNS_PER_ROLL*10 + TXNS_PER_FAIL; assertEquals(expectedTotalTxnCount, getNumberOfTransactions(jm, 1, true, false)); long skippedTxns = (3*TXNS_PER_ROLL); // skip first 3 files long startingTxId = skippedTxns + 1; long numLoadable = getNumberOfTransactions(jm, startingTxId, true, false); assertEquals(expectedTotalTxnCount - skippedTxns, numLoadable); }
/** * Make requests with starting transaction ids which don't match the beginning * txid of some log segments. * * This should succeed. */ @Test public void testAskForTransactionsMidfile() throws IOException { File f = new File(TestEditLog.TEST_DIR + "/askfortransactionsmidfile"); NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()), 10); StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next(); FileJournalManager jm = new FileJournalManager(conf, sd, storage); // 10 rolls, so 11 rolled files, 110 txids total. final int TOTAL_TXIDS = 10 * 11; for (int txid = 1; txid <= TOTAL_TXIDS; txid++) { assertEquals((TOTAL_TXIDS - txid) + 1, getNumberOfTransactions(jm, txid, true, false)); } }
/** * Test that we can load an edits directory with a corrupt inprogress file. * The corrupt inprogress file should be moved to the side. */ @Test public void testManyLogsWithCorruptInprogress() throws IOException { File f = new File(TestEditLog.TEST_DIR + "/manylogswithcorruptinprogress"); NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()), 10, new AbortSpec(10, 0)); StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next(); File[] files = new File(f, "current").listFiles(new FilenameFilter() { @Override public boolean accept(File dir, String name) { if (name.startsWith("edits_inprogress")) { return true; } return false; } }); assertEquals(files.length, 1); corruptAfterStartSegment(files[0]); FileJournalManager jm = new FileJournalManager(conf, sd, storage); assertEquals(10*TXNS_PER_ROLL+1, getNumberOfTransactions(jm, 1, true, false)); }
@Test public void testGetRemoteEditLog() throws IOException { StorageDirectory sd = FSImageTestUtil.mockStorageDirectory( NameNodeDirType.EDITS, false, NNStorage.getFinalizedEditsFileName(1, 100), NNStorage.getFinalizedEditsFileName(101, 200), NNStorage.getInProgressEditsFileName(201), NNStorage.getFinalizedEditsFileName(1001, 1100)); // passing null for NNStorage because this unit test will not use it FileJournalManager fjm = new FileJournalManager(conf, sd, null); assertEquals("[1,100],[101,200],[1001,1100]", getLogsAsString(fjm, 1)); assertEquals("[101,200],[1001,1100]", getLogsAsString(fjm, 101)); assertEquals("[101,200],[1001,1100]", getLogsAsString(fjm, 150)); assertEquals("[1001,1100]", getLogsAsString(fjm, 201)); assertEquals("Asking for a newer log than exists should return empty list", "", getLogsAsString(fjm, 9999)); }
/** * Make sure that we starting reading the correct op when we request a stream * with a txid in the middle of an edit log file. */ @Test public void testReadFromMiddleOfEditLog() throws CorruptionException, IOException { File f = new File(TestEditLog.TEST_DIR + "/readfrommiddleofeditlog"); NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()), 10); StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next(); FileJournalManager jm = new FileJournalManager(conf, sd, storage); EditLogInputStream elis = getJournalInputStream(jm, 5, true); try { FSEditLogOp op = elis.readOp(); assertEquals("read unexpected op", op.getTransactionId(), 5); } finally { IOUtils.cleanup(LOG, elis); } }
/** * Make sure that in-progress streams aren't counted if we don't ask for * them. */ @Test public void testExcludeInProgressStreams() throws CorruptionException, IOException { File f = new File(TestEditLog.TEST_DIR + "/excludeinprogressstreams"); // Don't close the edit log once the files have been set up. NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()), 10, false); StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next(); FileJournalManager jm = new FileJournalManager(conf, sd, storage); // If we exclude the in-progess stream, we should only have 100 tx. assertEquals(100, getNumberOfTransactions(jm, 1, false, false)); EditLogInputStream elis = getJournalInputStream(jm, 90, false); try { FSEditLogOp lastReadOp = null; while ((lastReadOp = elis.readOp()) != null) { assertTrue(lastReadOp.getTransactionId() <= 100); } } finally { IOUtils.cleanup(LOG, elis); } }
/** * Tests that internal renames are done using native code on platforms that * have it. The native rename includes more detailed information about the * failure, which can be useful for troubleshooting. */ @Test public void testDoPreUpgradeIOError() throws IOException { File storageDir = new File(TestEditLog.TEST_DIR, "preupgradeioerror"); List<URI> editUris = Collections.singletonList(storageDir.toURI()); NNStorage storage = setupEdits(editUris, 5); StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next(); assertNotNull(sd); // Change storage directory so that renaming current to previous.tmp fails. FileUtil.setWritable(storageDir, false); FileJournalManager jm = null; try { jm = new FileJournalManager(conf, sd, storage); exception.expect(IOException.class); if (NativeCodeLoader.isNativeCodeLoaded()) { exception.expectMessage("failure in native rename"); } jm.doPreUpgrade(); } finally { IOUtils.cleanup(LOG, jm); // Restore permissions on storage directory and make sure we can delete. FileUtil.setWritable(storageDir, true); FileUtil.fullyDelete(storageDir); } }
/** * verify that edits log and fsimage are in different directories and of a correct size */ private void verifyDifferentDirs(FSImage img, long expectedImgSize, long expectedEditsSize) { StorageDirectory sd =null; for (Iterator<StorageDirectory> it = img.getStorage().dirIterator(); it.hasNext();) { sd = it.next(); if(sd.getStorageDirType().isOfType(NameNodeDirType.IMAGE)) { img.getStorage(); File imf = NNStorage.getStorageFile(sd, NameNodeFile.IMAGE, 0); LOG.info("--image file " + imf.getAbsolutePath() + "; len = " + imf.length() + "; expected = " + expectedImgSize); assertEquals(expectedImgSize, imf.length()); } else if(sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) { img.getStorage(); File edf = NNStorage.getStorageFile(sd, NameNodeFile.EDITS, 0); LOG.info("-- edits file " + edf.getAbsolutePath() + "; len = " + edf.length() + "; expected = " + expectedEditsSize); assertEquals(expectedEditsSize, edf.length()); } else { fail("Image/Edits directories are not different"); } } }
/** * Return a standalone instance of FSEditLog that will log into the given * log directory. The returned instance is not yet opened. */ public static FSEditLog createStandaloneEditLog(File logDir) throws IOException { assertTrue(logDir.mkdirs() || logDir.exists()); if (!FileUtil.fullyDeleteContents(logDir)) { throw new IOException("Unable to delete contents of " + logDir); } NNStorage storage = Mockito.mock(NNStorage.class); StorageDirectory sd = FSImageTestUtil.mockStorageDirectory(logDir, NameNodeDirType.EDITS); List<StorageDirectory> sds = Lists.newArrayList(sd); Mockito.doReturn(sds).when(storage).dirIterable(NameNodeDirType.EDITS); Mockito.doReturn(sd).when(storage) .getStorageDirectory(Matchers.<URI>anyObject()); FSEditLog editLog = new FSEditLog(new Configuration(), storage, ImmutableList.of(logDir.toURI())); editLog.initJournalsForWrite(); return editLog; }
/** * Assert that all of the given directories have the same newest filename * for fsimage that they hold the same data. */ public static void assertSameNewestImage(List<File> dirs) throws Exception { if (dirs.size() < 2) return; long imageTxId = -1; List<File> imageFiles = new ArrayList<File>(); for (File dir : dirs) { FSImageTransactionalStorageInspector inspector = inspectStorageDirectory(dir, NameNodeDirType.IMAGE); List<FSImageFile> latestImages = inspector.getLatestImages(); assert(!latestImages.isEmpty()); long thisTxId = latestImages.get(0).getCheckpointTxId(); if (imageTxId != -1 && thisTxId != imageTxId) { fail("Storage directory " + dir + " does not have the same " + "last image index " + imageTxId + " as another"); } imageTxId = thisTxId; imageFiles.add(inspector.getLatestImages().get(0).getFile()); } assertFileContentsSame(imageFiles.toArray(new File[0])); }
