Java 类org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType 实例源码

项目:hadoop    文件:TransferFsImage.java   
public static MD5Hash downloadImageToStorage(URL fsName, long imageTxId,
    Storage dstStorage, boolean needDigest) throws IOException {
  String fileid = ImageServlet.getParamStringForImage(null,
      imageTxId, dstStorage);
  String fileName = NNStorage.getCheckpointImageFileName(imageTxId);

  List<File> dstFiles = dstStorage.getFiles(
      NameNodeDirType.IMAGE, fileName);
  if (dstFiles.isEmpty()) {
    throw new IOException("No targets in destination storage!");
  }

  MD5Hash hash = getFileClient(fsName, fileid, dstFiles, dstStorage, needDigest);
  LOG.info("Downloaded file " + dstFiles.get(0).getName() + " size " +
      dstFiles.get(0).length() + " bytes.");
  return hash;
}
项目:hadoop    文件:TransferFsImage.java   
static MD5Hash handleUploadImageRequest(HttpServletRequest request,
    long imageTxId, Storage dstStorage, InputStream stream,
    long advertisedSize, DataTransferThrottler throttler) throws IOException {

  String fileName = NNStorage.getCheckpointImageFileName(imageTxId);

  List<File> dstFiles = dstStorage.getFiles(NameNodeDirType.IMAGE, fileName);
  if (dstFiles.isEmpty()) {
    throw new IOException("No targets in destination storage!");
  }

  MD5Hash advertisedDigest = parseMD5Header(request);
  MD5Hash hash = receiveFile(fileName, dstFiles, dstStorage, true,
      advertisedSize, advertisedDigest, fileName, stream, throttler);
  LOG.info("Downloaded file " + dstFiles.get(0).getName() + " size "
      + dstFiles.get(0).length() + " bytes.");
  return hash;
}
项目:hadoop    文件:FSImage.java   
/**
 * Rename FSImage with the specific txid
 */
private void renameCheckpoint(long txid, NameNodeFile fromNnf,
    NameNodeFile toNnf, boolean renameMD5) throws IOException {
  ArrayList<StorageDirectory> al = null;

  for (StorageDirectory sd : storage.dirIterable(NameNodeDirType.IMAGE)) {
    try {
      renameImageFileInDir(sd, fromNnf, toNnf, txid, renameMD5);
    } catch (IOException ioe) {
      LOG.warn("Unable to rename checkpoint in " + sd, ioe);
      if (al == null) {
        al = Lists.newArrayList();
      }
      al.add(sd);
    }
  }
  if(al != null) storage.reportErrorsOnDirectories(al);
}
项目:hadoop    文件:TestParallelImageWrite.java   
/**
 * Confirm that FSImage files in all StorageDirectory are the same,
 * and non-empty, and there are the expected number of them.
 * @param fsn - the FSNamesystem being checked.
 * @param numImageDirs - the configured number of StorageDirectory of type IMAGE. 
 * @return - the md5 hash of the most recent FSImage files, which must all be the same.
 * @throws AssertionError if image files are empty or different,
 *     if less than two StorageDirectory are provided, or if the
 *     actual number of StorageDirectory is less than configured.
 */
public static String checkImages(
    FSNamesystem fsn, int numImageDirs)
throws Exception {    
  NNStorage stg = fsn.getFSImage().getStorage();
  //any failed StorageDirectory is removed from the storageDirs list
  assertEquals("Some StorageDirectories failed Upgrade",
      numImageDirs, stg.getNumStorageDirs(NameNodeDirType.IMAGE));
  assertTrue("Not enough fsimage copies in MiniDFSCluster " + 
      "to test parallel write", numImageDirs > 1);

  // List of "current/" directory from each SD
  List<File> dirs = FSImageTestUtil.getCurrentDirs(stg, NameNodeDirType.IMAGE);

  // across directories, all files with same names should be identical hashes   
  FSImageTestUtil.assertParallelFilesAreIdentical(
      dirs, Collections.<String>emptySet());
  FSImageTestUtil.assertSameNewestImage(dirs);

  // Return the hash of the newest image file
  StorageDirectory firstSd = stg.dirIterator(NameNodeDirType.IMAGE).next();
  File latestImage = FSImageTestUtil.findLatestImageFile(firstSd);
  String md5 = FSImageTestUtil.getImageFileMD5IgnoringTxId(latestImage);
  System.err.println("md5 of " + latestImage + ": " + md5);
  return md5;
}
项目:hadoop    文件:TestNNStorageRetentionManager.java   
/**
 * Test the "easy case" where we have more images in the
 * directory than we need to keep. Should purge the
 * old ones.
 */
@Test
public void testPurgeEasyCase() throws IOException {
  TestCaseDescription tc = new TestCaseDescription();
  tc.addRoot("/foo1", NameNodeDirType.IMAGE_AND_EDITS);
  tc.addImage("/foo1/current/" + getImageFileName(100), true);
  tc.addImage("/foo1/current/" + getImageFileName(200), true);
  tc.addImage("/foo1/current/" + getImageFileName(300), false);
  tc.addImage("/foo1/current/" + getImageFileName(400), false);
  tc.addLog("/foo1/current/" + getFinalizedEditsFileName(101,200), true);
  tc.addLog("/foo1/current/" + getFinalizedEditsFileName(201,300), true);
  tc.addLog("/foo1/current/" + getFinalizedEditsFileName(301,400), false);
  tc.addLog("/foo1/current/" + getInProgressEditsFileName(401), false);

  // Test that other files don't get purged
  tc.addLog("/foo1/current/VERSION", false);
  runTest(tc);
}
项目:hadoop    文件:TestNNStorageRetentionManager.java   
/**
 * Same as above, but across multiple directories
 */
@Test
public void testPurgeMultipleDirs() throws IOException {
  TestCaseDescription tc = new TestCaseDescription();
  tc.addRoot("/foo1", NameNodeDirType.IMAGE_AND_EDITS);
  tc.addRoot("/foo2", NameNodeDirType.IMAGE_AND_EDITS);
  tc.addImage("/foo1/current/" + getImageFileName(100), true);
  tc.addImage("/foo1/current/" + getImageFileName(200), true);
  tc.addImage("/foo2/current/" + getImageFileName(200), true);
  tc.addImage("/foo1/current/" + getImageFileName(300), false);
  tc.addImage("/foo1/current/" + getImageFileName(400), false);
  tc.addLog("/foo1/current/" + getFinalizedEditsFileName(101, 200), true);
  tc.addLog("/foo1/current/" + getFinalizedEditsFileName(201, 300), true);
  tc.addLog("/foo2/current/" + getFinalizedEditsFileName(201, 300), true);
  tc.addLog("/foo1/current/" + getFinalizedEditsFileName(301, 400), false);
  tc.addLog("/foo2/current/" + getFinalizedEditsFileName(301, 400), false);
  tc.addLog("/foo1/current/" + getInProgressEditsFileName(401), false);
  runTest(tc);
}
项目:hadoop    文件:TestNNStorageRetentionManager.java   
@Test
public void testSeparateEditDirs() throws IOException {
  TestCaseDescription tc = new TestCaseDescription();
  tc.addRoot("/foo1", NameNodeDirType.IMAGE);
  tc.addRoot("/foo2", NameNodeDirType.EDITS);
  tc.addImage("/foo1/current/" + getImageFileName(100), true);
  tc.addImage("/foo1/current/" + getImageFileName(200), true);
  tc.addImage("/foo1/current/" + getImageFileName(300), false);
  tc.addImage("/foo1/current/" + getImageFileName(400), false);

  tc.addLog("/foo2/current/" + getFinalizedEditsFileName(101, 200), true);
  tc.addLog("/foo2/current/" + getFinalizedEditsFileName(201, 300), true);
  tc.addLog("/foo2/current/" + getFinalizedEditsFileName(301, 400), false);
  tc.addLog("/foo2/current/" + getInProgressEditsFileName(401), false);
  runTest(tc);    
}
项目:hadoop    文件:TestNNStorageRetentionManager.java   
@Test
public void testRetainExtraLogs() throws IOException {
  conf.setLong(DFSConfigKeys.DFS_NAMENODE_NUM_EXTRA_EDITS_RETAINED_KEY,
      50);
  TestCaseDescription tc = new TestCaseDescription();
  tc.addRoot("/foo1", NameNodeDirType.IMAGE);
  tc.addRoot("/foo2", NameNodeDirType.EDITS);
  tc.addImage("/foo1/current/" + getImageFileName(100), true);
  tc.addImage("/foo1/current/" + getImageFileName(200), true);
  tc.addImage("/foo1/current/" + getImageFileName(300), false);
  tc.addImage("/foo1/current/" + getImageFileName(400), false);

  tc.addLog("/foo2/current/" + getFinalizedEditsFileName(101, 200), true);
  // Since we need 50 extra edits, *do* retain the 201-300 segment 
  tc.addLog("/foo2/current/" + getFinalizedEditsFileName(201, 300), false);
  tc.addLog("/foo2/current/" + getFinalizedEditsFileName(301, 400), false);
  tc.addLog("/foo2/current/" + getInProgressEditsFileName(401), false);
  runTest(tc);
}
项目:hadoop    文件:TestFSImageStorageInspector.java   
/**
 * Simple test with image, edits, and inprogress edits
 */
@Test
public void testCurrentStorageInspector() throws IOException {
  FSImageTransactionalStorageInspector inspector = 
      new FSImageTransactionalStorageInspector();

  StorageDirectory mockDir = FSImageTestUtil.mockStorageDirectory(
      NameNodeDirType.IMAGE_AND_EDITS,
      false,
      "/foo/current/" + getImageFileName(123),
      "/foo/current/" + getFinalizedEditsFileName(123, 456),
      "/foo/current/" + getImageFileName(456),
      "/foo/current/" + getInProgressEditsFileName(457));

  inspector.inspectDirectory(mockDir);
  assertEquals(2, inspector.foundImages.size());

  FSImageFile latestImage = inspector.getLatestImages().get(0);
  assertEquals(456, latestImage.txId);
  assertSame(mockDir, latestImage.sd);
  assertTrue(inspector.isUpgradeFinalized());

  assertEquals(new File("/foo/current/"+getImageFileName(456)), 
      latestImage.getFile());
}
项目:hadoop    文件:TestFileJournalManager.java   
/** 
 * Test the normal operation of loading transactions from
 * file journal manager. 3 edits directories are setup without any
 * failures. Test that we read in the expected number of transactions.
 */
@Test
public void testNormalOperation() throws IOException {
  File f1 = new File(TestEditLog.TEST_DIR + "/normtest0");
  File f2 = new File(TestEditLog.TEST_DIR + "/normtest1");
  File f3 = new File(TestEditLog.TEST_DIR + "/normtest2");

  List<URI> editUris = ImmutableList.of(f1.toURI(), f2.toURI(), f3.toURI());
  NNStorage storage = setupEdits(editUris, 5);

  long numJournals = 0;
  for (StorageDirectory sd : storage.dirIterable(NameNodeDirType.EDITS)) {
    FileJournalManager jm = new FileJournalManager(conf, sd, storage);
    assertEquals(6*TXNS_PER_ROLL, getNumberOfTransactions(jm, 1, true, false));
    numJournals++;
  }
  assertEquals(3, numJournals);
}
项目:hadoop    文件:TestFileJournalManager.java   
/**
 * Test a mixture of inprogress files and finalised. Set up 3 edits 
 * directories and fail the second on the last roll. Verify that reading
 * the transactions, reads from the finalised directories.
 */
@Test
public void testInprogressRecoveryMixed() throws IOException {
  File f1 = new File(TestEditLog.TEST_DIR + "/mixtest0");
  File f2 = new File(TestEditLog.TEST_DIR + "/mixtest1");
  File f3 = new File(TestEditLog.TEST_DIR + "/mixtest2");

  List<URI> editUris = ImmutableList.of(f1.toURI(), f2.toURI(), f3.toURI());

  // abort after the 5th roll 
  NNStorage storage = setupEdits(editUris,
                                 5, new AbortSpec(5, 1));
  Iterator<StorageDirectory> dirs = storage.dirIterator(NameNodeDirType.EDITS);
  StorageDirectory sd = dirs.next();
  FileJournalManager jm = new FileJournalManager(conf, sd, storage);
  assertEquals(6*TXNS_PER_ROLL, getNumberOfTransactions(jm, 1, true, false));

  sd = dirs.next();
  jm = new FileJournalManager(conf, sd, storage);
  assertEquals(5*TXNS_PER_ROLL + TXNS_PER_FAIL, getNumberOfTransactions(jm, 1,
      true, false));

  sd = dirs.next();
  jm = new FileJournalManager(conf, sd, storage);
  assertEquals(6*TXNS_PER_ROLL, getNumberOfTransactions(jm, 1, true, false));
}
项目:hadoop    文件:TestFileJournalManager.java   
@Test(expected=IllegalStateException.class)
public void testFinalizeErrorReportedToNNStorage() throws IOException, InterruptedException {
  File f = new File(TestEditLog.TEST_DIR + "/filejournaltestError");
  // abort after 10th roll
  NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()),
                                 10, new AbortSpec(10, 0));
  StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();

  FileJournalManager jm = new FileJournalManager(conf, sd, storage);
  String sdRootPath = sd.getRoot().getAbsolutePath();
  FileUtil.chmod(sdRootPath, "-w", true);
  try {
    jm.finalizeLogSegment(0, 1);
  } finally {
    FileUtil.chmod(sdRootPath, "+w", true);
    assertTrue(storage.getRemovedStorageDirs().contains(sd));
  }
}
项目:hadoop    文件:TestFileJournalManager.java   
/** 
 * Test that we can read from a stream created by FileJournalManager.
 * Create a single edits directory, failing it on the final roll.
 * Then try loading from the point of the 3rd roll. Verify that we read 
 * the correct number of transactions from this point.
 */
@Test 
public void testReadFromStream() throws IOException {
  File f = new File(TestEditLog.TEST_DIR + "/readfromstream");
  // abort after 10th roll
  NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()),
                                 10, new AbortSpec(10, 0));
  StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();

  FileJournalManager jm = new FileJournalManager(conf, sd, storage);
  long expectedTotalTxnCount = TXNS_PER_ROLL*10 + TXNS_PER_FAIL;
  assertEquals(expectedTotalTxnCount, getNumberOfTransactions(jm, 1,
      true, false));

  long skippedTxns = (3*TXNS_PER_ROLL); // skip first 3 files
  long startingTxId = skippedTxns + 1; 

  long numLoadable = getNumberOfTransactions(jm, startingTxId,
      true, false);
  assertEquals(expectedTotalTxnCount - skippedTxns, numLoadable); 
}
项目:hadoop    文件:TestFileJournalManager.java   
/**
 * Make requests with starting transaction ids which don't match the beginning
 * txid of some log segments.
 * 
 * This should succeed.
 */
@Test
public void testAskForTransactionsMidfile() throws IOException {
  File f = new File(TestEditLog.TEST_DIR + "/askfortransactionsmidfile");
  NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()), 
                                 10);
  StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();

  FileJournalManager jm = new FileJournalManager(conf, sd, storage);

  // 10 rolls, so 11 rolled files, 110 txids total.
  final int TOTAL_TXIDS = 10 * 11;
  for (int txid = 1; txid <= TOTAL_TXIDS; txid++) {
    assertEquals((TOTAL_TXIDS - txid) + 1, getNumberOfTransactions(jm, txid,
        true, false));
  }
}
项目:hadoop    文件:TestFileJournalManager.java   
/** 
 * Test that we can load an edits directory with a corrupt inprogress file.
 * The corrupt inprogress file should be moved to the side.
 */
@Test
public void testManyLogsWithCorruptInprogress() throws IOException {
  File f = new File(TestEditLog.TEST_DIR + "/manylogswithcorruptinprogress");
  NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()), 10, new AbortSpec(10, 0));
  StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();

  File[] files = new File(f, "current").listFiles(new FilenameFilter() {
      @Override
      public boolean accept(File dir, String name) {
        if (name.startsWith("edits_inprogress")) {
          return true;
        }
        return false;
      }
    });
  assertEquals(files.length, 1);

  corruptAfterStartSegment(files[0]);

  FileJournalManager jm = new FileJournalManager(conf, sd, storage);
  assertEquals(10*TXNS_PER_ROLL+1, 
               getNumberOfTransactions(jm, 1, true, false));
}
项目:hadoop    文件:TestFileJournalManager.java   
@Test
public void testGetRemoteEditLog() throws IOException {
  StorageDirectory sd = FSImageTestUtil.mockStorageDirectory(
      NameNodeDirType.EDITS, false,
      NNStorage.getFinalizedEditsFileName(1, 100),
      NNStorage.getFinalizedEditsFileName(101, 200),
      NNStorage.getInProgressEditsFileName(201),
      NNStorage.getFinalizedEditsFileName(1001, 1100));

  // passing null for NNStorage because this unit test will not use it
  FileJournalManager fjm = new FileJournalManager(conf, sd, null);
  assertEquals("[1,100],[101,200],[1001,1100]", getLogsAsString(fjm, 1));
  assertEquals("[101,200],[1001,1100]", getLogsAsString(fjm, 101));
  assertEquals("[101,200],[1001,1100]", getLogsAsString(fjm, 150));
  assertEquals("[1001,1100]", getLogsAsString(fjm, 201));
  assertEquals("Asking for a newer log than exists should return empty list",
      "", getLogsAsString(fjm, 9999));
}
项目:hadoop    文件:TestFileJournalManager.java   
/**
 * Make sure that we starting reading the correct op when we request a stream
 * with a txid in the middle of an edit log file.
 */
@Test
public void testReadFromMiddleOfEditLog() throws CorruptionException,
    IOException {
  File f = new File(TestEditLog.TEST_DIR + "/readfrommiddleofeditlog");
  NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()), 
                                 10);
  StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();

  FileJournalManager jm = new FileJournalManager(conf, sd, storage);

  EditLogInputStream elis = getJournalInputStream(jm, 5, true);
  try {
    FSEditLogOp op = elis.readOp();
    assertEquals("read unexpected op", op.getTransactionId(), 5);
  } finally {
    IOUtils.cleanup(LOG, elis);
  }
}
项目:hadoop    文件:TestFileJournalManager.java   
/**
 * Make sure that in-progress streams aren't counted if we don't ask for
 * them.
 */
@Test
public void testExcludeInProgressStreams() throws CorruptionException,
    IOException {
  File f = new File(TestEditLog.TEST_DIR + "/excludeinprogressstreams");

  // Don't close the edit log once the files have been set up.
  NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()), 
                                 10, false);
  StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();

  FileJournalManager jm = new FileJournalManager(conf, sd, storage);

  // If we exclude the in-progess stream, we should only have 100 tx.
  assertEquals(100, getNumberOfTransactions(jm, 1, false, false));

  EditLogInputStream elis = getJournalInputStream(jm, 90, false);
  try {
    FSEditLogOp lastReadOp = null;
    while ((lastReadOp = elis.readOp()) != null) {
      assertTrue(lastReadOp.getTransactionId() <= 100);
    }
  } finally {
    IOUtils.cleanup(LOG, elis);
  }
}
项目:hadoop    文件:TestFileJournalManager.java   
/**
 * Tests that internal renames are done using native code on platforms that
 * have it.  The native rename includes more detailed information about the
 * failure, which can be useful for troubleshooting.
 */
@Test
public void testDoPreUpgradeIOError() throws IOException {
  File storageDir = new File(TestEditLog.TEST_DIR, "preupgradeioerror");
  List<URI> editUris = Collections.singletonList(storageDir.toURI());
  NNStorage storage = setupEdits(editUris, 5);
  StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();
  assertNotNull(sd);
  // Change storage directory so that renaming current to previous.tmp fails.
  FileUtil.setWritable(storageDir, false);
  FileJournalManager jm = null;
  try {
    jm = new FileJournalManager(conf, sd, storage);
    exception.expect(IOException.class);
    if (NativeCodeLoader.isNativeCodeLoaded()) {
      exception.expectMessage("failure in native rename");
    }
    jm.doPreUpgrade();
  } finally {
    IOUtils.cleanup(LOG, jm);
    // Restore permissions on storage directory and make sure we can delete.
    FileUtil.setWritable(storageDir, true);
    FileUtil.fullyDelete(storageDir);
  }
}
项目:hadoop    文件:TestStartup.java   
/**
 * verify that edits log and fsimage are in different directories and of a correct size
 */
private void verifyDifferentDirs(FSImage img, long expectedImgSize, long expectedEditsSize) {
  StorageDirectory sd =null;
  for (Iterator<StorageDirectory> it = img.getStorage().dirIterator(); it.hasNext();) {
    sd = it.next();

    if(sd.getStorageDirType().isOfType(NameNodeDirType.IMAGE)) {
      img.getStorage();
      File imf = NNStorage.getStorageFile(sd, NameNodeFile.IMAGE, 0);
      LOG.info("--image file " + imf.getAbsolutePath() + "; len = " + imf.length() + "; expected = " + expectedImgSize);
      assertEquals(expectedImgSize, imf.length());  
    } else if(sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) {
      img.getStorage();
      File edf = NNStorage.getStorageFile(sd, NameNodeFile.EDITS, 0);
      LOG.info("-- edits file " + edf.getAbsolutePath() + "; len = " + edf.length()  + "; expected = " + expectedEditsSize);
      assertEquals(expectedEditsSize, edf.length());    
    } else {
      fail("Image/Edits directories are not different");
    }
  }

}
项目:hadoop    文件:FSImageTestUtil.java   
/**
 * Return a standalone instance of FSEditLog that will log into the given
 * log directory. The returned instance is not yet opened.
 */
public static FSEditLog createStandaloneEditLog(File logDir)
    throws IOException {
  assertTrue(logDir.mkdirs() || logDir.exists());
  if (!FileUtil.fullyDeleteContents(logDir)) {
    throw new IOException("Unable to delete contents of " + logDir);
  }
  NNStorage storage = Mockito.mock(NNStorage.class);
  StorageDirectory sd 
    = FSImageTestUtil.mockStorageDirectory(logDir, NameNodeDirType.EDITS);
  List<StorageDirectory> sds = Lists.newArrayList(sd);
  Mockito.doReturn(sds).when(storage).dirIterable(NameNodeDirType.EDITS);
  Mockito.doReturn(sd).when(storage)
    .getStorageDirectory(Matchers.<URI>anyObject());

  FSEditLog editLog = new FSEditLog(new Configuration(), 
                       storage,
                       ImmutableList.of(logDir.toURI()));
  editLog.initJournalsForWrite();
  return editLog;
}
项目:hadoop    文件:FSImageTestUtil.java   
/**
 * Assert that all of the given directories have the same newest filename
 * for fsimage that they hold the same data.
 */
public static void assertSameNewestImage(List<File> dirs) throws Exception {
  if (dirs.size() < 2) return;

  long imageTxId = -1;

  List<File> imageFiles = new ArrayList<File>();
  for (File dir : dirs) {
    FSImageTransactionalStorageInspector inspector =
      inspectStorageDirectory(dir, NameNodeDirType.IMAGE);
    List<FSImageFile> latestImages = inspector.getLatestImages();
    assert(!latestImages.isEmpty());
    long thisTxId = latestImages.get(0).getCheckpointTxId();
    if (imageTxId != -1 && thisTxId != imageTxId) {
      fail("Storage directory " + dir + " does not have the same " +
          "last image index " + imageTxId + " as another");
    }
    imageTxId = thisTxId;
    imageFiles.add(inspector.getLatestImages().get(0).getFile());
  }

  assertFileContentsSame(imageFiles.toArray(new File[0]));
}
项目:aliyun-oss-hadoop-fs    文件:TransferFsImage.java   
static MD5Hash handleUploadImageRequest(HttpServletRequest request,
    long imageTxId, Storage dstStorage, InputStream stream,
    long advertisedSize, DataTransferThrottler throttler) throws IOException {

  String fileName = NNStorage.getCheckpointImageFileName(imageTxId);

  List<File> dstFiles = dstStorage.getFiles(NameNodeDirType.IMAGE, fileName);
  if (dstFiles.isEmpty()) {
    throw new IOException("No targets in destination storage!");
  }

  MD5Hash advertisedDigest = parseMD5Header(request);
  MD5Hash hash = receiveFile(fileName, dstFiles, dstStorage, true,
      advertisedSize, advertisedDigest, fileName, stream, throttler);
  LOG.info("Downloaded file " + dstFiles.get(0).getName() + " size "
      + dstFiles.get(0).length() + " bytes.");
  return hash;
}
项目:aliyun-oss-hadoop-fs    文件:FSImage.java   
/**
 * Rename FSImage with the specific txid
 */
private void renameCheckpoint(long txid, NameNodeFile fromNnf,
    NameNodeFile toNnf, boolean renameMD5) throws IOException {
  ArrayList<StorageDirectory> al = null;

  for (StorageDirectory sd : storage.dirIterable(NameNodeDirType.IMAGE)) {
    try {
      renameImageFileInDir(sd, fromNnf, toNnf, txid, renameMD5);
    } catch (IOException ioe) {
      LOG.warn("Unable to rename checkpoint in " + sd, ioe);
      if (al == null) {
        al = Lists.newArrayList();
      }
      al.add(sd);
    }
  }
  if(al != null) storage.reportErrorsOnDirectories(al);
}
项目:big-c    文件:TestStartup.java   
/**
 * verify that edits log and fsimage are in different directories and of a correct size
 */
private void verifyDifferentDirs(FSImage img, long expectedImgSize, long expectedEditsSize) {
  StorageDirectory sd =null;
  for (Iterator<StorageDirectory> it = img.getStorage().dirIterator(); it.hasNext();) {
    sd = it.next();

    if(sd.getStorageDirType().isOfType(NameNodeDirType.IMAGE)) {
      img.getStorage();
      File imf = NNStorage.getStorageFile(sd, NameNodeFile.IMAGE, 0);
      LOG.info("--image file " + imf.getAbsolutePath() + "; len = " + imf.length() + "; expected = " + expectedImgSize);
      assertEquals(expectedImgSize, imf.length());  
    } else if(sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) {
      img.getStorage();
      File edf = NNStorage.getStorageFile(sd, NameNodeFile.EDITS, 0);
      LOG.info("-- edits file " + edf.getAbsolutePath() + "; len = " + edf.length()  + "; expected = " + expectedEditsSize);
      assertEquals(expectedEditsSize, edf.length());    
    } else {
      fail("Image/Edits directories are not different");
    }
  }

}
项目:aliyun-oss-hadoop-fs    文件:TestNNStorageRetentionManager.java   
/**
 * Same as above, but across multiple directories
 */
@Test
public void testPurgeMultipleDirs() throws IOException {
  TestCaseDescription tc = new TestCaseDescription();
  tc.addRoot("/foo1", NameNodeDirType.IMAGE_AND_EDITS);
  tc.addRoot("/foo2", NameNodeDirType.IMAGE_AND_EDITS);
  tc.addImage("/foo1/current/" + getImageFileName(100), true);
  tc.addImage("/foo1/current/" + getImageFileName(200), true);
  tc.addImage("/foo2/current/" + getImageFileName(200), true);
  tc.addImage("/foo1/current/" + getImageFileName(300), false);
  tc.addImage("/foo1/current/" + getImageFileName(400), false);
  tc.addLog("/foo1/current/" + getFinalizedEditsFileName(101, 200), true);
  tc.addLog("/foo1/current/" + getFinalizedEditsFileName(201, 300), true);
  tc.addLog("/foo2/current/" + getFinalizedEditsFileName(201, 300), true);
  tc.addLog("/foo1/current/" + getFinalizedEditsFileName(301, 400), false);
  tc.addLog("/foo2/current/" + getFinalizedEditsFileName(301, 400), false);
  tc.addLog("/foo1/current/" + getInProgressEditsFileName(401), false);
  runTest(tc);
}
项目:aliyun-oss-hadoop-fs    文件:TestNNStorageRetentionManager.java   
@Test
public void testSeparateEditDirs() throws IOException {
  TestCaseDescription tc = new TestCaseDescription();
  tc.addRoot("/foo1", NameNodeDirType.IMAGE);
  tc.addRoot("/foo2", NameNodeDirType.EDITS);
  tc.addImage("/foo1/current/" + getImageFileName(100), true);
  tc.addImage("/foo1/current/" + getImageFileName(200), true);
  tc.addImage("/foo1/current/" + getImageFileName(300), false);
  tc.addImage("/foo1/current/" + getImageFileName(400), false);

  tc.addLog("/foo2/current/" + getFinalizedEditsFileName(101, 200), true);
  tc.addLog("/foo2/current/" + getFinalizedEditsFileName(201, 300), true);
  tc.addLog("/foo2/current/" + getFinalizedEditsFileName(301, 400), false);
  tc.addLog("/foo2/current/" + getInProgressEditsFileName(401), false);
  runTest(tc);    
}
项目:aliyun-oss-hadoop-fs    文件:TestNNStorageRetentionManager.java   
@Test
public void testRetainExtraLogs() throws IOException {
  conf.setLong(DFSConfigKeys.DFS_NAMENODE_NUM_EXTRA_EDITS_RETAINED_KEY,
      50);
  TestCaseDescription tc = new TestCaseDescription();
  tc.addRoot("/foo1", NameNodeDirType.IMAGE);
  tc.addRoot("/foo2", NameNodeDirType.EDITS);
  tc.addImage("/foo1/current/" + getImageFileName(100), true);
  tc.addImage("/foo1/current/" + getImageFileName(200), true);
  tc.addImage("/foo1/current/" + getImageFileName(300), false);
  tc.addImage("/foo1/current/" + getImageFileName(400), false);

  tc.addLog("/foo2/current/" + getFinalizedEditsFileName(101, 200), true);
  // Since we need 50 extra edits, *do* retain the 201-300 segment 
  tc.addLog("/foo2/current/" + getFinalizedEditsFileName(201, 300), false);
  tc.addLog("/foo2/current/" + getFinalizedEditsFileName(301, 400), false);
  tc.addLog("/foo2/current/" + getInProgressEditsFileName(401), false);
  runTest(tc);
}
项目:aliyun-oss-hadoop-fs    文件:TestFSImageStorageInspector.java   
/**
 * Simple test with image, edits, and inprogress edits
 */
@Test
public void testCurrentStorageInspector() throws IOException {
  FSImageTransactionalStorageInspector inspector = 
      new FSImageTransactionalStorageInspector();

  StorageDirectory mockDir = FSImageTestUtil.mockStorageDirectory(
      NameNodeDirType.IMAGE_AND_EDITS,
      false,
      "/foo/current/" + getImageFileName(123),
      "/foo/current/" + getFinalizedEditsFileName(123, 456),
      "/foo/current/" + getImageFileName(456),
      "/foo/current/" + getInProgressEditsFileName(457));

  inspector.inspectDirectory(mockDir);
  assertEquals(2, inspector.foundImages.size());

  FSImageFile latestImage = inspector.getLatestImages().get(0);
  assertEquals(456, latestImage.txId);
  assertSame(mockDir, latestImage.sd);
  assertTrue(inspector.isUpgradeFinalized());

  assertEquals(new File("/foo/current/"+getImageFileName(456)), 
      latestImage.getFile());
}
项目:aliyun-oss-hadoop-fs    文件:TestFileJournalManager.java   
/** 
 * Test the normal operation of loading transactions from
 * file journal manager. 3 edits directories are setup without any
 * failures. Test that we read in the expected number of transactions.
 */
@Test
public void testNormalOperation() throws IOException {
  File f1 = new File(TestEditLog.TEST_DIR + "/normtest0");
  File f2 = new File(TestEditLog.TEST_DIR + "/normtest1");
  File f3 = new File(TestEditLog.TEST_DIR + "/normtest2");

  List<URI> editUris = ImmutableList.of(f1.toURI(), f2.toURI(), f3.toURI());
  NNStorage storage = setupEdits(editUris, 5);

  long numJournals = 0;
  for (StorageDirectory sd : storage.dirIterable(NameNodeDirType.EDITS)) {
    FileJournalManager jm = new FileJournalManager(conf, sd, storage);
    assertEquals(6*TXNS_PER_ROLL, getNumberOfTransactions(jm, 1, true, false));
    numJournals++;
  }
  assertEquals(3, numJournals);
}
项目:aliyun-oss-hadoop-fs    文件:TestFileJournalManager.java   
/**
 * Test a mixture of inprogress files and finalised. Set up 3 edits 
 * directories and fail the second on the last roll. Verify that reading
 * the transactions, reads from the finalised directories.
 */
@Test
public void testInprogressRecoveryMixed() throws IOException {
  File f1 = new File(TestEditLog.TEST_DIR + "/mixtest0");
  File f2 = new File(TestEditLog.TEST_DIR + "/mixtest1");
  File f3 = new File(TestEditLog.TEST_DIR + "/mixtest2");

  List<URI> editUris = ImmutableList.of(f1.toURI(), f2.toURI(), f3.toURI());

  // abort after the 5th roll 
  NNStorage storage = setupEdits(editUris,
                                 5, new AbortSpec(5, 1));
  Iterator<StorageDirectory> dirs = storage.dirIterator(NameNodeDirType.EDITS);
  StorageDirectory sd = dirs.next();
  FileJournalManager jm = new FileJournalManager(conf, sd, storage);
  assertEquals(6*TXNS_PER_ROLL, getNumberOfTransactions(jm, 1, true, false));

  sd = dirs.next();
  jm = new FileJournalManager(conf, sd, storage);
  assertEquals(5*TXNS_PER_ROLL + TXNS_PER_FAIL, getNumberOfTransactions(jm, 1,
      true, false));

  sd = dirs.next();
  jm = new FileJournalManager(conf, sd, storage);
  assertEquals(6*TXNS_PER_ROLL, getNumberOfTransactions(jm, 1, true, false));
}
项目:aliyun-oss-hadoop-fs    文件:TestFileJournalManager.java   
/** 
 * Test that we can read from a stream created by FileJournalManager.
 * Create a single edits directory, failing it on the final roll.
 * Then try loading from the point of the 3rd roll. Verify that we read 
 * the correct number of transactions from this point.
 */
@Test 
public void testReadFromStream() throws IOException {
  File f = new File(TestEditLog.TEST_DIR + "/readfromstream");
  // abort after 10th roll
  NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()),
                                 10, new AbortSpec(10, 0));
  StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();

  FileJournalManager jm = new FileJournalManager(conf, sd, storage);
  long expectedTotalTxnCount = TXNS_PER_ROLL*10 + TXNS_PER_FAIL;
  assertEquals(expectedTotalTxnCount, getNumberOfTransactions(jm, 1,
      true, false));

  long skippedTxns = (3*TXNS_PER_ROLL); // skip first 3 files
  long startingTxId = skippedTxns + 1; 

  long numLoadable = getNumberOfTransactions(jm, startingTxId,
      true, false);
  assertEquals(expectedTotalTxnCount - skippedTxns, numLoadable); 
}
项目:aliyun-oss-hadoop-fs    文件:TestFileJournalManager.java   
/** 
 * Test that we can load an edits directory with a corrupt inprogress file.
 * The corrupt inprogress file should be moved to the side.
 */
@Test
public void testManyLogsWithCorruptInprogress() throws IOException {
  File f = new File(TestEditLog.TEST_DIR + "/manylogswithcorruptinprogress");
  NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()), 10, new AbortSpec(10, 0));
  StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();

  File[] files = new File(f, "current").listFiles(new FilenameFilter() {
      @Override
      public boolean accept(File dir, String name) {
        if (name.startsWith("edits_inprogress")) {
          return true;
        }
        return false;
      }
    });
  assertEquals(files.length, 1);

  corruptAfterStartSegment(files[0]);

  FileJournalManager jm = new FileJournalManager(conf, sd, storage);
  assertEquals(10*TXNS_PER_ROLL+1, 
               getNumberOfTransactions(jm, 1, true, false));
}
项目:aliyun-oss-hadoop-fs    文件:TestFileJournalManager.java   
@Test
public void testGetRemoteEditLog() throws IOException {
  StorageDirectory sd = FSImageTestUtil.mockStorageDirectory(
      NameNodeDirType.EDITS, false,
      NNStorage.getFinalizedEditsFileName(1, 100),
      NNStorage.getFinalizedEditsFileName(101, 200),
      NNStorage.getInProgressEditsFileName(201),
      NNStorage.getFinalizedEditsFileName(1001, 1100));

  // passing null for NNStorage because this unit test will not use it
  FileJournalManager fjm = new FileJournalManager(conf, sd, null);
  assertEquals("[1,100],[101,200],[1001,1100]", getLogsAsString(fjm, 1));
  assertEquals("[101,200],[1001,1100]", getLogsAsString(fjm, 101));
  assertEquals("[101,200],[1001,1100]", getLogsAsString(fjm, 150));
  assertEquals("[1001,1100]", getLogsAsString(fjm, 201));
  assertEquals("Asking for a newer log than exists should return empty list",
      "", getLogsAsString(fjm, 9999));
}
项目:aliyun-oss-hadoop-fs    文件:TestFileJournalManager.java   
/**
 * Make sure that we starting reading the correct op when we request a stream
 * with a txid in the middle of an edit log file.
 */
@Test
public void testReadFromMiddleOfEditLog() throws CorruptionException,
    IOException {
  File f = new File(TestEditLog.TEST_DIR + "/readfrommiddleofeditlog");
  NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()), 
                                 10);
  StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();

  FileJournalManager jm = new FileJournalManager(conf, sd, storage);

  EditLogInputStream elis = getJournalInputStream(jm, 5, true);
  try {
    FSEditLogOp op = elis.readOp();
    assertEquals("read unexpected op", op.getTransactionId(), 5);
  } finally {
    IOUtils.cleanup(LOG, elis);
  }
}
项目:aliyun-oss-hadoop-fs    文件:TestFileJournalManager.java   
/**
 * Make sure that in-progress streams aren't counted if we don't ask for
 * them.
 */
@Test
public void testExcludeInProgressStreams() throws CorruptionException,
    IOException {
  File f = new File(TestEditLog.TEST_DIR + "/excludeinprogressstreams");

  // Don't close the edit log once the files have been set up.
  NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()), 
                                 10, false);
  StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();

  FileJournalManager jm = new FileJournalManager(conf, sd, storage);

  // If we exclude the in-progess stream, we should only have 100 tx.
  assertEquals(100, getNumberOfTransactions(jm, 1, false, false));

  EditLogInputStream elis = getJournalInputStream(jm, 90, false);
  try {
    FSEditLogOp lastReadOp = null;
    while ((lastReadOp = elis.readOp()) != null) {
      assertTrue(lastReadOp.getTransactionId() <= 100);
    }
  } finally {
    IOUtils.cleanup(LOG, elis);
  }
}
项目:aliyun-oss-hadoop-fs    文件:TestFileJournalManager.java   
/**
 * Tests that internal renames are done using native code on platforms that
 * have it.  The native rename includes more detailed information about the
 * failure, which can be useful for troubleshooting.
 */
@Test
public void testDoPreUpgradeIOError() throws IOException {
  File storageDir = new File(TestEditLog.TEST_DIR, "preupgradeioerror");
  List<URI> editUris = Collections.singletonList(storageDir.toURI());
  NNStorage storage = setupEdits(editUris, 5);
  StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();
  assertNotNull(sd);
  // Change storage directory so that renaming current to previous.tmp fails.
  FileUtil.setWritable(storageDir, false);
  FileJournalManager jm = null;
  try {
    jm = new FileJournalManager(conf, sd, storage);
    exception.expect(IOException.class);
    if (NativeCodeLoader.isNativeCodeLoaded()) {
      exception.expectMessage("failure in native rename");
    }
    jm.doPreUpgrade();
  } finally {
    IOUtils.cleanup(LOG, jm);
    // Restore permissions on storage directory and make sure we can delete.
    FileUtil.setWritable(storageDir, true);
    FileUtil.fullyDelete(storageDir);
  }
}
项目:aliyun-oss-hadoop-fs    文件:TestStartup.java   
/**
 * verify that edits log and fsimage are in different directories and of a correct size
 */
private void verifyDifferentDirs(FSImage img, long expectedImgSize, long expectedEditsSize) {
  StorageDirectory sd =null;
  for (Iterator<StorageDirectory> it = img.getStorage().dirIterator(); it.hasNext();) {
    sd = it.next();

    if(sd.getStorageDirType().isOfType(NameNodeDirType.IMAGE)) {
      img.getStorage();
      File imf = NNStorage.getStorageFile(sd, NameNodeFile.IMAGE, 0);
      LOG.info("--image file " + imf.getAbsolutePath() + "; len = " + imf.length() + "; expected = " + expectedImgSize);
      assertEquals(expectedImgSize, imf.length());  
    } else if(sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) {
      img.getStorage();
      File edf = NNStorage.getStorageFile(sd, NameNodeFile.EDITS, 0);
      LOG.info("-- edits file " + edf.getAbsolutePath() + "; len = " + edf.length()  + "; expected = " + expectedEditsSize);
      assertEquals(expectedEditsSize, edf.length());    
    } else {
      fail("Image/Edits directories are not different");
    }
  }

}
项目:aliyun-oss-hadoop-fs    文件:FSImageTestUtil.java   
/**
 * Return a standalone instance of FSEditLog that will log into the given
 * log directory. The returned instance is not yet opened.
 */
public static FSEditLog createStandaloneEditLog(File logDir)
    throws IOException {
  assertTrue(logDir.mkdirs() || logDir.exists());
  if (!FileUtil.fullyDeleteContents(logDir)) {
    throw new IOException("Unable to delete contents of " + logDir);
  }
  NNStorage storage = Mockito.mock(NNStorage.class);
  StorageDirectory sd 
    = FSImageTestUtil.mockStorageDirectory(logDir, NameNodeDirType.EDITS);
  List<StorageDirectory> sds = Lists.newArrayList(sd);
  Mockito.doReturn(sds).when(storage).dirIterable(NameNodeDirType.EDITS);
  Mockito.doReturn(sd).when(storage)
    .getStorageDirectory(Matchers.<URI>anyObject());

  FSEditLog editLog = new FSEditLog(new Configuration(), 
                       storage,
                       ImmutableList.of(logDir.toURI()));
  editLog.initJournalsForWrite();
  return editLog;
}
项目:aliyun-oss-hadoop-fs    文件:FSImageTestUtil.java   
/**
 * Assert that all of the given directories have the same newest filename
 * for fsimage that they hold the same data.
 */
public static void assertSameNewestImage(List<File> dirs) throws Exception {
  if (dirs.size() < 2) return;

  long imageTxId = -1;

  List<File> imageFiles = new ArrayList<File>();
  for (File dir : dirs) {
    FSImageTransactionalStorageInspector inspector =
      inspectStorageDirectory(dir, NameNodeDirType.IMAGE);
    List<FSImageFile> latestImages = inspector.getLatestImages();
    assert(!latestImages.isEmpty());
    long thisTxId = latestImages.get(0).getCheckpointTxId();
    if (imageTxId != -1 && thisTxId != imageTxId) {
      fail("Storage directory " + dir + " does not have the same " +
          "last image index " + imageTxId + " as another");
    }
    imageTxId = thisTxId;
    imageFiles.add(inspector.getLatestImages().get(0).getFile());
  }

  assertFileContentsSame(imageFiles.toArray(new File[0]));
}