private void saveNameSystemSection(FileSummary.Builder summary) throws IOException { final FSNamesystem fsn = context.getSourceNamesystem(); OutputStream out = sectionOutputStream; BlockIdManager blockIdManager = fsn.getBlockIdManager(); NameSystemSection.Builder b = NameSystemSection.newBuilder() .setGenstampV1(blockIdManager.getGenerationStampV1()) .setGenstampV1Limit(blockIdManager.getGenerationStampV1Limit()) .setGenstampV2(blockIdManager.getGenerationStampV2()) .setLastAllocatedBlockId(blockIdManager.getLastAllocatedBlockId()) .setTransactionId(context.getTxId()); // We use the non-locked version of getNamespaceInfo here since // the coordinating thread of saveNamespace already has read-locked // the namespace for us. If we attempt to take another readlock // from the actual saver thread, there's a potential of a // fairness-related deadlock. See the comments on HDFS-2223. b.setNamespaceId(fsn.unprotectedGetNamespaceInfo().getNamespaceID()); if (fsn.isRollingUpgrade()) { b.setRollingUpgradeStartTime(fsn.getRollingUpgradeInfo().getStartTime()); } NameSystemSection s = b.build(); s.writeDelimitedTo(out); commitSection(summary, SectionName.NS_INFO); }
private void loadNameSystemSection(InputStream in) throws IOException { NameSystemSection s = NameSystemSection.parseDelimitedFrom(in); BlockIdManager blockIdManager = fsn.getBlockIdManager(); blockIdManager.setGenerationStampV1(s.getGenstampV1()); blockIdManager.setGenerationStampV2(s.getGenstampV2()); blockIdManager.setGenerationStampV1Limit(s.getGenstampV1Limit()); blockIdManager.setLastAllocatedContiguousBlockId(s.getLastAllocatedBlockId()); if (s.hasLastAllocatedStripedBlockId()) { blockIdManager.setLastAllocatedStripedBlockId( s.getLastAllocatedStripedBlockId()); } imgTxId = s.getTransactionId(); if (s.hasRollingUpgradeStartTime() && fsn.getFSImage().hasRollbackFSImage()) { // we set the rollingUpgradeInfo only when we make sure we have the // rollback image fsn.setRollingUpgradeInfo(true, s.getRollingUpgradeStartTime()); } }
@Test public void testParseDummyStripedBlock() { LocatedStripedBlock lsb = createDummyLocatedBlock( BLK_GROUP_STRIPE_NUM * FULL_STRIPE_SIZE); LocatedBlock[] blocks = parseStripedBlockGroup( lsb, CELLSIZE, DATA_BLK_NUM, PARITY_BLK_NUM); assertEquals(DATA_BLK_NUM + PARITY_BLK_NUM, blocks.length); for (int i = 0; i < DATA_BLK_NUM; i++) { assertFalse(blocks[i].isStriped()); assertEquals(i, BlockIdManager.getBlockIndex(blocks[i].getBlock().getLocalBlock())); assertEquals(0, blocks[i].getStartOffset()); assertEquals(1, blocks[i].getLocations().length); assertEquals(i, blocks[i].getLocations()[0].getIpcPort()); assertEquals(i, blocks[i].getLocations()[0].getXferPort()); } }
private void loadNameSystemSection(InputStream in) throws IOException { NameSystemSection s = NameSystemSection.parseDelimitedFrom(in); BlockIdManager blockIdManager = fsn.getBlockIdManager(); blockIdManager.setGenerationStampV1(s.getGenstampV1()); blockIdManager.setGenerationStampV2(s.getGenstampV2()); blockIdManager.setGenerationStampV1Limit(s.getGenstampV1Limit()); blockIdManager.setLastAllocatedBlockId(s.getLastAllocatedBlockId()); imgTxId = s.getTransactionId(); if (s.hasRollingUpgradeStartTime() && fsn.getFSImage().hasRollbackFSImage()) { // we set the rollingUpgradeInfo only when we make sure we have the // rollback image fsn.setRollingUpgradeInfo(true, s.getRollingUpgradeStartTime()); } }
private void saveNameSystemSection(FileSummary.Builder summary) throws IOException { final FSNamesystem fsn = context.getSourceNamesystem(); OutputStream out = sectionOutputStream; BlockIdManager blockIdManager = fsn.getBlockIdManager(); NameSystemSection.Builder b = NameSystemSection.newBuilder() .setGenstampV1(blockIdManager.getGenerationStampV1()) .setGenstampV1Limit(blockIdManager.getGenerationStampV1Limit()) .setGenstampV2(blockIdManager.getGenerationStampV2()) .setLastAllocatedBlockId(blockIdManager.getLastAllocatedContiguousBlockId()) .setLastAllocatedStripedBlockId(blockIdManager.getLastAllocatedStripedBlockId()) .setTransactionId(context.getTxId()); // We use the non-locked version of getNamespaceInfo here since // the coordinating thread of saveNamespace already has read-locked // the namespace for us. If we attempt to take another readlock // from the actual saver thread, there's a potential of a // fairness-related deadlock. See the comments on HDFS-2223. b.setNamespaceId(fsn.unprotectedGetNamespaceInfo().getNamespaceID()); if (fsn.isRollingUpgrade()) { b.setRollingUpgradeStartTime(fsn.getRollingUpgradeInfo().getStartTime()); } NameSystemSection s = b.build(); s.writeDelimitedTo(out); commitSection(summary, SectionName.NS_INFO); }
public BlockIdManager getBlockIdManager() { return blockIdManager; }
@Test(timeout=20000) public void testCancelSaveNamespace() throws Exception { Configuration conf = getConf(); NameNode.initMetrics(conf, NamenodeRole.NAMENODE); DFSTestUtil.formatNameNode(conf); FSNamesystem fsn = FSNamesystem.loadFromDisk(conf); // Replace the FSImage with a spy final FSImage image = fsn.getFSImage(); NNStorage storage = image.getStorage(); storage.close(); // unlock any directories that FSNamesystem's initialization may have locked storage.setStorageDirectories( FSNamesystem.getNamespaceDirs(conf), FSNamesystem.getNamespaceEditsDirs(conf)); FSNamesystem spyFsn = spy(fsn); final FSNamesystem finalFsn = spyFsn; DelayAnswer delayer = new GenericTestUtils.DelayAnswer(LOG); BlockIdManager bid = spy(spyFsn.getBlockIdManager()); Whitebox.setInternalState(finalFsn, "blockIdManager", bid); doAnswer(delayer).when(bid).getGenerationStampV2(); ExecutorService pool = Executors.newFixedThreadPool(2); try { doAnEdit(fsn, 1); final Canceler canceler = new Canceler(); // Save namespace fsn.setSafeMode(SafeModeAction.SAFEMODE_ENTER); try { Future<Void> saverFuture = pool.submit(new Callable<Void>() { @Override public Void call() throws Exception { image.saveNamespace(finalFsn, NameNodeFile.IMAGE, canceler); return null; } }); // Wait until saveNamespace calls getGenerationStamp delayer.waitForCall(); // then cancel the saveNamespace Future<Void> cancelFuture = pool.submit(new Callable<Void>() { @Override public Void call() throws Exception { canceler.cancel("cancelled"); return null; } }); // give the cancel call time to run Thread.sleep(500); // allow saveNamespace to proceed - it should check the cancel flag after // this point and throw an exception delayer.proceed(); cancelFuture.get(); saverFuture.get(); fail("saveNamespace did not fail even though cancelled!"); } catch (Throwable t) { GenericTestUtils.assertExceptionContains( "SaveNamespaceCancelledException", t); } LOG.info("Successfully cancelled a saveNamespace"); // Check that we have only the original image and not any // cruft left over from half-finished images FSImageTestUtil.logStorageContents(LOG, storage); for (StorageDirectory sd : storage.dirIterable(null)) { File curDir = sd.getCurrentDir(); GenericTestUtils.assertGlobEquals(curDir, "fsimage_.*", NNStorage.getImageFileName(0), NNStorage.getImageFileName(0) + MD5FileUtils.MD5_SUFFIX); } } finally { fsn.close(); } }
