@Override // BlockCollection, the file should be under construction public BlockInfoUnderConstruction setLastBlock(BlockInfo lastBlock, DatanodeStorageInfo[] locations) throws IOException { Preconditions.checkState(isUnderConstruction(), "file is no longer under construction"); if (numBlocks() == 0) { throw new IOException("Failed to set last block: File is empty."); } BlockInfoUnderConstruction ucBlock = lastBlock.convertToBlockUnderConstruction( BlockUCState.UNDER_CONSTRUCTION, locations); ucBlock.setBlockCollection(this); setBlock(numBlocks() - 1, ucBlock); return ucBlock; }
/** * Compute file size of the current file. * * @param includesLastUcBlock * If the last block is under construction, should it be included? * @param usePreferredBlockSize4LastUcBlock * If the last block is under construction, should we use actual * block size or preferred block size? * Note that usePreferredBlockSize4LastUcBlock is ignored * if includesLastUcBlock == false. * @return file size */ public final long computeFileSize(boolean includesLastUcBlock, boolean usePreferredBlockSize4LastUcBlock) { if (blocks == null || blocks.length == 0) { return 0; } final int last = blocks.length - 1; //check if the last block is BlockInfoUnderConstruction long size = blocks[last].getNumBytes(); if (blocks[last] instanceof BlockInfoUnderConstruction) { if (!includesLastUcBlock) { size = 0; } else if (usePreferredBlockSize4LastUcBlock) { size = getPreferredBlockSize(); } } //sum other blocks for(int i = 0; i < last; i++) { size += blocks[i].getNumBytes(); } return size; }
/** * @return the node which is expected to run the recovery of the * given block, which is known to be under construction inside the * given NameNOde. */ public static DatanodeDescriptor getExpectedPrimaryNode(NameNode nn, ExtendedBlock blk) { BlockManager bm0 = nn.getNamesystem().getBlockManager(); BlockInfo storedBlock = bm0.getStoredBlock(blk.getLocalBlock()); assertTrue("Block " + blk + " should be under construction, " + "got: " + storedBlock, storedBlock instanceof BlockInfoUnderConstruction); BlockInfoUnderConstruction ucBlock = (BlockInfoUnderConstruction)storedBlock; // We expect that the replica with the most recent heart beat will be // the one to be in charge of the synchronization / recovery protocol. final DatanodeStorageInfo[] storages = ucBlock.getExpectedStorageLocations(); DatanodeStorageInfo expectedPrimary = storages[0]; long mostRecentLastUpdate = expectedPrimary.getDatanodeDescriptor().getLastUpdate(); for (int i = 1; i < storages.length; i++) { final long lastUpdate = storages[i].getDatanodeDescriptor().getLastUpdate(); if (lastUpdate > mostRecentLastUpdate) { expectedPrimary = storages[i]; mostRecentLastUpdate = lastUpdate; } } return expectedPrimary.getDatanodeDescriptor(); }
private FSNamesystem makeNameSystemSpy(Block block, INodeFileUnderConstruction file) throws IOException { Configuration conf = new Configuration(); FSImage image = new FSImage(conf); DatanodeDescriptor[] targets = new DatanodeDescriptor[0]; FSNamesystem namesystem = new FSNamesystem(conf, image); FSNamesystem namesystemSpy = spy(namesystem); BlockInfoUnderConstruction blockInfo = new BlockInfoUnderConstruction( block, 1, HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION, targets); blockInfo.setBlockCollection(file); blockInfo.setGenerationStamp(genStamp); blockInfo.initializeBlockRecovery(genStamp); doReturn(true).when(file).removeLastBlock(any(Block.class)); doReturn(blockInfo).when(namesystemSpy).getStoredBlock(any(Block.class)); doReturn("").when(namesystemSpy).closeFileCommitBlocks( any(INodeFileUnderConstruction.class), any(BlockInfo.class)); doReturn("").when(namesystemSpy).persistBlocks( any(INodeFileUnderConstruction.class), anyBoolean()); doReturn(mock(FSEditLog.class)).when(namesystemSpy).getEditLog(); return namesystemSpy; }
/** * @return the node which is expected to run the recovery of the * given block, which is known to be under construction inside the * given NameNOde. */ private DatanodeDescriptor getExpectedPrimaryNode(NameNode nn, ExtendedBlock blk) { BlockManager bm0 = nn.getNamesystem().getBlockManager(); BlockInfo storedBlock = bm0.getStoredBlock(blk.getLocalBlock()); assertTrue("Block " + blk + " should be under construction, " + "got: " + storedBlock, storedBlock instanceof BlockInfoUnderConstruction); BlockInfoUnderConstruction ucBlock = (BlockInfoUnderConstruction)storedBlock; // We expect that the replica with the most recent heart beat will be // the one to be in charge of the synchronization / recovery protocol. DatanodeDescriptor[] datanodes = ucBlock.getExpectedLocations(); DatanodeDescriptor expectedPrimary = datanodes[0]; long mostRecentLastUpdate = expectedPrimary.getLastUpdate(); for (int i = 1; i < datanodes.length; i++) { if (datanodes[i].getLastUpdate() > mostRecentLastUpdate) { expectedPrimary = datanodes[i]; mostRecentLastUpdate = expectedPrimary.getLastUpdate(); } } return expectedPrimary; }
static long computeFileSize(boolean includesBlockInfoUnderConstruction, BlockInfo[] blocks) throws StorageException { if (blocks == null || blocks.length == 0) { return 0; } final int last = blocks.length - 1; //check if the last block is BlockInfoUnderConstruction long bytes = 0; if(blocks[last] instanceof BlockInfoUnderConstruction){ if(includesBlockInfoUnderConstruction){ bytes = blocks[last].getNumBytes(); } }else{ bytes = blocks[last].getNumBytes(); } for (int i = 0; i < last; i++) { bytes += blocks[i].getNumBytes(); } return bytes; }
@Override public BlockInfo convertHDFStoDAL( org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo hdfsClass) throws StorageException { if (hdfsClass != null) { BlockInfo hopBlkInfo = new BlockInfo(hdfsClass.getBlockId(), hdfsClass.getBlockIndex(), hdfsClass.getInodeId(), hdfsClass.getNumBytes(), hdfsClass.getGenerationStamp(), hdfsClass.getBlockUCState().ordinal(), hdfsClass.getTimestamp()); if (hdfsClass instanceof BlockInfoUnderConstruction) { BlockInfoUnderConstruction ucBlock = (BlockInfoUnderConstruction) hdfsClass; hopBlkInfo.setPrimaryNodeIndex(ucBlock.getPrimaryNodeIndex()); hopBlkInfo.setBlockRecoveryId(ucBlock.getBlockRecoveryId()); } return hopBlkInfo; } else { return null; } }
public static INodeIdentifier resolveINodeFromBlock(final Block b) throws StorageException { if (b instanceof BlockInfo || b instanceof BlockInfoUnderConstruction) { INodeIdentifier inodeIden = new INodeIdentifier(((BlockInfo) b).getInodeId()); INodeDALAdaptor ida = (INodeDALAdaptor) HdfsStorageFactory .getDataAccess(INodeDataAccess.class); INode inode = ida.findInodeByIdFTIS(((BlockInfo) b).getInodeId()); if (inode != null) { inodeIden.setName(inode.getLocalName()); inodeIden.setPid(inode.getParentId()); inodeIden.setPartitionId(inode.getPartitionId()); } return inodeIden; } else { return resolveINodeFromBlockID(b.getBlockId()); } }
private FSNamesystem makeNameSystemSpy(Block block, INodeFile file) throws IOException { Configuration conf = new Configuration(); FSImage image = new FSImage(conf); final DatanodeStorageInfo[] targets = {}; FSNamesystem namesystem = new FSNamesystem(conf, image); FSNamesystem namesystemSpy = spy(namesystem); BlockInfoUnderConstruction blockInfo = new BlockInfoUnderConstruction( block, 1, HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION, targets); blockInfo.setBlockCollection(file); blockInfo.setGenerationStamp(genStamp); blockInfo.initializeBlockRecovery(genStamp); doReturn(true).when(file).removeLastBlock(any(Block.class)); doReturn(true).when(file).isUnderConstruction(); doReturn(blockInfo).when(namesystemSpy).getStoredBlock(any(Block.class)); doReturn("").when(namesystemSpy).closeFileCommitBlocks( any(INodeFile.class), any(BlockInfo.class)); doReturn("").when(namesystemSpy).persistBlocks( any(INodeFile.class), anyBoolean()); doReturn(mock(FSEditLog.class)).when(namesystemSpy).getEditLog(); return namesystemSpy; }
/** * @return the node which is expected to run the recovery of the * given block, which is known to be under construction inside the * given NameNOde. */ private DatanodeDescriptor getExpectedPrimaryNode(NameNode nn, ExtendedBlock blk) { BlockManager bm0 = nn.getNamesystem().getBlockManager(); BlockInfo storedBlock = bm0.getStoredBlock(blk.getLocalBlock()); assertTrue("Block " + blk + " should be under construction, " + "got: " + storedBlock, storedBlock instanceof BlockInfoUnderConstruction); BlockInfoUnderConstruction ucBlock = (BlockInfoUnderConstruction)storedBlock; // We expect that the replica with the most recent heart beat will be // the one to be in charge of the synchronization / recovery protocol. final DatanodeStorageInfo[] storages = ucBlock.getExpectedStorageLocations(); DatanodeStorageInfo expectedPrimary = storages[0]; long mostRecentLastUpdate = expectedPrimary.getDatanodeDescriptor().getLastUpdate(); for (int i = 1; i < storages.length; i++) { final long lastUpdate = storages[i].getDatanodeDescriptor().getLastUpdate(); if (lastUpdate > mostRecentLastUpdate) { expectedPrimary = storages[i]; mostRecentLastUpdate = lastUpdate; } } return expectedPrimary.getDatanodeDescriptor(); }
/** * Add a new block into the given INodeFile */ private void addNewBlock(FSDirectory fsDir, AddBlockOp op, INodeFile file) throws IOException { BlockInfo[] oldBlocks = file.getBlocks(); Block pBlock = op.getPenultimateBlock(); Block newBlock= op.getLastBlock(); if (pBlock != null) { // the penultimate block is not null Preconditions.checkState(oldBlocks != null && oldBlocks.length > 0); // compare pBlock with the last block of oldBlocks Block oldLastBlock = oldBlocks[oldBlocks.length - 1]; if (oldLastBlock.getBlockId() != pBlock.getBlockId() || oldLastBlock.getGenerationStamp() != pBlock.getGenerationStamp()) { throw new IOException( "Mismatched block IDs or generation stamps for the old last block of file " + op.getPath() + ", the old last block is " + oldLastBlock + ", and the block read from editlog is " + pBlock); } oldLastBlock.setNumBytes(pBlock.getNumBytes()); if (oldLastBlock instanceof BlockInfoUnderConstruction) { fsNamesys.getBlockManager().forceCompleteBlock(file, (BlockInfoUnderConstruction) oldLastBlock); fsNamesys.getBlockManager().processQueuedMessagesForBlock(pBlock); } } else { // the penultimate block is null Preconditions.checkState(oldBlocks == null || oldBlocks.length == 0); } // add the new block BlockInfo newBI = new BlockInfoUnderConstruction( newBlock, file.getBlockReplication()); fsNamesys.getBlockManager().addBlockCollection(newBI, file); file.addBlock(newBI); fsNamesys.getBlockManager().processQueuedMessagesForBlock(newBlock); }
/** * When deleting a file in the current fs directory, and the file is contained * in a snapshot, we should delete the last block if it's under construction * and its size is 0. */ void cleanZeroSizeBlock(final INodeFile f, final BlocksMapUpdateInfo collectedBlocks) { final BlockInfo[] blocks = f.getBlocks(); if (blocks != null && blocks.length > 0 && blocks[blocks.length - 1] instanceof BlockInfoUnderConstruction) { BlockInfoUnderConstruction lastUC = (BlockInfoUnderConstruction) blocks[blocks.length - 1]; if (lastUC.getNumBytes() == 0) { // this is a 0-sized block. do not need check its UC state here collectedBlocks.addDeleteBlock(lastUC); f.removeLastBlock(lastUC); } } }
/** * Add a block to the file. Returns a reference to the added block. */ BlockInfo addBlock(String path, INodesInPath inodesInPath, Block block, DatanodeStorageInfo[] targets) throws IOException { writeLock(); try { final INodeFile fileINode = inodesInPath.getLastINode().asFile(); Preconditions.checkState(fileINode.isUnderConstruction()); // check quota limits and updated space consumed updateCount(inodesInPath, 0, fileINode.getBlockDiskspace(), true); // associate new last block for the file BlockInfoUnderConstruction blockInfo = new BlockInfoUnderConstruction( block, fileINode.getFileReplication(), BlockUCState.UNDER_CONSTRUCTION, targets); getBlockManager().addBlockCollection(blockInfo, fileINode); fileINode.addBlock(blockInfo); if(NameNode.stateChangeLog.isDebugEnabled()) { NameNode.stateChangeLog.debug("DIR* FSDirectory.addBlock: " + path + " with " + block + " block is added to the in-memory " + "file system"); } return blockInfo; } finally { writeUnlock(); } }
@Override public boolean isInSnapshot(BlockInfoUnderConstruction blockUC) { assert hasReadLock(); final BlockCollection bc = blockUC.getBlockCollection(); if (bc == null || !(bc instanceof INodeFile) || !bc.isUnderConstruction()) { return false; } INodeFile inodeUC = (INodeFile) bc; String fullName = inodeUC.getName(); try { if (fullName != null && fullName.startsWith(Path.SEPARATOR) && dir.getINode(fullName) == inodeUC) { // If file exists in normal path then no need to look in snapshot return false; } } catch (UnresolvedLinkException e) { LOG.error("Error while resolving the link : " + fullName, e); return false; } /* * 1. if bc is an instance of INodeFileUnderConstructionWithSnapshot, and * bc is not in the current fsdirectory tree, bc must represent a snapshot * file. * 2. if fullName is not an absolute path, bc cannot be existent in the * current fsdirectory tree. * 3. if bc is not the current node associated with fullName, bc must be a * snapshot inode. */ return true; }
private FSNamesystem makeNameSystemSpy(Block block, INodeFile file) throws IOException { Configuration conf = new Configuration(); FSImage image = new FSImage(conf); final DatanodeStorageInfo[] targets = {}; FSNamesystem namesystem = new FSNamesystem(conf, image); namesystem.setImageLoaded(true); // set file's parent as root and put the file to inodeMap, so // FSNamesystem's isFileDeleted() method will return false on this file if (file.getParent() == null) { INodeDirectory mparent = mock(INodeDirectory.class); INodeDirectory parent = new INodeDirectory(mparent.getId(), new byte[0], mparent.getPermissionStatus(), mparent.getAccessTime()); parent.setLocalName(new byte[0]); parent.addChild(file); file.setParent(parent); } namesystem.dir.getINodeMap().put(file); FSNamesystem namesystemSpy = spy(namesystem); BlockInfoUnderConstruction blockInfo = new BlockInfoUnderConstruction( block, (short) 1, HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION, targets); blockInfo.setBlockCollection(file); blockInfo.setGenerationStamp(genStamp); blockInfo.initializeBlockRecovery(genStamp); doReturn(true).when(file).removeLastBlock(any(Block.class)); doReturn(true).when(file).isUnderConstruction(); doReturn(blockInfo).when(namesystemSpy).getStoredBlock(any(Block.class)); doReturn("").when(namesystemSpy).closeFileCommitBlocks( any(INodeFile.class), any(BlockInfo.class)); doReturn(mock(FSEditLog.class)).when(namesystemSpy).getEditLog(); return namesystemSpy; }
@Override boolean checkNamenodeBeforeReturn() throws Exception { INodeFile fileNode = cluster.getNamesystem(0).getFSDirectory() .getINode4Write(file).asFile(); BlockInfoUnderConstruction blkUC = (BlockInfoUnderConstruction) (fileNode.getBlocks())[1]; int datanodeNum = blkUC.getExpectedStorageLocations().length; for (int i = 0; i < CHECKTIMES && datanodeNum != 2; i++) { Thread.sleep(1000); datanodeNum = blkUC.getExpectedStorageLocations().length; } return datanodeNum == 2; }
/** * Test NameNode.getBlockLocations(..) on reading un-closed files. */ @Test public void testGetBlockLocations() throws IOException { final NamenodeProtocols namenode = cluster.getNameNodeRpc(); final Path p = new Path(BASE_DIR, "file2.dat"); final String src = p.toString(); final FSDataOutputStream out = TestFileCreation.createFile(hdfs, p, 3); // write a half block int len = BLOCK_SIZE >>> 1; writeFile(p, out, len); for(int i = 1; i < NUM_BLOCKS; ) { // verify consistency final LocatedBlocks lb = namenode.getBlockLocations(src, 0, len); final List<LocatedBlock> blocks = lb.getLocatedBlocks(); assertEquals(i, blocks.size()); final Block b = blocks.get(blocks.size() - 1).getBlock().getLocalBlock(); assertTrue(b instanceof BlockInfoUnderConstruction); if (++i < NUM_BLOCKS) { // write one more block writeFile(p, out, BLOCK_SIZE); len += BLOCK_SIZE; } } // close file out.close(); }
/** * Convert the last block of the file to an under-construction block. * Set its locations. */ @Override public BlockInfoUnderConstruction setLastBlock(BlockInfo lastBlock, DatanodeDescriptor[] targets) throws IOException { if (numBlocks() == 0) { throw new IOException("Failed to set last block: File is empty."); } BlockInfoUnderConstruction ucBlock = lastBlock.convertToBlockUnderConstruction( BlockUCState.UNDER_CONSTRUCTION, targets); ucBlock.setBlockCollection(this); setBlock(numBlocks()-1, ucBlock); return ucBlock; }
/** * Add a block to the file. Returns a reference to the added block. */ BlockInfo addBlock(String path, INodesInPath inodesInPath, Block block, DatanodeDescriptor targets[]) throws IOException { waitForReady(); writeLock(); try { final INodeFileUnderConstruction fileINode = INodeFileUnderConstruction.valueOf(inodesInPath.getLastINode(), path); // check quota limits and updated space consumed updateCount(inodesInPath, 0, fileINode.getBlockDiskspace(), true); // associate new last block for the file BlockInfoUnderConstruction blockInfo = new BlockInfoUnderConstruction( block, fileINode.getFileReplication(), BlockUCState.UNDER_CONSTRUCTION, targets); getBlockManager().addBlockCollection(blockInfo, fileINode); fileINode.addBlock(blockInfo); if(NameNode.stateChangeLog.isDebugEnabled()) { NameNode.stateChangeLog.debug("DIR* FSDirectory.addBlock: " + path + " with " + block + " block is added to the in-memory " + "file system"); } return blockInfo; } finally { writeUnlock(); } }
private FSNamesystem makeNameSystemSpy(Block block, INodeFile file) throws IOException { Configuration conf = new Configuration(); FSImage image = new FSImage(conf); final DatanodeStorageInfo[] targets = {}; FSNamesystem namesystem = new FSNamesystem(conf, image); namesystem.setImageLoaded(true); // set file's parent as root and put the file to inodeMap, so // FSNamesystem's isFileDeleted() method will return false on this file if (file.getParent() == null) { INodeDirectory parent = mock(INodeDirectory.class); parent.setLocalName(new byte[0]); parent.addChild(file); file.setParent(parent); } namesystem.dir.getINodeMap().put(file); FSNamesystem namesystemSpy = spy(namesystem); BlockInfoUnderConstruction blockInfo = new BlockInfoUnderConstruction( block, 1, HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION, targets); blockInfo.setBlockCollection(file); blockInfo.setGenerationStamp(genStamp); blockInfo.initializeBlockRecovery(genStamp); doReturn(true).when(file).removeLastBlock(any(Block.class)); doReturn(true).when(file).isUnderConstruction(); doReturn(blockInfo).when(namesystemSpy).getStoredBlock(any(Block.class)); doReturn("").when(namesystemSpy).closeFileCommitBlocks( any(INodeFile.class), any(BlockInfo.class)); doReturn(mock(FSEditLog.class)).when(namesystemSpy).getEditLog(); return namesystemSpy; }
/** * Convert the last block of the file to an under-construction block. * Set its locations. */ @Override public BlockInfoUnderConstruction setLastBlock(BlockInfo lastBlock, DatanodeDescriptor[] targets) throws IOException, StorageException { if (numBlocks() == 0) { throw new IOException("Failed to set last block: File is empty."); } BlockInfoUnderConstruction ucBlock = lastBlock .convertToBlockUnderConstruction(BlockUCState.UNDER_CONSTRUCTION, targets); ucBlock.setBlockCollection(this); setBlock(numBlocks() - 1, ucBlock); return ucBlock; }
