/** * @return true if the given path is a snapshot path and the corresponding * INode is still in the current fsdirectory. */ private boolean isSnapshotPathInCurrent(String path) throws IOException { // if the parent path contains "/.snapshot/", this is a snapshot path if (path.contains(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR_SEPARATOR)) { String[] pathComponents = INode.getPathNames(path); if (HdfsConstants.DOT_SNAPSHOT_DIR .equals(pathComponents[pathComponents.length - 2])) { // this is a path for a specific snapshot (e.g., /foo/.snapshot/s1) return false; } String nonSnapshotPath = convertSnapshotPath(pathComponents); return dfs.getFileInfo(nonSnapshotPath) != null; } else { return false; } }
public QuotaCounts cleanFile(final BlockStoragePolicySuite bsps, final INodeFile file, final int snapshotId, int priorSnapshotId, final BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes) { if (snapshotId == Snapshot.CURRENT_STATE_ID) { // delete the current file while the file has snapshot feature if (!isCurrentFileDeleted()) { file.recordModification(priorSnapshotId); deleteCurrentFile(); } collectBlocksAndClear(bsps, file, collectedBlocks, removedINodes); return new QuotaCounts.Builder().build(); } else { // delete the snapshot priorSnapshotId = getDiffs().updatePrior(snapshotId, priorSnapshotId); return diffs.deleteSnapshotDiff(bsps, snapshotId, priorSnapshotId, file, collectedBlocks, removedINodes); } }
/** * If some blocks at the end of the block list no longer belongs to * any inode, collect them and update the block list. */ public void collectBlocksAndClear(final BlockStoragePolicySuite bsps, final INodeFile file, final BlocksMapUpdateInfo info, final List<INode> removedINodes) { // check if everything is deleted. if (isCurrentFileDeleted() && getDiffs().asList().isEmpty()) { file.destroyAndCollectBlocks(bsps, info, removedINodes); return; } // find max file size. final long max; FileDiff diff = getDiffs().getLast(); if (isCurrentFileDeleted()) { max = diff == null? 0: diff.getFileSize(); } else { max = file.computeFileSize(); } // Collect blocks that should be deleted FileDiff last = diffs.getLast(); BlockInfoContiguous[] snapshotBlocks = last == null ? null : last.getBlocks(); if(snapshotBlocks == null) file.collectBlocksBeyondMax(max, info); else file.collectBlocksBeyondSnapshot(snapshotBlocks, info); }
/** * Save SnapshotDiff list for an INodeDirectoryWithSnapshot. * @param sNode The directory that the SnapshotDiff list belongs to. * @param out The {@link DataOutput} to write. */ private static <N extends INode, A extends INodeAttributes, D extends AbstractINodeDiff<N, A, D>> void saveINodeDiffs(final AbstractINodeDiffList<N, A, D> diffs, final DataOutput out, ReferenceMap referenceMap) throws IOException { // Record the diffs in reversed order, so that we can find the correct // reference for INodes in the created list when loading the FSImage if (diffs == null) { out.writeInt(-1); // no diffs } else { final List<D> list = diffs.asList(); final int size = list.size(); out.writeInt(size); for (int i = size - 1; i >= 0; i--) { list.get(i).write(out, referenceMap); } } }
/** * Load a node stored in the created list from fsimage. * @param createdNodeName The name of the created node. * @param parent The directory that the created list belongs to. * @return The created node. */ public static INode loadCreated(byte[] createdNodeName, INodeDirectory parent) throws IOException { // the INode in the created list should be a reference to another INode // in posterior SnapshotDiffs or one of the current children for (DirectoryDiff postDiff : parent.getDiffs()) { final INode d = postDiff.getChildrenDiff().search(ListType.DELETED, createdNodeName); if (d != null) { return d; } // else go to the next SnapshotDiff } // use the current child INode currentChild = parent.getChild(createdNodeName, Snapshot.CURRENT_STATE_ID); if (currentChild == null) { throw new IOException("Cannot find an INode associated with the INode " + DFSUtil.bytes2String(createdNodeName) + " in created list while loading FSImage."); } return currentChild; }
/** * Load the deleted list from the fsimage. * * @param parent The directory that the deleted list belongs to. * @param createdList The created list associated with the deleted list in * the same Diff. * @param in The {@link DataInput} to read. * @param loader The {@link Loader} instance. * @return The deleted list. */ private static List<INode> loadDeletedList(INodeDirectory parent, List<INode> createdList, DataInput in, FSImageFormat.Loader loader) throws IOException { int deletedSize = in.readInt(); List<INode> deletedList = new ArrayList<INode>(deletedSize); for (int i = 0; i < deletedSize; i++) { final INode deleted = loader.loadINodeWithLocalName(true, in, true); deletedList.add(deleted); // set parent: the parent field of an INode in the deleted list is not // useful, but set the parent here to be consistent with the original // fsdir tree. deleted.setParent(parent); if (deleted.isFile()) { loader.updateBlocksMap(deleted.asFile()); } } return deletedList; }
/** * Load {@link DirectoryDiff} from fsimage. * @param parent The directory that the SnapshotDiff belongs to. * @param in The {@link DataInput} instance to read. * @param loader The {@link Loader} instance that this loading procedure is * using. * @return A {@link DirectoryDiff}. */ private static DirectoryDiff loadDirectoryDiff(INodeDirectory parent, DataInput in, FSImageFormat.Loader loader) throws IOException { // 1. Read the full path of the Snapshot root to identify the Snapshot final Snapshot snapshot = loader.getSnapshot(in); // 2. Load DirectoryDiff#childrenSize int childrenSize = in.readInt(); // 3. Load DirectoryDiff#snapshotINode INodeDirectoryAttributes snapshotINode = loadSnapshotINodeInDirectoryDiff( snapshot, in, loader); // 4. Load the created list in SnapshotDiff#Diff List<INode> createdList = loadCreatedList(parent, in); // 5. Load the deleted list in SnapshotDiff#Diff List<INode> deletedList = loadDeletedList(parent, createdList, in, loader); // 6. Compose the SnapshotDiff List<DirectoryDiff> diffs = parent.getDiffs().asList(); DirectoryDiff sdiff = new DirectoryDiff(snapshot.getId(), snapshotINode, diffs.isEmpty() ? null : diffs.get(0), childrenSize, createdList, deletedList, snapshotINode == snapshot.getRoot()); return sdiff; }
public void writeINodeReferenceWithCount( INodeReference.WithCount withCount, DataOutput out, boolean writeUnderConstruction) throws IOException { final INode referred = withCount.getReferredINode(); final long id = withCount.getId(); final boolean firstReferred = !referenceMap.containsKey(id); out.writeBoolean(firstReferred); if (firstReferred) { FSImageSerialization.saveINode2Image(referred, out, writeUnderConstruction, this); referenceMap.put(id, withCount); } else { out.writeLong(id); } }
public INodeReference.WithCount loadINodeReferenceWithCount( boolean isSnapshotINode, DataInput in, FSImageFormat.Loader loader ) throws IOException { final boolean firstReferred = in.readBoolean(); final INodeReference.WithCount withCount; if (firstReferred) { final INode referred = loader.loadINodeWithLocalName(isSnapshotINode, in, true); withCount = new INodeReference.WithCount(null, referred); referenceMap.put(withCount.getId(), withCount); } else { final long id = in.readLong(); withCount = referenceMap.get(id); } return withCount; }
/** * Generate a {@link SnapshotDiffReport} based on detailed diff information. * @return A {@link SnapshotDiffReport} describing the difference */ public SnapshotDiffReport generateReport() { List<DiffReportEntry> diffReportList = new ArrayList<DiffReportEntry>(); for (Map.Entry<INode,byte[][]> drEntry : diffMap.entrySet()) { INode node = drEntry.getKey(); byte[][] path = drEntry.getValue(); diffReportList.add(new DiffReportEntry(DiffType.MODIFY, path, null)); if (node.isDirectory()) { List<DiffReportEntry> subList = generateReport(dirDiffMap.get(node), path, isFromEarlier(), renameMap); diffReportList.addAll(subList); } } return new SnapshotDiffReport(snapshotRoot.getFullPathName(), Snapshot.getSnapshotName(from), Snapshot.getSnapshotName(to), diffReportList); }
/** clear the created list */ private QuotaCounts destroyCreatedList( final BlockStoragePolicySuite bsps, final INodeDirectory currentINode, final BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes) { QuotaCounts counts = new QuotaCounts.Builder().build(); final List<INode> createdList = getList(ListType.CREATED); for (INode c : createdList) { c.computeQuotaUsage(bsps, counts, true); c.destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes); // c should be contained in the children list, remove it currentINode.removeChild(c); } createdList.clear(); return counts; }
@Override QuotaCounts combinePosteriorAndCollectBlocks( final BlockStoragePolicySuite bsps, final INodeDirectory currentDir, final DirectoryDiff posterior, final BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes) { final QuotaCounts counts = new QuotaCounts.Builder().build(); diff.combinePosterior(posterior.diff, new Diff.Processor<INode>() { /** Collect blocks for deleted files. */ @Override public void process(INode inode) { if (inode != null) { inode.computeQuotaUsage(bsps, counts, false); inode.destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes); } } }); return counts; }
/** @return the child with the given name. */ INode getChild(byte[] name, boolean checkPosterior, INodeDirectory currentDir) { for(DirectoryDiff d = this; ; d = d.getPosterior()) { final Container<INode> returned = d.diff.accessPrevious(name); if (returned != null) { // the diff is able to determine the inode return returned.getElement(); } else if (!checkPosterior) { // Since checkPosterior is false, return null, i.e. not found. return null; } else if (d.getPosterior() == null) { // no more posterior diff, get from current inode. return currentDir.getChild(name, Snapshot.CURRENT_STATE_ID); } } }
/** * Remove an inode from parent's children list. The caller of this method * needs to make sure that parent is in the given snapshot "latest". */ public boolean removeChild(INodeDirectory parent, INode child, int latestSnapshotId) { // For a directory that is not a renamed node, if isInLatestSnapshot returns // false, the directory is not in the latest snapshot, thus we do not need // to record the removed child in any snapshot. // For a directory that was moved/renamed, note that if the directory is in // any of the previous snapshots, we will create a reference node for the // directory while rename, and isInLatestSnapshot will return true in that // scenario (if all previous snapshots have been deleted, isInLatestSnapshot // still returns false). Thus if isInLatestSnapshot returns false, the // directory node cannot be in any snapshot (not in current tree, nor in // previous src tree). Thus we do not need to record the removed child in // any snapshot. ChildrenDiff diff = diffs.checkAndAddLatestSnapshotDiff(latestSnapshotId, parent).diff; UndoInfo<INode> undoInfo = diff.delete(child); final boolean removed = parent.removeChild(child); if (!removed && undoInfo != null) { // remove failed, undo diff.undoDelete(child, undoInfo); } return removed; }
/** Used to record the modification of a symlink node */ public INode saveChild2Snapshot(INodeDirectory currentINode, final INode child, final int latestSnapshotId, final INode snapshotCopy) { Preconditions.checkArgument(!child.isDirectory(), "child is a directory, child=%s", child); Preconditions.checkArgument(latestSnapshotId != Snapshot.CURRENT_STATE_ID); final DirectoryDiff diff = diffs.checkAndAddLatestSnapshotDiff( latestSnapshotId, currentINode); if (diff.getChild(child.getLocalNameBytes(), false, currentINode) != null) { // it was already saved in the latest snapshot earlier. return child; } diff.diff.modify(snapshotCopy, child); return child; }
private INodeReference loadINodeReference( INodeReferenceSection.INodeReference r) throws IOException { long referredId = r.getReferredId(); INode referred = fsDir.getInode(referredId); WithCount withCount = (WithCount) referred.getParentReference(); if (withCount == null) { withCount = new INodeReference.WithCount(null, referred); } final INodeReference ref; if (r.hasDstSnapshotId()) { // DstReference ref = new INodeReference.DstReference(null, withCount, r.getDstSnapshotId()); } else { ref = new INodeReference.WithName(null, withCount, r.getName() .toByteArray(), r.getLastSnapshotId()); } return ref; }
/** * Load the snapshot diff section from fsimage. */ public void loadSnapshotDiffSection(InputStream in) throws IOException { final List<INodeReference> refList = parent.getLoaderContext() .getRefList(); while (true) { SnapshotDiffSection.DiffEntry entry = SnapshotDiffSection.DiffEntry .parseDelimitedFrom(in); if (entry == null) { break; } long inodeId = entry.getInodeId(); INode inode = fsDir.getInode(inodeId); SnapshotDiffSection.DiffEntry.Type type = entry.getType(); switch (type) { case FILEDIFF: loadFileDiffList(in, inode.asFile(), entry.getNumOfDiff()); break; case DIRECTORYDIFF: loadDirectoryDiffList(in, inode.asDirectory(), entry.getNumOfDiff(), refList); break; } } }
/** * When we have nested snapshottable directories and if we try to reset the * snapshottable descendant back to an regular directory, we need to replace * the snapshottable descendant with an INodeDirectoryWithSnapshot */ @Test public void testDisallowNestedSnapshottableDir() throws Exception { cluster.getNamesystem().getSnapshotManager().setAllowNestedSnapshots(true); final Path dir = new Path("/dir"); final Path sub = new Path(dir, "sub"); hdfs.mkdirs(sub); SnapshotTestHelper.createSnapshot(hdfs, dir, "s1"); final Path file = new Path(sub, "file"); DFSTestUtil.createFile(hdfs, file, BLOCKSIZE, REPLICATION, SEED); FSDirectory fsdir = cluster.getNamesystem().getFSDirectory(); INode subNode = fsdir.getINode(sub.toString()); assertTrue(subNode.asDirectory().isWithSnapshot()); hdfs.allowSnapshot(sub); subNode = fsdir.getINode(sub.toString()); assertTrue(subNode.isDirectory() && subNode.asDirectory().isSnapshottable()); hdfs.disallowSnapshot(sub); subNode = fsdir.getINode(sub.toString()); assertTrue(subNode.asDirectory().isWithSnapshot()); }
/** * Test rename from a non-snapshottable dir to a snapshottable dir */ @Test (timeout=60000) public void testRenameFromNonSDir2SDir() throws Exception { final Path sdir1 = new Path("/dir1"); final Path sdir2 = new Path("/dir2"); hdfs.mkdirs(sdir1); hdfs.mkdirs(sdir2); final Path foo = new Path(sdir1, "foo"); final Path bar = new Path(foo, "bar"); DFSTestUtil.createFile(hdfs, bar, BLOCKSIZE, REPL, SEED); SnapshotTestHelper.createSnapshot(hdfs, sdir2, snap1); final Path newfoo = new Path(sdir2, "foo"); hdfs.rename(foo, newfoo); INode fooNode = fsdir.getINode4Write(newfoo.toString()); assertTrue(fooNode instanceof INodeDirectory); }
static boolean hasIdenticalElements(final List<INode> expected, final List<INode> computed) { if (expected == null) { return computed == null; } if (expected.size() != computed.size()) { return false; } for(int i = 0; i < expected.size(); i++) { // must be the same object (equals is not enough) if (expected.get(i) != computed.get(i)) { return false; } } return true; }
static void create(INode inode, final List<INode> current, Diff<byte[], INode> diff) { final int i = Diff.search(current, inode.getKey()); Assert.assertTrue(i < 0); current.add(-i - 1, inode); if (diff != null) { //test undo with 1/UNDO_TEST_P probability final boolean testUndo = RANDOM.nextInt(UNDO_TEST_P) == 0; String before = null; if (testUndo) { before = diff.toString(); } final int undoInfo = diff.create(inode); if (testUndo) { final String after = diff.toString(); //undo diff.undoCreate(inode, undoInfo); assertDiff(before, diff); //re-do diff.create(inode); assertDiff(after, diff); } } }
static void delete(INode inode, final List<INode> current, Diff<byte[], INode> diff) { final int i = Diff.search(current, inode.getKey()); current.remove(i); if (diff != null) { //test undo with 1/UNDO_TEST_P probability final boolean testUndo = RANDOM.nextInt(UNDO_TEST_P) == 0; String before = null; if (testUndo) { before = diff.toString(); } final UndoInfo<INode> undoInfo = diff.delete(inode); if (testUndo) { final String after = diff.toString(); //undo diff.undoDelete(inode, undoInfo); assertDiff(before, diff); //re-do diff.delete(inode); assertDiff(after, diff); } } }
/** * Remove the snapshot with the given name from {@link #snapshotsByNames}, * and delete all the corresponding DirectoryDiff. * * @param reclaimContext records blocks and inodes that need to be reclaimed * @param snapshotRoot The directory where we take snapshots * @param snapshotName The name of the snapshot to be removed * @return The removed snapshot. Null if no snapshot with the given name * exists. */ public Snapshot removeSnapshot( INode.ReclaimContext reclaimContext, INodeDirectory snapshotRoot, String snapshotName) throws SnapshotException { final int i = searchSnapshot(DFSUtil.string2Bytes(snapshotName)); if (i < 0) { throw new SnapshotException("Cannot delete snapshot " + snapshotName + " from path " + snapshotRoot.getFullPathName() + ": the snapshot does not exist."); } else { final Snapshot snapshot = snapshotsByNames.get(i); int prior = Snapshot.findLatestSnapshot(snapshotRoot, snapshot.getId()); snapshotRoot.cleanSubtree(reclaimContext, snapshot.getId(), prior); // remove from snapshotsByNames after successfully cleaning the subtree snapshotsByNames.remove(i); return snapshot; } }
