/** * Generate a {@link SnapshotDiffReport} based on detailed diff information. * @return A {@link SnapshotDiffReport} describing the difference */ public SnapshotDiffReport generateReport() { List<DiffReportEntry> diffReportList = new ArrayList<DiffReportEntry>(); for (INode node : diffMap.keySet()) { diffReportList.add(new DiffReportEntry(DiffType.MODIFY, diffMap .get(node))); if (node.isDirectory()) { ChildrenDiff dirDiff = dirDiffMap.get(node); List<DiffReportEntry> subList = dirDiff.generateReport( diffMap.get(node), isFromEarlier()); diffReportList.addAll(subList); } } return new SnapshotDiffReport(snapshotRoot.getFullPathName(), Snapshot.getSnapshotName(from), Snapshot.getSnapshotName(to), diffReportList); }
/** * Test the undo section of rename. Before the rename, we create the renamed * file/dir before taking the snapshot. */ @Test public void testRenameUndo_1() throws Exception { final Path sdir1 = new Path("/dir1"); final Path sdir2 = new Path("/dir2"); hdfs.mkdirs(sdir1); hdfs.mkdirs(sdir2); final Path foo = new Path(sdir1, "foo"); final Path bar = new Path(foo, "bar"); DFSTestUtil.createFile(hdfs, bar, BLOCKSIZE, REPL, SEED); final Path dir2file = new Path(sdir2, "file"); DFSTestUtil.createFile(hdfs, dir2file, BLOCKSIZE, REPL, SEED); SnapshotTestHelper.createSnapshot(hdfs, sdir1, "s1"); INodeDirectory dir2 = fsdir.getINode4Write(sdir2.toString()).asDirectory(); INodeDirectory mockDir2 = spy(dir2); doReturn(false).when(mockDir2).addChild((INode) anyObject(), anyBoolean(), Mockito.anyInt()); INodeDirectory root = fsdir.getINode4Write("/").asDirectory(); root.replaceChild(dir2, mockDir2, fsdir.getINodeMap()); final Path newfoo = new Path(sdir2, "foo"); boolean result = hdfs.rename(foo, newfoo); assertFalse(result); // check the current internal details INodeDirectory dir1Node = fsdir.getINode4Write(sdir1.toString()) .asDirectory(); Snapshot s1 = dir1Node.getSnapshot(DFSUtil.string2Bytes("s1")); ReadOnlyList<INode> dir1Children = dir1Node .getChildrenList(Snapshot.CURRENT_STATE_ID); assertEquals(1, dir1Children.size()); assertEquals(foo.getName(), dir1Children.get(0).getLocalName()); List<DirectoryDiff> dir1Diffs = dir1Node.getDiffs().asList(); assertEquals(1, dir1Diffs.size()); assertEquals(s1.getId(), dir1Diffs.get(0).getSnapshotId()); // after the undo of rename, both the created and deleted list of sdir1 // should be empty ChildrenDiff childrenDiff = dir1Diffs.get(0).getChildrenDiff(); assertEquals(0, childrenDiff.getList(ListType.DELETED).size()); assertEquals(0, childrenDiff.getList(ListType.CREATED).size()); INode fooNode = fsdir.getINode4Write(foo.toString()); assertTrue(fooNode.isDirectory() && fooNode.asDirectory().isWithSnapshot()); List<DirectoryDiff> fooDiffs = fooNode.asDirectory().getDiffs().asList(); assertEquals(1, fooDiffs.size()); assertEquals(s1.getId(), fooDiffs.get(0).getSnapshotId()); final Path foo_s1 = SnapshotTestHelper.getSnapshotPath(sdir1, "s1", "foo"); INode fooNode_s1 = fsdir.getINode(foo_s1.toString()); assertTrue(fooNode_s1 == fooNode); // check sdir2 assertFalse(hdfs.exists(newfoo)); INodeDirectory dir2Node = fsdir.getINode4Write(sdir2.toString()) .asDirectory(); assertFalse(dir2Node.isWithSnapshot()); ReadOnlyList<INode> dir2Children = dir2Node .getChildrenList(Snapshot.CURRENT_STATE_ID); assertEquals(1, dir2Children.size()); assertEquals(dir2file.getName(), dir2Children.get(0).getLocalName()); }
/** * Test the undo section of rename. Before the rename, we create the renamed * file/dir after taking the snapshot. */ @Test public void testRenameUndo_2() throws Exception { final Path sdir1 = new Path("/dir1"); final Path sdir2 = new Path("/dir2"); hdfs.mkdirs(sdir1); hdfs.mkdirs(sdir2); final Path dir2file = new Path(sdir2, "file"); DFSTestUtil.createFile(hdfs, dir2file, BLOCKSIZE, REPL, SEED); SnapshotTestHelper.createSnapshot(hdfs, sdir1, "s1"); // create foo after taking snapshot final Path foo = new Path(sdir1, "foo"); final Path bar = new Path(foo, "bar"); DFSTestUtil.createFile(hdfs, bar, BLOCKSIZE, REPL, SEED); INodeDirectory dir2 = fsdir.getINode4Write(sdir2.toString()).asDirectory(); INodeDirectory mockDir2 = spy(dir2); doReturn(false).when(mockDir2).addChild((INode) anyObject(), anyBoolean(), Mockito.anyInt()); INodeDirectory root = fsdir.getINode4Write("/").asDirectory(); root.replaceChild(dir2, mockDir2, fsdir.getINodeMap()); final Path newfoo = new Path(sdir2, "foo"); boolean result = hdfs.rename(foo, newfoo); assertFalse(result); // check the current internal details INodeDirectory dir1Node = fsdir.getINode4Write(sdir1.toString()) .asDirectory(); Snapshot s1 = dir1Node.getSnapshot(DFSUtil.string2Bytes("s1")); ReadOnlyList<INode> dir1Children = dir1Node .getChildrenList(Snapshot.CURRENT_STATE_ID); assertEquals(1, dir1Children.size()); assertEquals(foo.getName(), dir1Children.get(0).getLocalName()); List<DirectoryDiff> dir1Diffs = dir1Node.getDiffs().asList(); assertEquals(1, dir1Diffs.size()); assertEquals(s1.getId(), dir1Diffs.get(0).getSnapshotId()); // after the undo of rename, the created list of sdir1 should contain // 1 element ChildrenDiff childrenDiff = dir1Diffs.get(0).getChildrenDiff(); assertEquals(0, childrenDiff.getList(ListType.DELETED).size()); assertEquals(1, childrenDiff.getList(ListType.CREATED).size()); INode fooNode = fsdir.getINode4Write(foo.toString()); assertTrue(fooNode instanceof INodeDirectory); assertTrue(childrenDiff.getList(ListType.CREATED).get(0) == fooNode); final Path foo_s1 = SnapshotTestHelper.getSnapshotPath(sdir1, "s1", "foo"); assertFalse(hdfs.exists(foo_s1)); // check sdir2 assertFalse(hdfs.exists(newfoo)); INodeDirectory dir2Node = fsdir.getINode4Write(sdir2.toString()) .asDirectory(); assertFalse(dir2Node.isWithSnapshot()); ReadOnlyList<INode> dir2Children = dir2Node .getChildrenList(Snapshot.CURRENT_STATE_ID); assertEquals(1, dir2Children.size()); assertEquals(dir2file.getName(), dir2Children.get(0).getLocalName()); }
/** * After the following operations: * Rename a dir -> create a snapshot s on dst tree -> delete the renamed dir * -> delete snapshot s on dst tree * * Make sure we destroy everything created after the rename under the renamed * dir. */ @Test public void testRenameDirAndDeleteSnapshot_3() throws Exception { final Path sdir1 = new Path("/dir1"); final Path sdir2 = new Path("/dir2"); final Path foo = new Path(sdir1, "foo"); final Path bar = new Path(foo, "bar"); DFSTestUtil.createFile(hdfs, bar, BLOCKSIZE, REPL, SEED); hdfs.mkdirs(sdir2); SnapshotTestHelper.createSnapshot(hdfs, sdir1, "s1"); SnapshotTestHelper.createSnapshot(hdfs, sdir2, "s2"); final Path foo2 = new Path(sdir2, "foo"); hdfs.rename(foo, foo2); // create two new files under foo2 final Path bar2 = new Path(foo2, "bar2"); DFSTestUtil.createFile(hdfs, bar2, BLOCKSIZE, REPL, SEED); final Path bar3 = new Path(foo2, "bar3"); DFSTestUtil.createFile(hdfs, bar3, BLOCKSIZE, REPL, SEED); // create a new snapshot on sdir2 hdfs.createSnapshot(sdir2, "s3"); // delete foo2 hdfs.delete(foo2, true); // delete s3 hdfs.deleteSnapshot(sdir2, "s3"); // check final INodeDirectory dir1Node = fsdir.getINode4Write(sdir1.toString()) .asDirectory(); QuotaCounts q1 = dir1Node.getDirectoryWithQuotaFeature().getSpaceConsumed(); assertEquals(3, q1.getNameSpace()); final INodeDirectory dir2Node = fsdir.getINode4Write(sdir2.toString()) .asDirectory(); QuotaCounts q2 = dir2Node.getDirectoryWithQuotaFeature().getSpaceConsumed(); assertEquals(1, q2.getNameSpace()); final Path foo_s1 = SnapshotTestHelper.getSnapshotPath(sdir1, "s1", foo.getName()); INode fooRef = fsdir.getINode(foo_s1.toString()); assertTrue(fooRef instanceof INodeReference.WithName); INodeReference.WithCount wc = (WithCount) fooRef.asReference().getReferredINode(); assertEquals(1, wc.getReferenceCount()); INodeDirectory fooNode = wc.getReferredINode().asDirectory(); ReadOnlyList<INode> children = fooNode .getChildrenList(Snapshot.CURRENT_STATE_ID); assertEquals(1, children.size()); assertEquals(bar.getName(), children.get(0).getLocalName()); List<DirectoryDiff> diffList = fooNode.getDiffs().asList(); assertEquals(1, diffList.size()); Snapshot s1 = dir1Node.getSnapshot(DFSUtil.string2Bytes("s1")); assertEquals(s1.getId(), diffList.get(0).getSnapshotId()); ChildrenDiff diff = diffList.get(0).getChildrenDiff(); assertEquals(0, diff.getList(ListType.CREATED).size()); assertEquals(0, diff.getList(ListType.DELETED).size()); restartClusterAndCheckImage(true); }
/** * After the following operations: * Rename a dir -> create a snapshot s on dst tree -> rename the renamed dir * again -> delete snapshot s on dst tree * * Make sure we only delete the snapshot s under the renamed dir. */ @Test public void testRenameDirAndDeleteSnapshot_4() throws Exception { final Path sdir1 = new Path("/dir1"); final Path sdir2 = new Path("/dir2"); final Path foo = new Path(sdir1, "foo"); final Path bar = new Path(foo, "bar"); DFSTestUtil.createFile(hdfs, bar, BLOCKSIZE, REPL, SEED); hdfs.mkdirs(sdir2); SnapshotTestHelper.createSnapshot(hdfs, sdir1, "s1"); SnapshotTestHelper.createSnapshot(hdfs, sdir2, "s2"); final Path foo2 = new Path(sdir2, "foo"); hdfs.rename(foo, foo2); // create two new files under foo2 final Path bar2 = new Path(foo2, "bar2"); DFSTestUtil.createFile(hdfs, bar2, BLOCKSIZE, REPL, SEED); final Path bar3 = new Path(foo2, "bar3"); DFSTestUtil.createFile(hdfs, bar3, BLOCKSIZE, REPL, SEED); // create a new snapshot on sdir2 hdfs.createSnapshot(sdir2, "s3"); // rename foo2 again hdfs.rename(foo2, foo); // delete snapshot s3 hdfs.deleteSnapshot(sdir2, "s3"); // check final INodeDirectory dir1Node = fsdir.getINode4Write(sdir1.toString()) .asDirectory(); // sdir1 + s1 + foo_s1 (foo) + foo (foo + s1 + bar~bar3) QuotaCounts q1 = dir1Node.getDirectoryWithQuotaFeature().getSpaceConsumed(); assertEquals(7, q1.getNameSpace()); final INodeDirectory dir2Node = fsdir.getINode4Write(sdir2.toString()) .asDirectory(); QuotaCounts q2 = dir2Node.getDirectoryWithQuotaFeature().getSpaceConsumed(); assertEquals(1, q2.getNameSpace()); final Path foo_s1 = SnapshotTestHelper.getSnapshotPath(sdir1, "s1", foo.getName()); final INode fooRef = fsdir.getINode(foo_s1.toString()); assertTrue(fooRef instanceof INodeReference.WithName); INodeReference.WithCount wc = (WithCount) fooRef.asReference().getReferredINode(); assertEquals(2, wc.getReferenceCount()); INodeDirectory fooNode = wc.getReferredINode().asDirectory(); ReadOnlyList<INode> children = fooNode .getChildrenList(Snapshot.CURRENT_STATE_ID); assertEquals(3, children.size()); assertEquals(bar.getName(), children.get(0).getLocalName()); assertEquals(bar2.getName(), children.get(1).getLocalName()); assertEquals(bar3.getName(), children.get(2).getLocalName()); List<DirectoryDiff> diffList = fooNode.getDiffs().asList(); assertEquals(1, diffList.size()); Snapshot s1 = dir1Node.getSnapshot(DFSUtil.string2Bytes("s1")); assertEquals(s1.getId(), diffList.get(0).getSnapshotId()); ChildrenDiff diff = diffList.get(0).getChildrenDiff(); // bar2 and bar3 in the created list assertEquals(2, diff.getList(ListType.CREATED).size()); assertEquals(0, diff.getList(ListType.DELETED).size()); final INode fooRef2 = fsdir.getINode4Write(foo.toString()); assertTrue(fooRef2 instanceof INodeReference.DstReference); INodeReference.WithCount wc2 = (WithCount) fooRef2.asReference().getReferredINode(); assertSame(wc, wc2); assertSame(fooRef2, wc.getParentReference()); restartClusterAndCheckImage(true); }
/** * After the following operations: * Rename a dir -> create a snapshot s on dst tree -> delete the renamed dir * -> delete snapshot s on dst tree * * Make sure we destroy everything created after the rename under the renamed * dir. */ @Test public void testRenameDirAndDeleteSnapshot_3() throws Exception { final Path sdir1 = new Path("/dir1"); final Path sdir2 = new Path("/dir2"); final Path foo = new Path(sdir1, "foo"); final Path bar = new Path(foo, "bar"); DFSTestUtil.createFile(hdfs, bar, BLOCKSIZE, REPL, SEED); hdfs.mkdirs(sdir2); SnapshotTestHelper.createSnapshot(hdfs, sdir1, "s1"); SnapshotTestHelper.createSnapshot(hdfs, sdir2, "s2"); final Path foo2 = new Path(sdir2, "foo"); hdfs.rename(foo, foo2); // create two new files under foo2 final Path bar2 = new Path(foo2, "bar2"); DFSTestUtil.createFile(hdfs, bar2, BLOCKSIZE, REPL, SEED); final Path bar3 = new Path(foo2, "bar3"); DFSTestUtil.createFile(hdfs, bar3, BLOCKSIZE, REPL, SEED); // create a new snapshot on sdir2 hdfs.createSnapshot(sdir2, "s3"); // delete foo2 hdfs.delete(foo2, true); // delete s3 hdfs.deleteSnapshot(sdir2, "s3"); // check final INodeDirectory dir1Node = fsdir.getINode4Write(sdir1.toString()) .asDirectory(); Quota.Counts q1 = dir1Node.getDirectoryWithQuotaFeature().getSpaceConsumed(); assertEquals(3, q1.get(Quota.NAMESPACE)); final INodeDirectory dir2Node = fsdir.getINode4Write(sdir2.toString()) .asDirectory(); Quota.Counts q2 = dir2Node.getDirectoryWithQuotaFeature().getSpaceConsumed(); assertEquals(1, q2.get(Quota.NAMESPACE)); final Path foo_s1 = SnapshotTestHelper.getSnapshotPath(sdir1, "s1", foo.getName()); INode fooRef = fsdir.getINode(foo_s1.toString()); assertTrue(fooRef instanceof INodeReference.WithName); INodeReference.WithCount wc = (WithCount) fooRef.asReference().getReferredINode(); assertEquals(1, wc.getReferenceCount()); INodeDirectory fooNode = wc.getReferredINode().asDirectory(); ReadOnlyList<INode> children = fooNode .getChildrenList(Snapshot.CURRENT_STATE_ID); assertEquals(1, children.size()); assertEquals(bar.getName(), children.get(0).getLocalName()); List<DirectoryDiff> diffList = fooNode.getDiffs().asList(); assertEquals(1, diffList.size()); Snapshot s1 = dir1Node.getSnapshot(DFSUtil.string2Bytes("s1")); assertEquals(s1.getId(), diffList.get(0).getSnapshotId()); ChildrenDiff diff = diffList.get(0).getChildrenDiff(); assertEquals(0, diff.getList(ListType.CREATED).size()); assertEquals(0, diff.getList(ListType.DELETED).size()); restartClusterAndCheckImage(true); }
/** * After the following operations: * Rename a dir -> create a snapshot s on dst tree -> rename the renamed dir * again -> delete snapshot s on dst tree * * Make sure we only delete the snapshot s under the renamed dir. */ @Test public void testRenameDirAndDeleteSnapshot_4() throws Exception { final Path sdir1 = new Path("/dir1"); final Path sdir2 = new Path("/dir2"); final Path foo = new Path(sdir1, "foo"); final Path bar = new Path(foo, "bar"); DFSTestUtil.createFile(hdfs, bar, BLOCKSIZE, REPL, SEED); hdfs.mkdirs(sdir2); SnapshotTestHelper.createSnapshot(hdfs, sdir1, "s1"); SnapshotTestHelper.createSnapshot(hdfs, sdir2, "s2"); final Path foo2 = new Path(sdir2, "foo"); hdfs.rename(foo, foo2); // create two new files under foo2 final Path bar2 = new Path(foo2, "bar2"); DFSTestUtil.createFile(hdfs, bar2, BLOCKSIZE, REPL, SEED); final Path bar3 = new Path(foo2, "bar3"); DFSTestUtil.createFile(hdfs, bar3, BLOCKSIZE, REPL, SEED); // create a new snapshot on sdir2 hdfs.createSnapshot(sdir2, "s3"); // rename foo2 again hdfs.rename(foo2, foo); // delete snapshot s3 hdfs.deleteSnapshot(sdir2, "s3"); // check final INodeDirectory dir1Node = fsdir.getINode4Write(sdir1.toString()) .asDirectory(); // sdir1 + s1 + foo_s1 (foo) + foo (foo + s1 + bar~bar3) Quota.Counts q1 = dir1Node.getDirectoryWithQuotaFeature().getSpaceConsumed(); assertEquals(7, q1.get(Quota.NAMESPACE)); final INodeDirectory dir2Node = fsdir.getINode4Write(sdir2.toString()) .asDirectory(); Quota.Counts q2 = dir2Node.getDirectoryWithQuotaFeature().getSpaceConsumed(); assertEquals(1, q2.get(Quota.NAMESPACE)); final Path foo_s1 = SnapshotTestHelper.getSnapshotPath(sdir1, "s1", foo.getName()); final INode fooRef = fsdir.getINode(foo_s1.toString()); assertTrue(fooRef instanceof INodeReference.WithName); INodeReference.WithCount wc = (WithCount) fooRef.asReference().getReferredINode(); assertEquals(2, wc.getReferenceCount()); INodeDirectory fooNode = wc.getReferredINode().asDirectory(); ReadOnlyList<INode> children = fooNode .getChildrenList(Snapshot.CURRENT_STATE_ID); assertEquals(3, children.size()); assertEquals(bar.getName(), children.get(0).getLocalName()); assertEquals(bar2.getName(), children.get(1).getLocalName()); assertEquals(bar3.getName(), children.get(2).getLocalName()); List<DirectoryDiff> diffList = fooNode.getDiffs().asList(); assertEquals(1, diffList.size()); Snapshot s1 = dir1Node.getSnapshot(DFSUtil.string2Bytes("s1")); assertEquals(s1.getId(), diffList.get(0).getSnapshotId()); ChildrenDiff diff = diffList.get(0).getChildrenDiff(); // bar2 and bar3 in the created list assertEquals(2, diff.getList(ListType.CREATED).size()); assertEquals(0, diff.getList(ListType.DELETED).size()); final INode fooRef2 = fsdir.getINode4Write(foo.toString()); assertTrue(fooRef2 instanceof INodeReference.DstReference); INodeReference.WithCount wc2 = (WithCount) fooRef2.asReference().getReferredINode(); assertSame(wc, wc2); assertSame(fooRef2, wc.getParentReference()); restartClusterAndCheckImage(true); }
/** * After the following operations: * Rename a dir -> create a snapshot s on dst tree -> delete the renamed dir * -> delete snapshot s on dst tree * * Make sure we destroy everything created after the rename under the renamed * dir. */ @Test public void testRenameDirAndDeleteSnapshot_3() throws Exception { final Path sdir1 = new Path("/dir1"); final Path sdir2 = new Path("/dir2"); final Path foo = new Path(sdir1, "foo"); final Path bar = new Path(foo, "bar"); DFSTestUtil.createFile(hdfs, bar, BLOCKSIZE, REPL, SEED); hdfs.mkdirs(sdir2); SnapshotTestHelper.createSnapshot(hdfs, sdir1, "s1"); SnapshotTestHelper.createSnapshot(hdfs, sdir2, "s2"); final Path foo2 = new Path(sdir2, "foo"); hdfs.rename(foo, foo2); // create two new files under foo2 final Path bar2 = new Path(foo2, "bar2"); DFSTestUtil.createFile(hdfs, bar2, BLOCKSIZE, REPL, SEED); final Path bar3 = new Path(foo2, "bar3"); DFSTestUtil.createFile(hdfs, bar3, BLOCKSIZE, REPL, SEED); // create a new snapshot on sdir2 hdfs.createSnapshot(sdir2, "s3"); // delete foo2 hdfs.delete(foo2, true); // delete s3 hdfs.deleteSnapshot(sdir2, "s3"); // check final INodeDirectory dir1Node = fsdir.getINode4Write(sdir1.toString()) .asDirectory(); Quota.Counts q1 = dir1Node.getDirectoryWithQuotaFeature().getSpaceConsumed(); assertEquals(4, q1.get(Quota.NAMESPACE)); final INodeDirectory dir2Node = fsdir.getINode4Write(sdir2.toString()) .asDirectory(); Quota.Counts q2 = dir2Node.getDirectoryWithQuotaFeature().getSpaceConsumed(); assertEquals(2, q2.get(Quota.NAMESPACE)); final Path foo_s1 = SnapshotTestHelper.getSnapshotPath(sdir1, "s1", foo.getName()); INode fooRef = fsdir.getINode(foo_s1.toString()); assertTrue(fooRef instanceof INodeReference.WithName); INodeReference.WithCount wc = (WithCount) fooRef.asReference().getReferredINode(); assertEquals(1, wc.getReferenceCount()); INodeDirectory fooNode = wc.getReferredINode().asDirectory(); ReadOnlyList<INode> children = fooNode .getChildrenList(Snapshot.CURRENT_STATE_ID); assertEquals(1, children.size()); assertEquals(bar.getName(), children.get(0).getLocalName()); List<DirectoryDiff> diffList = fooNode.getDiffs().asList(); assertEquals(1, diffList.size()); Snapshot s1 = dir1Node.getSnapshot(DFSUtil.string2Bytes("s1")); assertEquals(s1.getId(), diffList.get(0).getSnapshotId()); ChildrenDiff diff = diffList.get(0).getChildrenDiff(); assertEquals(0, diff.getList(ListType.CREATED).size()); assertEquals(0, diff.getList(ListType.DELETED).size()); restartClusterAndCheckImage(true); }
/** * After the following operations: * Rename a dir -> create a snapshot s on dst tree -> rename the renamed dir * again -> delete snapshot s on dst tree * * Make sure we only delete the snapshot s under the renamed dir. */ @Test public void testRenameDirAndDeleteSnapshot_4() throws Exception { final Path sdir1 = new Path("/dir1"); final Path sdir2 = new Path("/dir2"); final Path foo = new Path(sdir1, "foo"); final Path bar = new Path(foo, "bar"); DFSTestUtil.createFile(hdfs, bar, BLOCKSIZE, REPL, SEED); hdfs.mkdirs(sdir2); SnapshotTestHelper.createSnapshot(hdfs, sdir1, "s1"); SnapshotTestHelper.createSnapshot(hdfs, sdir2, "s2"); final Path foo2 = new Path(sdir2, "foo"); hdfs.rename(foo, foo2); // create two new files under foo2 final Path bar2 = new Path(foo2, "bar2"); DFSTestUtil.createFile(hdfs, bar2, BLOCKSIZE, REPL, SEED); final Path bar3 = new Path(foo2, "bar3"); DFSTestUtil.createFile(hdfs, bar3, BLOCKSIZE, REPL, SEED); // create a new snapshot on sdir2 hdfs.createSnapshot(sdir2, "s3"); // rename foo2 again hdfs.rename(foo2, foo); // delete snapshot s3 hdfs.deleteSnapshot(sdir2, "s3"); // check final INodeDirectory dir1Node = fsdir.getINode4Write(sdir1.toString()) .asDirectory(); // sdir1 + s1 + foo_s1 (foo) + foo (foo + s1 + bar~bar3) Quota.Counts q1 = dir1Node.getDirectoryWithQuotaFeature().getSpaceConsumed(); assertEquals(9, q1.get(Quota.NAMESPACE)); final INodeDirectory dir2Node = fsdir.getINode4Write(sdir2.toString()) .asDirectory(); Quota.Counts q2 = dir2Node.getDirectoryWithQuotaFeature().getSpaceConsumed(); assertEquals(2, q2.get(Quota.NAMESPACE)); final Path foo_s1 = SnapshotTestHelper.getSnapshotPath(sdir1, "s1", foo.getName()); final INode fooRef = fsdir.getINode(foo_s1.toString()); assertTrue(fooRef instanceof INodeReference.WithName); INodeReference.WithCount wc = (WithCount) fooRef.asReference().getReferredINode(); assertEquals(2, wc.getReferenceCount()); INodeDirectory fooNode = wc.getReferredINode().asDirectory(); ReadOnlyList<INode> children = fooNode .getChildrenList(Snapshot.CURRENT_STATE_ID); assertEquals(3, children.size()); assertEquals(bar.getName(), children.get(0).getLocalName()); assertEquals(bar2.getName(), children.get(1).getLocalName()); assertEquals(bar3.getName(), children.get(2).getLocalName()); List<DirectoryDiff> diffList = fooNode.getDiffs().asList(); assertEquals(1, diffList.size()); Snapshot s1 = dir1Node.getSnapshot(DFSUtil.string2Bytes("s1")); assertEquals(s1.getId(), diffList.get(0).getSnapshotId()); ChildrenDiff diff = diffList.get(0).getChildrenDiff(); // bar2 and bar3 in the created list assertEquals(2, diff.getList(ListType.CREATED).size()); assertEquals(0, diff.getList(ListType.DELETED).size()); final INode fooRef2 = fsdir.getINode4Write(foo.toString()); assertTrue(fooRef2 instanceof INodeReference.DstReference); INodeReference.WithCount wc2 = (WithCount) fooRef2.asReference().getReferredINode(); assertSame(wc, wc2); assertSame(fooRef2, wc.getParentReference()); restartClusterAndCheckImage(true); }
/** Add a dir-diff pair */ private void addDirDiff(INodeDirectory dir, byte[][] relativePath, ChildrenDiff diff) { dirDiffMap.put(dir, diff); diffMap.put(dir, relativePath); }
/** * Test the undo section of rename. Before the rename, we create the renamed * file/dir before taking the snapshot. */ @Test public void testRenameUndo_1() throws Exception { final Path sdir1 = new Path("/dir1"); final Path sdir2 = new Path("/dir2"); hdfs.mkdirs(sdir1); hdfs.mkdirs(sdir2); final Path foo = new Path(sdir1, "foo"); final Path bar = new Path(foo, "bar"); DFSTestUtil.createFile(hdfs, bar, BLOCKSIZE, REPL, SEED); final Path dir2file = new Path(sdir2, "file"); DFSTestUtil.createFile(hdfs, dir2file, BLOCKSIZE, REPL, SEED); SnapshotTestHelper.createSnapshot(hdfs, sdir1, "s1"); INodeDirectory dir2 = fsdir.getINode4Write(sdir2.toString()).asDirectory(); INodeDirectory mockDir2 = spy(dir2); doReturn(false).when(mockDir2).addChild((INode) anyObject(), anyBoolean(), Mockito.anyInt()); INodeDirectory root = fsdir.getINode4Write("/").asDirectory(); root.replaceChild(dir2, mockDir2, fsdir.getINodeMap()); final Path newfoo = new Path(sdir2, "foo"); boolean result = hdfs.rename(foo, newfoo); assertFalse(result); // check the current internal details INodeDirectorySnapshottable dir1Node = (INodeDirectorySnapshottable) fsdir .getINode4Write(sdir1.toString()); Snapshot s1 = dir1Node.getSnapshot(DFSUtil.string2Bytes("s1")); ReadOnlyList<INode> dir1Children = dir1Node .getChildrenList(Snapshot.CURRENT_STATE_ID); assertEquals(1, dir1Children.size()); assertEquals(foo.getName(), dir1Children.get(0).getLocalName()); List<DirectoryDiff> dir1Diffs = dir1Node.getDiffs().asList(); assertEquals(1, dir1Diffs.size()); assertEquals(s1.getId(), dir1Diffs.get(0).getSnapshotId()); // after the undo of rename, both the created and deleted list of sdir1 // should be empty ChildrenDiff childrenDiff = dir1Diffs.get(0).getChildrenDiff(); assertEquals(0, childrenDiff.getList(ListType.DELETED).size()); assertEquals(0, childrenDiff.getList(ListType.CREATED).size()); INode fooNode = fsdir.getINode4Write(foo.toString()); assertTrue(fooNode.isDirectory() && fooNode.asDirectory().isWithSnapshot()); List<DirectoryDiff> fooDiffs = fooNode.asDirectory().getDiffs().asList(); assertEquals(1, fooDiffs.size()); assertEquals(s1.getId(), fooDiffs.get(0).getSnapshotId()); final Path foo_s1 = SnapshotTestHelper.getSnapshotPath(sdir1, "s1", "foo"); INode fooNode_s1 = fsdir.getINode(foo_s1.toString()); assertTrue(fooNode_s1 == fooNode); // check sdir2 assertFalse(hdfs.exists(newfoo)); INodeDirectory dir2Node = fsdir.getINode4Write(sdir2.toString()) .asDirectory(); assertFalse(dir2Node.isWithSnapshot()); ReadOnlyList<INode> dir2Children = dir2Node .getChildrenList(Snapshot.CURRENT_STATE_ID); assertEquals(1, dir2Children.size()); assertEquals(dir2file.getName(), dir2Children.get(0).getLocalName()); }
/** * Test the undo section of rename. Before the rename, we create the renamed * file/dir after taking the snapshot. */ @Test public void testRenameUndo_2() throws Exception { final Path sdir1 = new Path("/dir1"); final Path sdir2 = new Path("/dir2"); hdfs.mkdirs(sdir1); hdfs.mkdirs(sdir2); final Path dir2file = new Path(sdir2, "file"); DFSTestUtil.createFile(hdfs, dir2file, BLOCKSIZE, REPL, SEED); SnapshotTestHelper.createSnapshot(hdfs, sdir1, "s1"); // create foo after taking snapshot final Path foo = new Path(sdir1, "foo"); final Path bar = new Path(foo, "bar"); DFSTestUtil.createFile(hdfs, bar, BLOCKSIZE, REPL, SEED); INodeDirectory dir2 = fsdir.getINode4Write(sdir2.toString()).asDirectory(); INodeDirectory mockDir2 = spy(dir2); doReturn(false).when(mockDir2).addChild((INode) anyObject(), anyBoolean(), Mockito.anyInt()); INodeDirectory root = fsdir.getINode4Write("/").asDirectory(); root.replaceChild(dir2, mockDir2, fsdir.getINodeMap()); final Path newfoo = new Path(sdir2, "foo"); boolean result = hdfs.rename(foo, newfoo); assertFalse(result); // check the current internal details INodeDirectorySnapshottable dir1Node = (INodeDirectorySnapshottable) fsdir .getINode4Write(sdir1.toString()); Snapshot s1 = dir1Node.getSnapshot(DFSUtil.string2Bytes("s1")); ReadOnlyList<INode> dir1Children = dir1Node .getChildrenList(Snapshot.CURRENT_STATE_ID); assertEquals(1, dir1Children.size()); assertEquals(foo.getName(), dir1Children.get(0).getLocalName()); List<DirectoryDiff> dir1Diffs = dir1Node.getDiffs().asList(); assertEquals(1, dir1Diffs.size()); assertEquals(s1.getId(), dir1Diffs.get(0).getSnapshotId()); // after the undo of rename, the created list of sdir1 should contain // 1 element ChildrenDiff childrenDiff = dir1Diffs.get(0).getChildrenDiff(); assertEquals(0, childrenDiff.getList(ListType.DELETED).size()); assertEquals(1, childrenDiff.getList(ListType.CREATED).size()); INode fooNode = fsdir.getINode4Write(foo.toString()); assertTrue(fooNode instanceof INodeDirectory); assertTrue(childrenDiff.getList(ListType.CREATED).get(0) == fooNode); final Path foo_s1 = SnapshotTestHelper.getSnapshotPath(sdir1, "s1", "foo"); assertFalse(hdfs.exists(foo_s1)); // check sdir2 assertFalse(hdfs.exists(newfoo)); INodeDirectory dir2Node = fsdir.getINode4Write(sdir2.toString()) .asDirectory(); assertFalse(dir2Node.isWithSnapshot()); ReadOnlyList<INode> dir2Children = dir2Node .getChildrenList(Snapshot.CURRENT_STATE_ID); assertEquals(1, dir2Children.size()); assertEquals(dir2file.getName(), dir2Children.get(0).getLocalName()); }
/** * After the following operations: * Rename a dir -> create a snapshot s on dst tree -> delete the renamed dir * -> delete snapshot s on dst tree * * Make sure we destroy everything created after the rename under the renamed * dir. */ @Test public void testRenameDirAndDeleteSnapshot_3() throws Exception { final Path sdir1 = new Path("/dir1"); final Path sdir2 = new Path("/dir2"); final Path foo = new Path(sdir1, "foo"); final Path bar = new Path(foo, "bar"); DFSTestUtil.createFile(hdfs, bar, BLOCKSIZE, REPL, SEED); hdfs.mkdirs(sdir2); SnapshotTestHelper.createSnapshot(hdfs, sdir1, "s1"); SnapshotTestHelper.createSnapshot(hdfs, sdir2, "s2"); final Path foo2 = new Path(sdir2, "foo"); hdfs.rename(foo, foo2); // create two new files under foo2 final Path bar2 = new Path(foo2, "bar2"); DFSTestUtil.createFile(hdfs, bar2, BLOCKSIZE, REPL, SEED); final Path bar3 = new Path(foo2, "bar3"); DFSTestUtil.createFile(hdfs, bar3, BLOCKSIZE, REPL, SEED); // create a new snapshot on sdir2 hdfs.createSnapshot(sdir2, "s3"); // delete foo2 hdfs.delete(foo2, true); // delete s3 hdfs.deleteSnapshot(sdir2, "s3"); // check final INodeDirectorySnapshottable dir1Node = (INodeDirectorySnapshottable) fsdir.getINode4Write(sdir1.toString()); Quota.Counts q1 = dir1Node.getDirectoryWithQuotaFeature().getSpaceConsumed(); assertEquals(4, q1.get(Quota.NAMESPACE)); final INodeDirectorySnapshottable dir2Node = (INodeDirectorySnapshottable) fsdir.getINode4Write(sdir2.toString()); Quota.Counts q2 = dir2Node.getDirectoryWithQuotaFeature().getSpaceConsumed(); assertEquals(2, q2.get(Quota.NAMESPACE)); final Path foo_s1 = SnapshotTestHelper.getSnapshotPath(sdir1, "s1", foo.getName()); INode fooRef = fsdir.getINode(foo_s1.toString()); assertTrue(fooRef instanceof INodeReference.WithName); INodeReference.WithCount wc = (WithCount) fooRef.asReference().getReferredINode(); assertEquals(1, wc.getReferenceCount()); INodeDirectory fooNode = wc.getReferredINode().asDirectory(); ReadOnlyList<INode> children = fooNode .getChildrenList(Snapshot.CURRENT_STATE_ID); assertEquals(1, children.size()); assertEquals(bar.getName(), children.get(0).getLocalName()); List<DirectoryDiff> diffList = fooNode.getDiffs().asList(); assertEquals(1, diffList.size()); Snapshot s1 = dir1Node.getSnapshot(DFSUtil.string2Bytes("s1")); assertEquals(s1.getId(), diffList.get(0).getSnapshotId()); ChildrenDiff diff = diffList.get(0).getChildrenDiff(); assertEquals(0, diff.getList(ListType.CREATED).size()); assertEquals(0, diff.getList(ListType.DELETED).size()); restartClusterAndCheckImage(true); }
/** * After the following operations: * Rename a dir -> create a snapshot s on dst tree -> rename the renamed dir * again -> delete snapshot s on dst tree * * Make sure we only delete the snapshot s under the renamed dir. */ @Test public void testRenameDirAndDeleteSnapshot_4() throws Exception { final Path sdir1 = new Path("/dir1"); final Path sdir2 = new Path("/dir2"); final Path foo = new Path(sdir1, "foo"); final Path bar = new Path(foo, "bar"); DFSTestUtil.createFile(hdfs, bar, BLOCKSIZE, REPL, SEED); hdfs.mkdirs(sdir2); SnapshotTestHelper.createSnapshot(hdfs, sdir1, "s1"); SnapshotTestHelper.createSnapshot(hdfs, sdir2, "s2"); final Path foo2 = new Path(sdir2, "foo"); hdfs.rename(foo, foo2); // create two new files under foo2 final Path bar2 = new Path(foo2, "bar2"); DFSTestUtil.createFile(hdfs, bar2, BLOCKSIZE, REPL, SEED); final Path bar3 = new Path(foo2, "bar3"); DFSTestUtil.createFile(hdfs, bar3, BLOCKSIZE, REPL, SEED); // create a new snapshot on sdir2 hdfs.createSnapshot(sdir2, "s3"); // rename foo2 again hdfs.rename(foo2, foo); // delete snapshot s3 hdfs.deleteSnapshot(sdir2, "s3"); // check final INodeDirectorySnapshottable dir1Node = (INodeDirectorySnapshottable) fsdir.getINode4Write(sdir1.toString()); // sdir1 + s1 + foo_s1 (foo) + foo (foo + s1 + bar~bar3) Quota.Counts q1 = dir1Node.getDirectoryWithQuotaFeature().getSpaceConsumed(); assertEquals(9, q1.get(Quota.NAMESPACE)); final INodeDirectorySnapshottable dir2Node = (INodeDirectorySnapshottable) fsdir.getINode4Write(sdir2.toString()); Quota.Counts q2 = dir2Node.getDirectoryWithQuotaFeature().getSpaceConsumed(); assertEquals(2, q2.get(Quota.NAMESPACE)); final Path foo_s1 = SnapshotTestHelper.getSnapshotPath(sdir1, "s1", foo.getName()); final INode fooRef = fsdir.getINode(foo_s1.toString()); assertTrue(fooRef instanceof INodeReference.WithName); INodeReference.WithCount wc = (WithCount) fooRef.asReference().getReferredINode(); assertEquals(2, wc.getReferenceCount()); INodeDirectory fooNode = wc.getReferredINode().asDirectory(); ReadOnlyList<INode> children = fooNode .getChildrenList(Snapshot.CURRENT_STATE_ID); assertEquals(3, children.size()); assertEquals(bar.getName(), children.get(0).getLocalName()); assertEquals(bar2.getName(), children.get(1).getLocalName()); assertEquals(bar3.getName(), children.get(2).getLocalName()); List<DirectoryDiff> diffList = fooNode.getDiffs().asList(); assertEquals(1, diffList.size()); Snapshot s1 = dir1Node.getSnapshot(DFSUtil.string2Bytes("s1")); assertEquals(s1.getId(), diffList.get(0).getSnapshotId()); ChildrenDiff diff = diffList.get(0).getChildrenDiff(); // bar2 and bar3 in the created list assertEquals(2, diff.getList(ListType.CREATED).size()); assertEquals(0, diff.getList(ListType.DELETED).size()); final INode fooRef2 = fsdir.getINode4Write(foo.toString()); assertTrue(fooRef2 instanceof INodeReference.DstReference); INodeReference.WithCount wc2 = (WithCount) fooRef2.asReference().getReferredINode(); assertSame(wc, wc2); assertSame(fooRef2, wc.getParentReference()); restartClusterAndCheckImage(true); }