/** * Create two snapshots in each iteration. Each time we will create a snapshot * for the top node, then randomly pick a dir in the tree and create * snapshot for it. * * Finally check the snapshots are created correctly. */ protected TestDirectoryTree.Node[] createSnapshots() throws Exception { TestDirectoryTree.Node[] nodes = new TestDirectoryTree.Node[2]; // Each time we will create a snapshot for the top level dir Path root = SnapshotTestHelper.createSnapshot(hdfs, dirTree.topNode.nodePath, nextSnapshotName()); snapshotList.add(root); nodes[0] = dirTree.topNode; SnapshotTestHelper.checkSnapshotCreation(hdfs, root, nodes[0].nodePath); // Then randomly pick one dir from the tree (cannot be the top node) and // create snapshot for it ArrayList<TestDirectoryTree.Node> excludedList = new ArrayList<TestDirectoryTree.Node>(); excludedList.add(nodes[0]); nodes[1] = dirTree.getRandomDirNode(random, excludedList); root = SnapshotTestHelper.createSnapshot(hdfs, nodes[1].nodePath, nextSnapshotName()); snapshotList.add(root); SnapshotTestHelper.checkSnapshotCreation(hdfs, root, nodes[1].nodePath); return nodes; }
DirCreationOrDeletion(Path file, FileSystem fs, TestDirectoryTree.Node node, boolean isCreation) { super(file, fs, "dircreation"); this.node = node; // If the node's nonSnapshotChildren is empty, we still need to create // sub-directories this.isCreation = isCreation || node.nonSnapshotChildren.isEmpty(); if (this.isCreation) { // Generate the path for the dir to be created changedPath = new Path(node.nodePath, "sub" + node.nonSnapshotChildren.size()); } else { // If deletion, we delete the current last dir in nonSnapshotChildren changedPath = node.nonSnapshotChildren.get(node.nonSnapshotChildren .size() - 1).nodePath; } this.statusMap = new HashMap<Path, FileStatus>(); }
@Override void modify() throws Exception { if (isCreation) { // creation TestDirectoryTree.Node newChild = new TestDirectoryTree.Node( changedPath, node.level + 1, node, hdfs); // create file under the new non-snapshottable directory newChild.initFileList(hdfs, node.nodePath.getName(), BLOCKSIZE, REPLICATION, seed, 2); node.nonSnapshotChildren.add(newChild); } else { // deletion TestDirectoryTree.Node childToDelete = node.nonSnapshotChildren .remove(node.nonSnapshotChildren.size() - 1); hdfs.delete(childToDelete.nodePath, true); } }
DirRename(Path file, FileSystem fs, TestDirectoryTree.Node src, TestDirectoryTree.Node dst) throws Exception { super(file, fs, "dirrename"); this.srcParent = src; this.dstParent = dst; dstPath = new Path(dstParent.nodePath, "sub" + dstParent.nonSnapshotChildren.size()); // If the srcParent's nonSnapshotChildren is empty, we need to create // sub-directories if (srcParent.nonSnapshotChildren.isEmpty()) { srcPath = new Path(srcParent.nodePath, "sub" + srcParent.nonSnapshotChildren.size()); // creation TestDirectoryTree.Node newChild = new TestDirectoryTree.Node( srcPath, srcParent.level + 1, srcParent, hdfs); // create file under the new non-snapshottable directory newChild.initFileList(hdfs, srcParent.nodePath.getName(), BLOCKSIZE, REPLICATION, seed, 2); srcParent.nonSnapshotChildren.add(newChild); } else { srcPath = new Path(srcParent.nodePath, "sub" + (srcParent.nonSnapshotChildren.size() - 1)); } this.statusMap = new HashMap<Path, FileStatus>(); }
@Before public void setUp() throws Exception { conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCKSIZE); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPLICATION) .build(); cluster.waitActive(); fsn = cluster.getNamesystem(); fsdir = fsn.getFSDirectory(); hdfs = cluster.getFileSystem(); dirTree = new TestDirectoryTree(DIRECTORY_TREE_LEVEL, hdfs); }
private void runTestSnapshot(int iteration) throws Exception { for (int i = 0; i < iteration; i++) { // create snapshot and check the creation cluster.getNamesystem().getSnapshotManager().setAllowNestedSnapshots(true); TestDirectoryTree.Node[] ssNodes = createSnapshots(); // prepare the modifications for the snapshotted dirs // we cover the following directories: top, new, and a random ArrayList<TestDirectoryTree.Node> excludedList = new ArrayList<TestDirectoryTree.Node>(); TestDirectoryTree.Node[] modNodes = new TestDirectoryTree.Node[ssNodes.length + 1]; for (int n = 0; n < ssNodes.length; n++) { modNodes[n] = ssNodes[n]; excludedList.add(ssNodes[n]); } modNodes[modNodes.length - 1] = dirTree.getRandomDirNode(random, excludedList); Modification[] mods = prepareModifications(modNodes); // make changes to the directories/files modifyCurrentDirAndCheckSnapshots(mods); // also update the metadata of directories TestDirectoryTree.Node chmodDir = dirTree.getRandomDirNode(random, null); Modification chmod = new FileChangePermission(chmodDir.nodePath, hdfs, genRandomPermission()); String[] userGroup = genRandomOwner(); TestDirectoryTree.Node chownDir = dirTree.getRandomDirNode(random, Arrays.asList(chmodDir)); Modification chown = new FileChown(chownDir.nodePath, hdfs, userGroup[0], userGroup[1]); modifyCurrentDirAndCheckSnapshots(new Modification[]{chmod, chown}); // check fsimage saving/loading checkFSImage(); } }
@Override void modify() throws Exception { hdfs.rename(srcPath, dstPath); TestDirectoryTree.Node newDstChild = new TestDirectoryTree.Node( dstPath, dstParent.level + 1, dstParent, hdfs); dstParent.nonSnapshotChildren.add(newDstChild); }
