static SnapshotDiffReport getSnapshotDiffReport(FSDirectory fsd, SnapshotManager snapshotManager, String path, String fromSnapshot, String toSnapshot) throws IOException { SnapshotDiffReport diffs; final FSPermissionChecker pc = fsd.getPermissionChecker(); fsd.readLock(); try { if (fsd.isPermissionEnabled()) { checkSubtreeReadPermission(fsd, pc, path, fromSnapshot); checkSubtreeReadPermission(fsd, pc, path, toSnapshot); } INodesInPath iip = fsd.getINodesInPath(path, true); diffs = snapshotManager.diff(iip, path, fromSnapshot, toSnapshot); } finally { fsd.readUnlock(); } return diffs; }
/** * Get the difference between two snapshots (or between a snapshot and the * current status) of a snapshottable directory. * * @param path The full path of the snapshottable directory. * @param fromSnapshot Name of the snapshot to calculate the diff from. Null * or empty string indicates the current tree. * @param toSnapshot Name of the snapshot to calculated the diff to. Null or * empty string indicates the current tree. * @return A report about the difference between {@code fromSnapshot} and * {@code toSnapshot}. Modified/deleted/created/renamed files and * directories belonging to the snapshottable directories are listed * and labeled as M/-/+/R respectively. * @throws IOException */ SnapshotDiffReport getSnapshotDiffReport(String path, String fromSnapshot, String toSnapshot) throws IOException { SnapshotDiffReport diffs = null; checkOperation(OperationCategory.READ); readLock(); try { checkOperation(OperationCategory.READ); diffs = FSDirSnapshotOp.getSnapshotDiffReport(dir, snapshotManager, path, fromSnapshot, toSnapshot); } finally { readUnlock(); } logAuditEvent(diffs != null, "computeSnapshotDiff", null, null, null); return diffs; }
/** * Compute the difference between two snapshots of a directory, or between a * snapshot of the directory and its current tree. */ public SnapshotDiffReport diff(final INodesInPath iip, final String snapshotRootPath, final String from, final String to) throws IOException { // Find the source root directory path where the snapshots were taken. // All the check for path has been included in the valueOf method. final INodeDirectory snapshotRoot = getSnapshottableRoot(iip); if ((from == null || from.isEmpty()) && (to == null || to.isEmpty())) { // both fromSnapshot and toSnapshot indicate the current tree return new SnapshotDiffReport(snapshotRootPath, from, to, Collections.<DiffReportEntry> emptyList()); } final SnapshotDiffInfo diffs = snapshotRoot .getDirectorySnapshottableFeature().computeDiff(snapshotRoot, from, to); return diffs != null ? diffs.generateReport() : new SnapshotDiffReport( snapshotRootPath, from, to, Collections.<DiffReportEntry> emptyList()); }
/** * Generate a {@link SnapshotDiffReport} based on detailed diff information. * @return A {@link SnapshotDiffReport} describing the difference */ public SnapshotDiffReport generateReport() { List<DiffReportEntry> diffReportList = new ArrayList<DiffReportEntry>(); for (Map.Entry<INode,byte[][]> drEntry : diffMap.entrySet()) { INode node = drEntry.getKey(); byte[][] path = drEntry.getValue(); diffReportList.add(new DiffReportEntry(DiffType.MODIFY, path, null)); if (node.isDirectory()) { List<DiffReportEntry> subList = generateReport(dirDiffMap.get(node), path, isFromEarlier(), renameMap); diffReportList.addAll(subList); } } return new SnapshotDiffReport(snapshotRoot.getFullPathName(), Snapshot.getSnapshotName(from), Snapshot.getSnapshotName(to), diffReportList); }
public static SnapshotDiffReport convert(SnapshotDiffReportProto reportProto) { if (reportProto == null) { return null; } String snapshotDir = reportProto.getSnapshotRoot(); String fromSnapshot = reportProto.getFromSnapshot(); String toSnapshot = reportProto.getToSnapshot(); List<SnapshotDiffReportEntryProto> list = reportProto .getDiffReportEntriesList(); List<DiffReportEntry> entries = new ArrayList<DiffReportEntry>(); for (SnapshotDiffReportEntryProto entryProto : list) { DiffReportEntry entry = convert(entryProto); if (entry != null) entries.add(entry); } return new SnapshotDiffReport(snapshotDir, fromSnapshot, toSnapshot, entries); }
public static SnapshotDiffReportProto convert(SnapshotDiffReport report) { if (report == null) { return null; } List<DiffReportEntry> entries = report.getDiffList(); List<SnapshotDiffReportEntryProto> entryProtos = new ArrayList<SnapshotDiffReportEntryProto>(); for (DiffReportEntry entry : entries) { SnapshotDiffReportEntryProto entryProto = convert(entry); if (entryProto != null) entryProtos.add(entryProto); } SnapshotDiffReportProto reportProto = SnapshotDiffReportProto.newBuilder() .setSnapshotRoot(report.getSnapshotRoot()) .setFromSnapshot(report.getFromSnapshot()) .setToSnapshot(report.getLaterSnapshotName()) .addAllDiffReportEntries(entryProtos).build(); return reportProto; }
/** * Rename a file under a snapshottable directory, file does not exist * in a snapshot. */ @Test (timeout=60000) public void testRenameFileNotInSnapshot() throws Exception { hdfs.mkdirs(sub1); hdfs.allowSnapshot(sub1); hdfs.createSnapshot(sub1, snap1); DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, REPL, SEED); hdfs.rename(file1, file2); // Query the diff report and make sure it looks as expected. SnapshotDiffReport diffReport = hdfs.getSnapshotDiffReport(sub1, snap1, ""); List<DiffReportEntry> entries = diffReport.getDiffList(); assertTrue(entries.size() == 2); assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null)); assertTrue(existsInDiffReport(entries, DiffType.CREATE, file2.getName(), null)); }
/** * Rename a file under a snapshottable directory, file exists * in a snapshot. */ @Test public void testRenameFileInSnapshot() throws Exception { hdfs.mkdirs(sub1); hdfs.allowSnapshot(sub1); DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, REPL, SEED); hdfs.createSnapshot(sub1, snap1); hdfs.rename(file1, file2); // Query the diff report and make sure it looks as expected. SnapshotDiffReport diffReport = hdfs.getSnapshotDiffReport(sub1, snap1, ""); System.out.println("DiffList is " + diffReport.toString()); List<DiffReportEntry> entries = diffReport.getDiffList(); assertTrue(entries.size() == 2); assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null)); assertTrue(existsInDiffReport(entries, DiffType.RENAME, file1.getName(), file2.getName())); }
@Test (timeout=60000) public void testRenameFileInSubDirOfDirWithSnapshot() throws Exception { final Path sub2 = new Path(sub1, "sub2"); final Path sub2file1 = new Path(sub2, "sub2file1"); final Path sub2file2 = new Path(sub2, "sub2file2"); final String sub1snap1 = "sub1snap1"; hdfs.mkdirs(sub1); hdfs.mkdirs(sub2); DFSTestUtil.createFile(hdfs, sub2file1, BLOCKSIZE, REPL, SEED); SnapshotTestHelper.createSnapshot(hdfs, sub1, sub1snap1); // Rename the file in the subdirectory. hdfs.rename(sub2file1, sub2file2); // Query the diff report and make sure it looks as expected. SnapshotDiffReport diffReport = hdfs.getSnapshotDiffReport(sub1, sub1snap1, ""); LOG.info("DiffList is \n\"" + diffReport.toString() + "\""); List<DiffReportEntry> entries = diffReport.getDiffList(); assertTrue(existsInDiffReport(entries, DiffType.MODIFY, sub2.getName(), null)); assertTrue(existsInDiffReport(entries, DiffType.RENAME, sub2.getName() + "/" + sub2file1.getName(), sub2.getName() + "/" + sub2file2.getName())); }
@Test (timeout=60000) public void testRenameDirectoryInSnapshot() throws Exception { final Path sub2 = new Path(sub1, "sub2"); final Path sub3 = new Path(sub1, "sub3"); final Path sub2file1 = new Path(sub2, "sub2file1"); final String sub1snap1 = "sub1snap1"; hdfs.mkdirs(sub1); hdfs.mkdirs(sub2); DFSTestUtil.createFile(hdfs, sub2file1, BLOCKSIZE, REPL, SEED); SnapshotTestHelper.createSnapshot(hdfs, sub1, sub1snap1); // First rename the sub-directory. hdfs.rename(sub2, sub3); // Query the diff report and make sure it looks as expected. SnapshotDiffReport diffReport = hdfs.getSnapshotDiffReport(sub1, sub1snap1, ""); LOG.info("DiffList is \n\"" + diffReport.toString() + "\""); List<DiffReportEntry> entries = diffReport.getDiffList(); assertEquals(2, entries.size()); assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null)); assertTrue(existsInDiffReport(entries, DiffType.RENAME, sub2.getName(), sub3.getName())); }
/** * Compute the snapshot diff on the given file system. Return true if the diff * is empty, i.e., no changes have happened in the FS. */ private static boolean checkNoChange(DistCpOptions inputOptions, DistributedFileSystem fs, Path path) { try { SnapshotDiffReport targetDiff = fs.getSnapshotDiffReport(path, inputOptions.getFromSnapshot(), ""); if (!targetDiff.getDiffList().isEmpty()) { DistCp.LOG.warn("The target has been modified since snapshot " + inputOptions.getFromSnapshot()); return false; } else { return true; } } catch (IOException e) { DistCp.LOG.warn("Failed to compute snapshot diff on " + path, e); } return false; }
@Test public void testSync2() throws Exception { initData2(source); initData2(target); dfs.allowSnapshot(source); dfs.allowSnapshot(target); dfs.createSnapshot(source, "s1"); dfs.createSnapshot(target, "s1"); // make changes under source changeData2(source); dfs.createSnapshot(source, "s2"); SnapshotDiffReport report = dfs.getSnapshotDiffReport(source, "s1", "s2"); System.out.println(report); // do the sync Assert.assertTrue(DistCpSync.sync(options, conf)); verifyCopy(dfs.getFileStatus(source), dfs.getFileStatus(target), false); }
/** * Test a case where there are multiple source files with the same name */ @Test public void testSync3() throws Exception { initData3(source); initData3(target); dfs.allowSnapshot(source); dfs.allowSnapshot(target); dfs.createSnapshot(source, "s1"); dfs.createSnapshot(target, "s1"); // make changes under source changeData3(source); dfs.createSnapshot(source, "s2"); SnapshotDiffReport report = dfs.getSnapshotDiffReport(source, "s1", "s2"); System.out.println(report); // do the sync Assert.assertTrue(DistCpSync.sync(options, conf)); verifyCopy(dfs.getFileStatus(source), dfs.getFileStatus(target), false); }
public static SnapshotDiffReport convert( SnapshotDiffReportProto reportProto) { if (reportProto == null) { return null; } String snapshotDir = reportProto.getSnapshotRoot(); String fromSnapshot = reportProto.getFromSnapshot(); String toSnapshot = reportProto.getToSnapshot(); List<SnapshotDiffReportEntryProto> list = reportProto .getDiffReportEntriesList(); List<DiffReportEntry> entries = new ArrayList<>(); for (SnapshotDiffReportEntryProto entryProto : list) { DiffReportEntry entry = convert(entryProto); if (entry != null) entries.add(entry); } return new SnapshotDiffReport(snapshotDir, fromSnapshot, toSnapshot, entries); }
public static SnapshotDiffReportProto convert(SnapshotDiffReport report) { if (report == null) { return null; } List<DiffReportEntry> entries = report.getDiffList(); List<SnapshotDiffReportEntryProto> entryProtos = new ArrayList<>(); for (DiffReportEntry entry : entries) { SnapshotDiffReportEntryProto entryProto = convert(entry); if (entryProto != null) entryProtos.add(entryProto); } return SnapshotDiffReportProto.newBuilder() .setSnapshotRoot(report.getSnapshotRoot()) .setFromSnapshot(report.getFromSnapshot()) .setToSnapshot(report.getLaterSnapshotName()) .addAllDiffReportEntries(entryProtos).build(); }
/** * Get the difference between two snapshots (or between a snapshot and the * current status) of a snapshottable directory. * * @param path The full path of the snapshottable directory. * @param fromSnapshot Name of the snapshot to calculate the diff from. Null * or empty string indicates the current tree. * @param toSnapshot Name of the snapshot to calculated the diff to. Null or * empty string indicates the current tree. * @return A report about the difference between {@code fromSnapshot} and * {@code toSnapshot}. Modified/deleted/created/renamed files and * directories belonging to the snapshottable directories are listed * and labeled as M/-/+/R respectively. * @throws IOException */ SnapshotDiffReport getSnapshotDiffReport(String path, String fromSnapshot, String toSnapshot) throws IOException { SnapshotDiffReport diffs = null; checkOperation(OperationCategory.READ); readLock(); try { checkOperation(OperationCategory.READ); diffs = FSDirSnapshotOp.getSnapshotDiffReport(dir, snapshotManager, path, fromSnapshot, toSnapshot); } finally { readUnlock(); } String fromSnapshotRoot = (fromSnapshot == null || fromSnapshot.isEmpty()) ? path : Snapshot.getSnapshotPath(path, fromSnapshot); String toSnapshotRoot = (toSnapshot == null || toSnapshot.isEmpty()) ? path : Snapshot.getSnapshotPath(path, toSnapshot); logAuditEvent(diffs != null, "computeSnapshotDiff", fromSnapshotRoot, toSnapshotRoot, null); return diffs; }
/** * Compute the snapshot diff on the given file system. Return true if the diff * is empty, i.e., no changes have happened in the FS. */ private boolean checkNoChange(DistributedFileSystem fs, Path path) { try { SnapshotDiffReport targetDiff = fs.getSnapshotDiffReport(path, inputOptions.getFromSnapshot(), ""); if (!targetDiff.getDiffList().isEmpty()) { DistCp.LOG.warn("The target has been modified since snapshot " + inputOptions.getFromSnapshot()); return false; } else { return true; } } catch (IOException e) { DistCp.LOG.warn("Failed to compute snapshot diff on " + path, e); } return false; }
/** * Find the possible rename item which equals to the parent or self of * a created/modified file/directory. * @param diff a modify/create diff item * @param renameDiffArray all rename diffs * @return possible rename item */ private DiffInfo getRenameItem(DiffInfo diff, DiffInfo[] renameDiffArray) { for (DiffInfo renameItem : renameDiffArray) { if (diff.source.equals(renameItem.source)) { // The same path string may appear in: // 1. both renamed and modified snapshot diff entries. // 2. both renamed and created snapshot diff entries. // Case 1 is the about same file/directory, whereas case 2 // is about two different files/directories. // We are finding case 1 here, thus we check against DiffType.MODIFY. if (diff.getType() == SnapshotDiffReport.DiffType.MODIFY) { return renameItem; } } else if (isParentOf(renameItem.source, diff.source)) { // If rename entry is the parent of diff entry, then both MODIFY and // CREATE diff entries should be handled. return renameItem; } } return null; }
/** * Prepare the diff list. * This diff list only includes created or modified files/directories, since * delete and rename items are synchronized already. * * If the parent or self of a source path is renamed, we need to change its * target path according the correspondent rename item. * @return a diff list */ public ArrayList<DiffInfo> prepareDiffList() { DiffInfo[] modifyAndCreateDiffs = getCreateAndModifyDiffs(); List<DiffInfo> renameDiffsList = diffMap.get(SnapshotDiffReport.DiffType.RENAME); DiffInfo[] renameDiffArray = renameDiffsList.toArray(new DiffInfo[renameDiffsList.size()]); Arrays.sort(renameDiffArray, DiffInfo.sourceComparator); ArrayList<DiffInfo> finalListWithTarget = new ArrayList<>(); for (DiffInfo diff : modifyAndCreateDiffs) { DiffInfo renameItem = getRenameItem(diff, renameDiffArray); if (renameItem == null) { diff.target = diff.source; } else { diff.target = getTargetPath(diff.source, renameItem); } finalListWithTarget.add(diff); } return finalListWithTarget; }
/** * Test a case where there are multiple source files with the same name. */ @Test public void testSync3() throws Exception { initData3(source); initData3(target); enableAndCreateFirstSnapshot(); // make changes under source changeData3(source); dfs.createSnapshot(source, "s2"); SnapshotDiffReport report = dfs.getSnapshotDiffReport(source, "s1", "s2"); System.out.println(report); syncAndVerify(); }
/** * Test a case where multiple level dirs are renamed. */ @Test public void testSync4() throws Exception { initData4(source); initData4(target); enableAndCreateFirstSnapshot(); // make changes under source changeData4(source); dfs.createSnapshot(source, "s2"); SnapshotDiffReport report = dfs.getSnapshotDiffReport(source, "s1", "s2"); System.out.println(report); syncAndVerify(); }
/** * Test a case with different delete and rename sequences. */ @Test public void testSync5() throws Exception { initData5(source); initData5(target); enableAndCreateFirstSnapshot(); // make changes under source changeData5(source); dfs.createSnapshot(source, "s2"); SnapshotDiffReport report = dfs.getSnapshotDiffReport(source, "s1", "s2"); System.out.println(report); syncAndVerify(); }
