/** * Compute the difference between two snapshots of a directory, or between a * snapshot of the directory and its current tree. */ public SnapshotDiffReport diff(final INodesInPath iip, final String snapshotRootPath, final String from, final String to) throws IOException { // Find the source root directory path where the snapshots were taken. // All the check for path has been included in the valueOf method. final INodeDirectory snapshotRoot = getSnapshottableRoot(iip); if ((from == null || from.isEmpty()) && (to == null || to.isEmpty())) { // both fromSnapshot and toSnapshot indicate the current tree return new SnapshotDiffReport(snapshotRootPath, from, to, Collections.<DiffReportEntry> emptyList()); } final SnapshotDiffInfo diffs = snapshotRoot .getDirectorySnapshottableFeature().computeDiff(snapshotRoot, from, to); return diffs != null ? diffs.generateReport() : new SnapshotDiffReport( snapshotRootPath, from, to, Collections.<DiffReportEntry> emptyList()); }
/** * Generate a {@link SnapshotDiffReport} based on detailed diff information. * @return A {@link SnapshotDiffReport} describing the difference */ public SnapshotDiffReport generateReport() { List<DiffReportEntry> diffReportList = new ArrayList<DiffReportEntry>(); for (Map.Entry<INode,byte[][]> drEntry : diffMap.entrySet()) { INode node = drEntry.getKey(); byte[][] path = drEntry.getValue(); diffReportList.add(new DiffReportEntry(DiffType.MODIFY, path, null)); if (node.isDirectory()) { List<DiffReportEntry> subList = generateReport(dirDiffMap.get(node), path, isFromEarlier(), renameMap); diffReportList.addAll(subList); } } return new SnapshotDiffReport(snapshotRoot.getFullPathName(), Snapshot.getSnapshotName(from), Snapshot.getSnapshotName(to), diffReportList); }
public static SnapshotDiffReportEntryProto convert(DiffReportEntry entry) { if (entry == null) { return null; } ByteString sourcePath = ByteString .copyFrom(entry.getSourcePath() == null ? DFSUtil.EMPTY_BYTES : entry .getSourcePath()); String modification = entry.getType().getLabel(); SnapshotDiffReportEntryProto.Builder builder = SnapshotDiffReportEntryProto .newBuilder().setFullpath(sourcePath) .setModificationLabel(modification); if (entry.getType() == DiffType.RENAME) { ByteString targetPath = ByteString .copyFrom(entry.getTargetPath() == null ? DFSUtil.EMPTY_BYTES : entry .getTargetPath()); builder.setTargetPath(targetPath); } return builder.build(); }
public static SnapshotDiffReport convert(SnapshotDiffReportProto reportProto) { if (reportProto == null) { return null; } String snapshotDir = reportProto.getSnapshotRoot(); String fromSnapshot = reportProto.getFromSnapshot(); String toSnapshot = reportProto.getToSnapshot(); List<SnapshotDiffReportEntryProto> list = reportProto .getDiffReportEntriesList(); List<DiffReportEntry> entries = new ArrayList<DiffReportEntry>(); for (SnapshotDiffReportEntryProto entryProto : list) { DiffReportEntry entry = convert(entryProto); if (entry != null) entries.add(entry); } return new SnapshotDiffReport(snapshotDir, fromSnapshot, toSnapshot, entries); }
public static SnapshotDiffReportProto convert(SnapshotDiffReport report) { if (report == null) { return null; } List<DiffReportEntry> entries = report.getDiffList(); List<SnapshotDiffReportEntryProto> entryProtos = new ArrayList<SnapshotDiffReportEntryProto>(); for (DiffReportEntry entry : entries) { SnapshotDiffReportEntryProto entryProto = convert(entry); if (entryProto != null) entryProtos.add(entryProto); } SnapshotDiffReportProto reportProto = SnapshotDiffReportProto.newBuilder() .setSnapshotRoot(report.getSnapshotRoot()) .setFromSnapshot(report.getFromSnapshot()) .setToSnapshot(report.getLaterSnapshotName()) .addAllDiffReportEntries(entryProtos).build(); return reportProto; }
/** * Rename a file under a snapshottable directory, file does not exist * in a snapshot. */ @Test (timeout=60000) public void testRenameFileNotInSnapshot() throws Exception { hdfs.mkdirs(sub1); hdfs.allowSnapshot(sub1); hdfs.createSnapshot(sub1, snap1); DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, REPL, SEED); hdfs.rename(file1, file2); // Query the diff report and make sure it looks as expected. SnapshotDiffReport diffReport = hdfs.getSnapshotDiffReport(sub1, snap1, ""); List<DiffReportEntry> entries = diffReport.getDiffList(); assertTrue(entries.size() == 2); assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null)); assertTrue(existsInDiffReport(entries, DiffType.CREATE, file2.getName(), null)); }
/** * Rename a file under a snapshottable directory, file exists * in a snapshot. */ @Test public void testRenameFileInSnapshot() throws Exception { hdfs.mkdirs(sub1); hdfs.allowSnapshot(sub1); DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, REPL, SEED); hdfs.createSnapshot(sub1, snap1); hdfs.rename(file1, file2); // Query the diff report and make sure it looks as expected. SnapshotDiffReport diffReport = hdfs.getSnapshotDiffReport(sub1, snap1, ""); System.out.println("DiffList is " + diffReport.toString()); List<DiffReportEntry> entries = diffReport.getDiffList(); assertTrue(entries.size() == 2); assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null)); assertTrue(existsInDiffReport(entries, DiffType.RENAME, file1.getName(), file2.getName())); }
@Test (timeout=60000) public void testRenameFileInSubDirOfDirWithSnapshot() throws Exception { final Path sub2 = new Path(sub1, "sub2"); final Path sub2file1 = new Path(sub2, "sub2file1"); final Path sub2file2 = new Path(sub2, "sub2file2"); final String sub1snap1 = "sub1snap1"; hdfs.mkdirs(sub1); hdfs.mkdirs(sub2); DFSTestUtil.createFile(hdfs, sub2file1, BLOCKSIZE, REPL, SEED); SnapshotTestHelper.createSnapshot(hdfs, sub1, sub1snap1); // Rename the file in the subdirectory. hdfs.rename(sub2file1, sub2file2); // Query the diff report and make sure it looks as expected. SnapshotDiffReport diffReport = hdfs.getSnapshotDiffReport(sub1, sub1snap1, ""); LOG.info("DiffList is \n\"" + diffReport.toString() + "\""); List<DiffReportEntry> entries = diffReport.getDiffList(); assertTrue(existsInDiffReport(entries, DiffType.MODIFY, sub2.getName(), null)); assertTrue(existsInDiffReport(entries, DiffType.RENAME, sub2.getName() + "/" + sub2file1.getName(), sub2.getName() + "/" + sub2file2.getName())); }
@Test (timeout=60000) public void testRenameDirectoryInSnapshot() throws Exception { final Path sub2 = new Path(sub1, "sub2"); final Path sub3 = new Path(sub1, "sub3"); final Path sub2file1 = new Path(sub2, "sub2file1"); final String sub1snap1 = "sub1snap1"; hdfs.mkdirs(sub1); hdfs.mkdirs(sub2); DFSTestUtil.createFile(hdfs, sub2file1, BLOCKSIZE, REPL, SEED); SnapshotTestHelper.createSnapshot(hdfs, sub1, sub1snap1); // First rename the sub-directory. hdfs.rename(sub2, sub3); // Query the diff report and make sure it looks as expected. SnapshotDiffReport diffReport = hdfs.getSnapshotDiffReport(sub1, sub1snap1, ""); LOG.info("DiffList is \n\"" + diffReport.toString() + "\""); List<DiffReportEntry> entries = diffReport.getDiffList(); assertEquals(2, entries.size()); assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null)); assertTrue(existsInDiffReport(entries, DiffType.RENAME, sub2.getName(), sub3.getName())); }
@Test public void testDiffReportWithRenameToNewDir() throws Exception { final Path root = new Path("/"); final Path foo = new Path(root, "foo"); final Path fileInFoo = new Path(foo, "file"); DFSTestUtil.createFile(hdfs, fileInFoo, BLOCKSIZE, REPLICATION, seed); SnapshotTestHelper.createSnapshot(hdfs, root, "s0"); final Path bar = new Path(root, "bar"); hdfs.mkdirs(bar); final Path fileInBar = new Path(bar, "file"); hdfs.rename(fileInFoo, fileInBar); SnapshotTestHelper.createSnapshot(hdfs, root, "s1"); verifyDiffReport(root, "s0", "s1", new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("foo")), new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("bar")), new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("foo/file"), DFSUtil.string2Bytes("bar/file"))); }
/** * Rename a file and then append some data to it */ @Test public void testDiffReportWithRenameAndAppend() throws Exception { final Path root = new Path("/"); final Path foo = new Path(root, "foo"); DFSTestUtil.createFile(hdfs, foo, BLOCKSIZE, REPLICATION, seed); SnapshotTestHelper.createSnapshot(hdfs, root, "s0"); final Path bar = new Path(root, "bar"); hdfs.rename(foo, bar); DFSTestUtil.appendFile(hdfs, bar, 10); // append 10 bytes SnapshotTestHelper.createSnapshot(hdfs, root, "s1"); // we always put modification on the file before rename verifyDiffReport(root, "s0", "s1", new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("foo")), new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("foo"), DFSUtil.string2Bytes("bar"))); }
public static SnapshotDiffReport convert( SnapshotDiffReportProto reportProto) { if (reportProto == null) { return null; } String snapshotDir = reportProto.getSnapshotRoot(); String fromSnapshot = reportProto.getFromSnapshot(); String toSnapshot = reportProto.getToSnapshot(); List<SnapshotDiffReportEntryProto> list = reportProto .getDiffReportEntriesList(); List<DiffReportEntry> entries = new ArrayList<>(); for (SnapshotDiffReportEntryProto entryProto : list) { DiffReportEntry entry = convert(entryProto); if (entry != null) entries.add(entry); } return new SnapshotDiffReport(snapshotDir, fromSnapshot, toSnapshot, entries); }
public static SnapshotDiffReportEntryProto convert(DiffReportEntry entry) { if (entry == null) { return null; } ByteString sourcePath = getByteString(entry.getSourcePath() == null ? DFSUtilClient.EMPTY_BYTES : entry.getSourcePath()); String modification = entry.getType().getLabel(); SnapshotDiffReportEntryProto.Builder builder = SnapshotDiffReportEntryProto .newBuilder().setFullpath(sourcePath) .setModificationLabel(modification); if (entry.getType() == DiffType.RENAME) { ByteString targetPath = getByteString(entry.getTargetPath() == null ? DFSUtilClient.EMPTY_BYTES : entry.getTargetPath()); builder.setTargetPath(targetPath); } return builder.build(); }
public static SnapshotDiffReportProto convert(SnapshotDiffReport report) { if (report == null) { return null; } List<DiffReportEntry> entries = report.getDiffList(); List<SnapshotDiffReportEntryProto> entryProtos = new ArrayList<>(); for (DiffReportEntry entry : entries) { SnapshotDiffReportEntryProto entryProto = convert(entry); if (entryProto != null) entryProtos.add(entryProto); } return SnapshotDiffReportProto.newBuilder() .setSnapshotRoot(report.getSnapshotRoot()) .setFromSnapshot(report.getFromSnapshot()) .setToSnapshot(report.getLaterSnapshotName()) .addAllDiffReportEntries(entryProtos).build(); }
/** * Compute the difference between two snapshots of a directory, or between a * snapshot of the directory and its current tree. */ public SnapshotDiffReport diff(final String path, final String from, final String to) throws IOException { // Find the source root directory path where the snapshots were taken. // All the check for path has been included in the valueOf method. final INodeDirectory snapshotRoot = getSnapshottableRoot(path); if ((from == null || from.isEmpty()) && (to == null || to.isEmpty())) { // both fromSnapshot and toSnapshot indicate the current tree return new SnapshotDiffReport(path, from, to, Collections.<DiffReportEntry> emptyList()); } final SnapshotDiffInfo diffs = snapshotRoot .getDirectorySnapshottableFeature().computeDiff(snapshotRoot, from, to); return diffs != null ? diffs.generateReport() : new SnapshotDiffReport( path, from, to, Collections.<DiffReportEntry> emptyList()); }
/** * Generate a {@link SnapshotDiffReport} based on detailed diff information. * @return A {@link SnapshotDiffReport} describing the difference */ public SnapshotDiffReport generateReport() { List<DiffReportEntry> diffReportList = new ArrayList<DiffReportEntry>(); for (INode node : diffMap.keySet()) { diffReportList.add(new DiffReportEntry(DiffType.MODIFY, diffMap .get(node), null)); if (node.isDirectory()) { List<DiffReportEntry> subList = generateReport(dirDiffMap.get(node), diffMap.get(node), isFromEarlier(), renameMap); diffReportList.addAll(subList); } } return new SnapshotDiffReport(snapshotRoot.getFullPathName(), Snapshot.getSnapshotName(from), Snapshot.getSnapshotName(to), diffReportList); }
