private SnapshotMock.SnapshotBuilder createAndTestSnapshotV1(final SnapshotFileCache cache, final String name, final boolean tmp, final boolean removeOnExit) throws IOException { SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir); SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV1(name, name); createAndTestSnapshot(cache, builder, tmp, removeOnExit); return builder; }
private void restoreAndVerify(final String snapshotName, final String tableName) throws IOException { // Test Rolling-Upgrade like Snapshot. // half machines writing using v1 and the others using v2 format. SnapshotMock snapshotMock = new SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir); SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2("snapshot", tableName); builder.addRegionV1(); builder.addRegionV2(); builder.addRegionV2(); builder.addRegionV1(); Path snapshotDir = builder.commit(); HTableDescriptor htd = builder.getTableDescriptor(); SnapshotDescription desc = builder.getSnapshotDescription(); // Test clone a snapshot HTableDescriptor htdClone = snapshotMock.createHtd("testtb-clone"); testRestore(snapshotDir, desc, htdClone); verifyRestore(rootDir, htd, htdClone); // Test clone a clone ("link to link") SnapshotDescription cloneDesc = SnapshotDescription.newBuilder() .setName("cloneSnapshot") .setTable("testtb-clone") .build(); Path cloneDir = FSUtils.getTableDir(rootDir, htdClone.getTableName()); HTableDescriptor htdClone2 = snapshotMock.createHtd("testtb-clone2"); testRestore(cloneDir, cloneDesc, htdClone2); verifyRestore(rootDir, htd, htdClone2); }
/** * Generates a couple of regions for the specified SnapshotMock, * and then it will run the export and verification. */ private void testSnapshotWithRefsExportFileSystemState(SnapshotMock.SnapshotBuilder builder) throws Exception { Path[] r1Files = builder.addRegion(); Path[] r2Files = builder.addRegion(); builder.commit(); int snapshotFilesCount = r1Files.length + r2Files.length; byte[] snapshotName = Bytes.toBytes(builder.getSnapshotDescription().getName()); TableName tableName = builder.getTableDescriptor().getTableName(); testExportFileSystemState(tableName, snapshotName, snapshotName, snapshotFilesCount); }
private SnapshotMock.SnapshotBuilder createAndTestSnapshotV1(final SnapshotFileCache cache, final String name, final boolean tmp, final boolean removeOnExit) throws IOException { SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir); SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV1(name); createAndTestSnapshot(cache, builder, tmp, removeOnExit); return builder; }
@Test public void testRestore() throws IOException { // Test Rolling-Upgrade like Snapshot. // half machines writing using v1 and the others using v2 format. SnapshotMock snapshotMock = new SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir); SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2("snapshot"); builder.addRegionV1(); builder.addRegionV2(); builder.addRegionV2(); builder.addRegionV1(); Path snapshotDir = builder.commit(); HTableDescriptor htd = builder.getTableDescriptor(); SnapshotDescription desc = builder.getSnapshotDescription(); // Test clone a snapshot HTableDescriptor htdClone = snapshotMock.createHtd("testtb-clone"); testRestore(snapshotDir, desc, htdClone); verifyRestore(rootDir, htd, htdClone); // Test clone a clone ("link to link") SnapshotDescription cloneDesc = SnapshotDescription.newBuilder() .setName("cloneSnapshot") .setTable("testtb-clone") .build(); Path cloneDir = FSUtils.getTableDir(rootDir, htdClone.getTableName()); HTableDescriptor htdClone2 = snapshotMock.createHtd("testtb-clone2"); testRestore(cloneDir, cloneDesc, htdClone2); verifyRestore(rootDir, htd, htdClone2); }
private void restoreAndVerify(final String snapshotName, final String tableName) throws IOException { // Test Rolling-Upgrade like Snapshot. // half machines writing using v1 and the others using v2 format. SnapshotMock snapshotMock = createSnapshotMock(); SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2("snapshot", tableName); builder.addRegionV1(); builder.addRegionV2(); builder.addRegionV2(); builder.addRegionV1(); Path snapshotDir = builder.commit(); TableDescriptor htd = builder.getTableDescriptor(); SnapshotDescription desc = builder.getSnapshotDescription(); // Test clone a snapshot TableDescriptor htdClone = snapshotMock.createHtd("testtb-clone"); testRestore(snapshotDir, desc, htdClone); verifyRestore(rootDir, htd, htdClone); // Test clone a clone ("link to link") SnapshotDescription cloneDesc = SnapshotDescription.newBuilder() .setName("cloneSnapshot") .setTable("testtb-clone") .build(); Path cloneDir = FSUtils.getTableDir(rootDir, htdClone.getTableName()); TableDescriptor htdClone2 = snapshotMock.createHtd("testtb-clone2"); testRestore(cloneDir, cloneDesc, htdClone2); verifyRestore(rootDir, htd, htdClone2); }
/** * Generates a couple of regions for the specified SnapshotMock, * and then it will run the export and verification. */ private void testSnapshotWithRefsExportFileSystemState(SnapshotMock.SnapshotBuilder builder) throws Exception { Path[] r1Files = builder.addRegion(); Path[] r2Files = builder.addRegion(); builder.commit(); int snapshotFilesCount = r1Files.length + r2Files.length; byte[] snapshotName = Bytes.toBytes(builder.getSnapshotDescription().getName()); TableName tableName = builder.getTableDescriptor().getTableName(); TestExportSnapshot.testExportFileSystemState(TEST_UTIL.getConfiguration(), tableName, snapshotName, snapshotName, snapshotFilesCount, testDir, getDestinationDir(), false, null, true); }
@Test public void testWeNeverCacheTmpDirAndLoadIt() throws Exception { final AtomicInteger count = new AtomicInteger(0); // don't refresh the cache unless we tell it to long period = Long.MAX_VALUE; SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, "test-snapshot-file-cache-refresh", new SnapshotFiles()) { @Override List<String> getSnapshotsInProgress() throws IOException { List<String> result = super.getSnapshotsInProgress(); count.incrementAndGet(); return result; } @Override public void triggerCacheRefreshForTesting() { super.triggerCacheRefreshForTesting(); } }; SnapshotMock.SnapshotBuilder complete = createAndTestSnapshotV1(cache, "snapshot", false, false); SnapshotMock.SnapshotBuilder inProgress = createAndTestSnapshotV1(cache, "snapshotInProgress", true, false); int countBeforeCheck = count.get(); FSUtils.logFileSystemState(fs, rootDir, LOG); List<FileStatus> allStoreFiles = getStoreFilesForSnapshot(complete); Iterable<FileStatus> deletableFiles = cache.getUnreferencedFiles(allStoreFiles); assertTrue(Iterables.isEmpty(deletableFiles)); // no need for tmp dir check as all files are accounted for. assertEquals(0, count.get() - countBeforeCheck); // add a random file to make sure we refresh FileStatus randomFile = mockStoreFile(UUID.randomUUID().toString()); allStoreFiles.add(randomFile); deletableFiles = cache.getUnreferencedFiles(allStoreFiles); assertEquals(randomFile, Iterables.getOnlyElement(deletableFiles)); assertEquals(1, count.get() - countBeforeCheck); // we check the tmp directory }
private void createAndTestSnapshotV2(final SnapshotFileCache cache, final String name, final boolean tmp, final boolean removeOnExit) throws IOException { SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir); SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(name, name); createAndTestSnapshot(cache, builder, tmp, removeOnExit); }
private void createAndTestSnapshotV2(final SnapshotFileCache cache, final String name, final boolean tmp, final boolean removeOnExit) throws IOException { SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir); SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(name); createAndTestSnapshot(cache, builder, tmp, removeOnExit); }
@Test public void testWeNeverCacheTmpDirAndLoadIt() throws Exception { final AtomicInteger count = new AtomicInteger(0); // don't refresh the cache unless we tell it to long period = Long.MAX_VALUE; SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, "test-snapshot-file-cache-refresh", new SnapshotFiles()) { @Override List<String> getSnapshotsInProgress(final SnapshotManager snapshotManager) throws IOException { List<String> result = super.getSnapshotsInProgress(snapshotManager); count.incrementAndGet(); return result; } @Override public void triggerCacheRefreshForTesting() { super.triggerCacheRefreshForTesting(); } }; SnapshotMock.SnapshotBuilder complete = createAndTestSnapshotV1(cache, "snapshot", false, false); SnapshotMock.SnapshotBuilder inProgress = createAndTestSnapshotV1(cache, "snapshotInProgress", true, false); int countBeforeCheck = count.get(); FSUtils.logFileSystemState(fs, rootDir, LOG); List<FileStatus> allStoreFiles = getStoreFilesForSnapshot(complete); Iterable<FileStatus> deletableFiles = cache.getUnreferencedFiles(allStoreFiles, null); assertTrue(Iterables.isEmpty(deletableFiles)); // no need for tmp dir check as all files are accounted for. assertEquals(0, count.get() - countBeforeCheck); // add a random file to make sure we refresh FileStatus randomFile = mockStoreFile(UUID.randomUUID().toString()); allStoreFiles.add(randomFile); deletableFiles = cache.getUnreferencedFiles(allStoreFiles, null); assertEquals(randomFile, Iterables.getOnlyElement(deletableFiles)); assertEquals(1, count.get() - countBeforeCheck); // we check the tmp directory }
protected SnapshotMock createSnapshotMock() throws IOException { return new SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir); }
private void createAndTestSnapshotV1(final SnapshotFileCache cache, final String name, final boolean tmp, final boolean removeOnExit) throws IOException { SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir); SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV1(name); createAndTestSnapshot(cache, builder, tmp, removeOnExit); }