/** * Take a snapshot using the specified handler. * On failure the snapshot temporary working directory is removed. * NOTE: prepareToTakeSnapshot() called before this one takes care of the rejecting the * snapshot request if the table is busy with another snapshot/restore operation. * @param snapshot the snapshot description * @param handler the snapshot handler */ private synchronized void snapshotTable(SnapshotDescription snapshot, final TakeSnapshotHandler handler) throws HBaseSnapshotException { try { handler.prepare(); this.executorService.submit(handler); this.snapshotHandlers.put(TableName.valueOf(snapshot.getTable()), handler); } catch (Exception e) { // cleanup the working directory by trying to delete it from the fs. Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir); try { if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) { LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" + ClientSnapshotDescriptionUtils.toString(snapshot)); } } catch (IOException e1) { LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" + ClientSnapshotDescriptionUtils.toString(snapshot)); } // fail the snapshot throw new SnapshotCreationException("Could not build snapshot handler", e, snapshot); } }
/** * Migrates all snapshots, user tables and system tables that require migration. * First migrates snapshots. * Then migrates each user table in order, * then attempts ROOT (should be gone) * Migrates hbase:meta last to indicate migration is complete. */ private static void migrateFsTableDescriptors(FileSystem fs, Path rootDir) throws IOException { // First migrate snapshots - will migrate any snapshot dir that contains a table info file Path snapshotsDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir); if (fs.exists(snapshotsDir)) { LOG.info("Migrating snapshots"); FileStatus[] snapshots = fs.listStatus(snapshotsDir, new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs)); for (FileStatus snapshot : snapshots) { migrateTable(fs, snapshot.getPath()); } } LOG.info("Migrating user tables"); List<Path> userTableDirs = FSUtils.getTableDirs(fs, rootDir); for (Path userTableDir : userTableDirs) { migrateTable(fs, userTableDir); } LOG.info("Migrating system tables"); // migrate meta last because that's what we check to see if migration is complete migrateTableIfExists(fs, rootDir, TableName.META_TABLE_NAME); }
@Test(timeout = 300000) public void testDeleteSnapshot() throws Exception { String snapshotName = "completed"; SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName(snapshotName).build(); DeleteSnapshotRequest request = DeleteSnapshotRequest.newBuilder().setSnapshot(snapshot) .build(); try { master.getMasterRpcServices().deleteSnapshot(null, request); fail("Master didn't throw exception when attempting to delete snapshot that doesn't exist"); } catch (ServiceException e) { LOG.debug("Correctly failed delete of non-existant snapshot:" + e.getMessage()); } // write one snapshot to the fs Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, snapshotDir, fs); // then delete the existing snapshot,which shouldn't cause an exception to be thrown master.getMasterRpcServices().deleteSnapshot(null, request); }
/** * Take a snapshot using the specified handler. * On failure the snapshot temporary working directory is removed. * NOTE: prepareToTakeSnapshot() called before this one takes care of the rejecting the * snapshot request if the table is busy with another snapshot/restore operation. * @param snapshot the snapshot description * @param handler the snapshot handler */ private synchronized void snapshotTable(SnapshotDescription snapshot, final TakeSnapshotHandler handler) throws HBaseSnapshotException { try { handler.prepare(); this.executorService.submit(handler); this.snapshotHandlers.put(snapshot.getTable(), handler); } catch (Exception e) { // cleanup the working directory by trying to delete it from the fs. Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir); try { if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) { LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" + SnapshotDescriptionUtils.toString(snapshot)); } } catch (IOException e1) { LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" + SnapshotDescriptionUtils.toString(snapshot)); } // fail the snapshot throw new SnapshotCreationException("Could not build snapshot handler", e, snapshot); } }
@Test public void testDeleteSnapshot() throws Exception { String snapshotName = "completed"; SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName(snapshotName).build(); try { master.deleteSnapshot(new HSnapshotDescription(snapshot)); fail("Master didn't throw exception when attempting to delete snapshot that doesn't exist"); } catch (IOException e) { LOG.debug("Correctly failed delete of non-existant snapshot:" + e.getMessage()); } // write one snapshot to the fs Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, snapshotDir, fs); // then delete the existing snapshot,which shouldn't cause an exception to be thrown master.deleteSnapshot(new HSnapshotDescription(snapshot)); }
/** * Check that we don't get an exception if there is no recovered edits directory to copy * @throws Exception on failure */ @Test public void testNoEditsDir() throws Exception { SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("snapshot").build(); ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class); FileSystem fs = UTIL.getTestFileSystem(); Path root = UTIL.getDataTestDir(); String regionName = "regionA"; Path regionDir = new Path(root, regionName); Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, root); try { // doesn't really matter where the region's snapshot directory is, but this is pretty close Path snapshotRegionDir = new Path(workingDir, regionName); fs.mkdirs(snapshotRegionDir); Path regionEdits = HLog.getRegionDirRecoveredEditsDir(regionDir); assertFalse("Edits dir exists already - it shouldn't", fs.exists(regionEdits)); CopyRecoveredEditsTask task = new CopyRecoveredEditsTask(snapshot, monitor, fs, regionDir, snapshotRegionDir); task.call(); } finally { // cleanup the working directory FSUtils.delete(fs, regionDir, true); FSUtils.delete(fs, workingDir, true); } }
@Test(timeout = 300000) public void testDeleteSnapshot() throws Exception { String snapshotName = "completed"; SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName(snapshotName).build(); DeleteSnapshotRequest request = DeleteSnapshotRequest.newBuilder().setSnapshot(snapshot) .build(); try { master.deleteSnapshot(null, request); fail("Master didn't throw exception when attempting to delete snapshot that doesn't exist"); } catch (ServiceException e) { LOG.debug("Correctly failed delete of non-existant snapshot:" + e.getMessage()); } // write one snapshot to the fs Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, snapshotDir, fs); // then delete the existing snapshot,which shouldn't cause an exception to be thrown master.deleteSnapshot(null, request); }
@Test public void testSnapshotTempDirReload() throws IOException { long period = Long.MAX_VALUE; // This doesn't refresh cache until we invoke it explicitly Path snapshotDir = new Path(SnapshotDescriptionUtils.getSnapshotsDir(rootDir), SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME); SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, "test-snapshot-file-cache-refresh", new SnapshotFiles()); // Add a new snapshot Path snapshot1 = new Path(snapshotDir, "snapshot1"); Path file1 = new Path(new Path(new Path(snapshot1, "7e91021"), "fam"), "file1"); fs.createNewFile(file1); assertTrue(cache.contains(file1.getName())); // Add another snapshot Path snapshot2 = new Path(snapshotDir, "snapshot2"); Path file2 = new Path(new Path(new Path(snapshot2, "7e91021"), "fam2"), "file2"); fs.createNewFile(file2); assertTrue(cache.contains(file2.getName())); }
/** * Check that we don't get an exception if there is no recovered edits directory to copy * @throws Exception on failure */ @Test public void testNoEditsDir() throws Exception { SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("snapshot").build(); ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class); FileSystem fs = UTIL.getTestFileSystem(); Path root = UTIL.getDataTestDir(); String regionName = "regionA"; Path regionDir = new Path(root, regionName); Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, root); try { // doesn't really matter where the region's snapshot directory is, but this is pretty close Path snapshotRegionDir = new Path(workingDir, regionName); fs.mkdirs(snapshotRegionDir); Path regionEdits = HLogUtil.getRegionDirRecoveredEditsDir(regionDir); assertFalse("Edits dir exists already - it shouldn't", fs.exists(regionEdits)); CopyRecoveredEditsTask task = new CopyRecoveredEditsTask(snapshot, monitor, fs, regionDir, snapshotRegionDir); task.call(); } finally { // cleanup the working directory FSUtils.delete(fs, regionDir, true); FSUtils.delete(fs, workingDir, true); } }
/** * Get table descriptor * @param tableName is the table backed up * @return {@link TableDescriptor} saved in backup image of the table */ TableDescriptor getTableDesc(TableName tableName) throws IOException { Path tableInfoPath = this.getTableInfoPath(tableName); SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, tableInfoPath); SnapshotManifest manifest = SnapshotManifest.open(conf, fs, tableInfoPath, desc); TableDescriptor tableDescriptor = manifest.getTableDescriptor(); if (!tableDescriptor.getTableName().equals(tableName)) { LOG.error("couldn't find Table Desc for table: " + tableName + " under tableInfoPath: " + tableInfoPath.toString()); LOG.error("tableDescriptor.getNameAsString() = " + tableDescriptor.getTableName().getNameAsString()); throw new FileNotFoundException("couldn't find Table Desc for table: " + tableName + " under tableInfoPath: " + tableInfoPath.toString()); } return tableDescriptor; }
private void openWithoutRestoringSnapshot() throws IOException { Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); SnapshotProtos.SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc); List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests(); if (regionManifests == null) { throw new IllegalArgumentException("Snapshot seems empty, snapshotName: " + snapshotName); } regions = new ArrayList<>(regionManifests.size()); regionManifests.stream().map(r -> HRegionInfo.convert(r.getRegionInfo())) .filter(this::isValidRegion).sorted().forEach(r -> regions.add(r)); htd = manifest.getTableDescriptor(); }
/** * Action before cloning from snapshot. * @param env MasterProcedureEnv * @throws IOException * @throws InterruptedException */ private void preCloneSnapshot(final MasterProcedureEnv env) throws IOException, InterruptedException { if (!getTableName().isSystemTable()) { // Check and update namespace quota final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); SnapshotManifest manifest = SnapshotManifest.open( env.getMasterConfiguration(), mfs.getFileSystem(), SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, mfs.getRootDir()), snapshot); ProcedureSyncWait.getMasterQuotaManager(env) .checkNamespaceTableAndRegionQuota(getTableName(), manifest.getRegionManifestsMap().size()); } final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); if (cpHost != null) { cpHost.preCreateTableAction(tableDescriptor, null, getUser()); } }
/** * Take a snapshot using the specified handler. * On failure the snapshot temporary working directory is removed. * NOTE: prepareToTakeSnapshot() called before this one takes care of the rejecting the * snapshot request if the table is busy with another snapshot/restore operation. * @param snapshot the snapshot description * @param handler the snapshot handler */ private synchronized void snapshotTable(SnapshotDescription snapshot, final TakeSnapshotHandler handler) throws HBaseSnapshotException { try { handler.prepare(); this.executorService.submit(handler); this.snapshotHandlers.put(TableName.valueOf(snapshot.getTable()), handler); } catch (Exception e) { // cleanup the working directory by trying to delete it from the fs. Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir); try { if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) { LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" + ClientSnapshotDescriptionUtils.toString(snapshot)); } } catch (IOException e1) { LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" + ClientSnapshotDescriptionUtils.toString(snapshot)); } // fail the snapshot throw new SnapshotCreationException("Could not build snapshot handler", e, ProtobufUtil.createSnapshotDesc(snapshot)); } }
/** * If there is a corrupted region manifest, it should throw out CorruptedSnapshotException, * instead of an IOException */ @Test public void testCorruptedRegionManifest() throws IOException { SnapshotTestingUtils.SnapshotMock snapshotMock = new SnapshotTestingUtils.SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir); SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2( SNAPSHOT_NAME_STR, TABLE_NAME_STR); builder.addRegionV2(); builder.corruptOneRegionManifest(); long period = Long.MAX_VALUE; SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, "test-snapshot-file-cache-refresh", new SnapshotFiles()); try { cache.getSnapshotsInProgress(null); } catch (CorruptedSnapshotException cse) { LOG.info("Expected exception " + cse); } finally { fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir), true); } }
/** * If there is a corrupted data manifest, it should throw out CorruptedSnapshotException, * instead of an IOException */ @Test public void testCorruptedDataManifest() throws IOException { SnapshotTestingUtils.SnapshotMock snapshotMock = new SnapshotTestingUtils.SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir); SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2( SNAPSHOT_NAME_STR, TABLE_NAME_STR); builder.addRegionV2(); // consolidate to generate a data.manifest file builder.consolidate(); builder.corruptDataManifest(); long period = Long.MAX_VALUE; SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, "test-snapshot-file-cache-refresh", new SnapshotFiles()); try { cache.getSnapshotsInProgress(null); } catch (CorruptedSnapshotException cse) { LOG.info("Expected exception " + cse); } finally { fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir), true); } }
@Override public void preListSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx, final SnapshotDescription snapshot) throws IOException { if (SnapshotDescriptionUtils.isSnapshotOwner(snapshot, getActiveUser())) { // list it, if user is the owner of snapshot } else { requirePermission("listSnapshot", Action.ADMIN); } }
@Override public void preRestoreSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx, final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor) throws IOException { if (SnapshotDescriptionUtils.isSnapshotOwner(snapshot, getActiveUser())) { requirePermission("restoreSnapshot", hTableDescriptor.getTableName(), null, null, Permission.Action.ADMIN); } else { requirePermission("restore", Action.ADMIN); } }
@Override public void preDeleteSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx, final SnapshotDescription snapshot) throws IOException { if (SnapshotDescriptionUtils.isSnapshotOwner(snapshot, getActiveUser())) { // Snapshot owner is allowed to delete the snapshot // TODO: We are not logging this for audit } else { requirePermission("deleteSnapshot", Action.ADMIN); } }
public void migrateSnapshots() throws IOException { //migrate snapshot dir Path oldSnapshotDir = new Path(rootDir, HConstants.OLD_SNAPSHOT_DIR_NAME); Path newSnapshotDir = new Path(rootDir, HConstants.SNAPSHOT_DIR_NAME); if (fs.exists(oldSnapshotDir)) { boolean foundOldSnapshotDir = false; // Logic to verify old snapshot dir culled from SnapshotManager // ignore all the snapshots in progress FileStatus[] snapshots = fs.listStatus(oldSnapshotDir, new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs)); // loop through all the completed snapshots for (FileStatus snapshot : snapshots) { Path info = new Path(snapshot.getPath(), SnapshotDescriptionUtils.SNAPSHOTINFO_FILE); // if the snapshot is bad if (fs.exists(info)) { foundOldSnapshotDir = true; break; } } if(foundOldSnapshotDir) { LOG.info("Migrating snapshot dir"); if (!fs.rename(oldSnapshotDir, newSnapshotDir)) { throw new IOException("Failed to move old snapshot dir "+ oldSnapshotDir+" to new "+newSnapshotDir); } } } }
/** * Create a snapshot file cache for all snapshots under the specified [root]/.snapshot on the * filesystem * @param fs {@link FileSystem} where the snapshots are stored * @param rootDir hbase root directory * @param cacheRefreshPeriod period (ms) with which the cache should be refreshed * @param cacheRefreshDelay amount of time to wait for the cache to be refreshed * @param refreshThreadName name of the cache refresh thread * @param inspectSnapshotFiles Filter to apply to each snapshot to extract the files. */ public SnapshotFileCache(FileSystem fs, Path rootDir, long cacheRefreshPeriod, long cacheRefreshDelay, String refreshThreadName, SnapshotFileInspector inspectSnapshotFiles) { this.fs = fs; this.fileInspector = inspectSnapshotFiles; this.snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir); // periodically refresh the file cache to make sure we aren't superfluously saving files. this.refreshTimer = new Timer(refreshThreadName, true); this.refreshTimer.scheduleAtFixedRate(new RefreshCacheTask(), cacheRefreshDelay, cacheRefreshPeriod); }
@VisibleForTesting List<String> getSnapshotsInProgress() throws IOException { List<String> snapshotInProgress = Lists.newArrayList(); // only add those files to the cache, but not to the known snapshots Path snapshotTmpDir = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME); // only add those files to the cache, but not to the known snapshots FileStatus[] running = FSUtils.listStatus(fs, snapshotTmpDir); if (running != null) { for (FileStatus run : running) { snapshotInProgress.addAll(fileInspector.filesUnderSnapshot(run.getPath())); } } return snapshotInProgress; }
/** * Check that the snapshot description written in the filesystem matches the current snapshot * @param snapshotDir snapshot directory to check */ private void verifySnapshotDescription(Path snapshotDir) throws CorruptedSnapshotException { SnapshotDescription found = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); if (!this.snapshot.equals(found)) { throw new CorruptedSnapshotException("Snapshot read (" + found + ") doesn't equal snapshot we ran (" + snapshot + ").", snapshot); } }
/** * Cleans up any snapshots in the snapshot/.tmp directory that were left from failed * snapshot attempts. * * @throws IOException if we can't reach the filesystem */ void resetTempDir() throws IOException { // cleanup any existing snapshots. Path tmpdir = SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir); if (master.getMasterFileSystem().getFileSystem().exists(tmpdir)) { if (!master.getMasterFileSystem().getFileSystem().delete(tmpdir, true)) { LOG.warn("Couldn't delete working snapshot directory: " + tmpdir); } } }
/** * Delete the specified snapshot * @param snapshot * @throws SnapshotDoesNotExistException If the specified snapshot does not exist. * @throws IOException For filesystem IOExceptions */ public void deleteSnapshot(SnapshotDescription snapshot) throws SnapshotDoesNotExistException, IOException { // check to see if it is completed if (!isSnapshotCompleted(snapshot)) { throw new SnapshotDoesNotExistException(snapshot); } String snapshotName = snapshot.getName(); // first create the snapshot description and check to see if it exists FileSystem fs = master.getMasterFileSystem().getFileSystem(); Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); // Get snapshot info from file system. The one passed as parameter is a "fake" snapshotInfo with // just the "name" and it does not contains the "real" snapshot information snapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); // call coproc pre hook MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost(); if (cpHost != null) { cpHost.preDeleteSnapshot(snapshot); } LOG.debug("Deleting snapshot: " + snapshotName); // delete the existing snapshot if (!fs.delete(snapshotDir, true)) { throw new HBaseSnapshotException("Failed to delete snapshot directory: " + snapshotDir); } // call coproc post hook if (cpHost != null) { cpHost.postDeleteSnapshot(snapshot); } }
/** * Check to see if the snapshot is one of the currently completed snapshots * Returns true if the snapshot exists in the "completed snapshots folder". * * @param snapshot expected snapshot to check * @return <tt>true</tt> if the snapshot is stored on the {@link FileSystem}, <tt>false</tt> if is * not stored * @throws IOException if the filesystem throws an unexpected exception, * @throws IllegalArgumentException if snapshot name is invalid. */ private boolean isSnapshotCompleted(SnapshotDescription snapshot) throws IOException { try { final Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir); FileSystem fs = master.getMasterFileSystem().getFileSystem(); // check to see if the snapshot already exists return fs.exists(snapshotDir); } catch (IllegalArgumentException iae) { throw new UnknownSnapshotException("Unexpected exception thrown", iae); } }
@Override public void initialize(MasterServices master, MetricsMaster metricsMaster) throws KeeperException, IOException, UnsupportedOperationException { this.master = master; this.rootDir = master.getMasterFileSystem().getRootDir(); checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem()); // get the configuration for the coordinator Configuration conf = master.getConfiguration(); long wakeFrequency = conf.getInt(SNAPSHOT_WAKE_MILLIS_KEY, SNAPSHOT_WAKE_MILLIS_DEFAULT); long timeoutMillis = Math.max(conf.getLong(SnapshotDescriptionUtils.SNAPSHOT_TIMEOUT_MILLIS_KEY, SnapshotDescriptionUtils.SNAPSHOT_TIMEOUT_MILLIS_DEFAULT), conf.getLong(SnapshotDescriptionUtils.MASTER_SNAPSHOT_TIMEOUT_MILLIS, SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME)); int opThreads = conf.getInt(SNAPSHOT_POOL_THREADS_KEY, SNAPSHOT_POOL_THREADS_DEFAULT); // setup the default procedure coordinator String name = master.getServerName().toString(); ThreadPoolExecutor tpool = ProcedureCoordinator.defaultPool(name, opThreads); ProcedureCoordinatorRpcs comms = new ZKProcedureCoordinatorRpcs( master.getZooKeeper(), SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, name); this.coordinator = new ProcedureCoordinator(comms, tpool, timeoutMillis, wakeFrequency); this.executorService = master.getExecutorService(); resetTempDir(); }
@Test(timeout = 300000) public void testGetCompletedSnapshots() throws Exception { // first check when there are no snapshots GetCompletedSnapshotsRequest request = GetCompletedSnapshotsRequest.newBuilder().build(); GetCompletedSnapshotsResponse response = master.getMasterRpcServices().getCompletedSnapshots(null, request); assertEquals("Found unexpected number of snapshots", 0, response.getSnapshotsCount()); // write one snapshot to the fs String snapshotName = "completed"; Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName(snapshotName).build(); SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, snapshotDir, fs); // check that we get one snapshot response = master.getMasterRpcServices().getCompletedSnapshots(null, request); assertEquals("Found unexpected number of snapshots", 1, response.getSnapshotsCount()); List<SnapshotDescription> snapshots = response.getSnapshotsList(); List<SnapshotDescription> expected = Lists.newArrayList(snapshot); assertEquals("Returned snapshots don't match created snapshots", expected, snapshots); // write a second snapshot snapshotName = "completed_two"; snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); snapshot = SnapshotDescription.newBuilder().setName(snapshotName).build(); SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, snapshotDir, fs); expected.add(snapshot); // check that we get one snapshot response = master.getMasterRpcServices().getCompletedSnapshots(null, request); assertEquals("Found unexpected number of snapshots", 2, response.getSnapshotsCount()); snapshots = response.getSnapshotsList(); assertEquals("Returned snapshots don't match created snapshots", expected, snapshots); }
@Test public void testFindsSnapshotFilesWhenCleaning() throws IOException { Configuration conf = TEST_UTIL.getConfiguration(); FSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir()); Path rootDir = FSUtils.getRootDir(conf); Path archivedHfileDir = new Path(TEST_UTIL.getDataTestDir(), HConstants.HFILE_ARCHIVE_DIRECTORY); FileSystem fs = FileSystem.get(conf); SnapshotHFileCleaner cleaner = new SnapshotHFileCleaner(); cleaner.setConf(conf); // write an hfile to the snapshot directory String snapshotName = "snapshot"; byte[] snapshot = Bytes.toBytes(snapshotName); TableName tableName = TableName.valueOf("table"); Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); HRegionInfo mockRegion = new HRegionInfo(tableName); Path regionSnapshotDir = new Path(snapshotDir, mockRegion.getEncodedName()); Path familyDir = new Path(regionSnapshotDir, "family"); // create a reference to a supposedly valid hfile String hfile = "fd1e73e8a96c486090c5cec07b4894c4"; Path refFile = new Path(familyDir, hfile); // make sure the reference file exists fs.create(refFile); // create the hfile in the archive fs.mkdirs(archivedHfileDir); fs.createNewFile(new Path(archivedHfileDir, hfile)); // make sure that the file isn't deletable assertFalse(cleaner.isFileDeletable(fs.getFileStatus(refFile))); }
