/** * Take a snapshot of the specified disabled region */ protected void snapshotDisabledRegion(final HRegionInfo regionInfo) throws IOException { // 1 copy the regionInfo files to the snapshot Path snapshotRegionDir = TakeSnapshotUtils.getRegionSnapshotDirectory(snapshot, rootDir, regionInfo.getEncodedName()); HRegion.writeRegioninfoOnFilesystem(regionInfo, snapshotRegionDir, fs, conf); // check for error for each region monitor.rethrowException(); // 2 for each region, copy over its recovered.edits directory Path regionDir = HRegion.getRegionDir(rootDir, regionInfo); new CopyRecoveredEditsTask(snapshot, monitor, fs, regionDir, snapshotRegionDir).call(); monitor.rethrowException(); status.setStatus("Completed copying recovered edits for offline snapshot of table: " + snapshot.getTable()); // 3 reference all the files in the region new ReferenceRegionHFilesTask(snapshot, monitor, regionDir, fs, snapshotRegionDir).call(); monitor.rethrowException(); status.setStatus("Completed referencing HFiles for offline snapshot of table: " + snapshot.getTable()); }
/** * Take a snapshot of the specified disabled region */ protected void snapshotDisabledRegion(final HRegionInfo regionInfo) throws IOException { // 2 copy the regionInfo files to the snapshot HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, workingDir, regionInfo); // check for error for each region monitor.rethrowException(); // 2 for each region, copy over its recovered.edits directory Path regionDir = HRegion.getRegionDir(rootDir, regionInfo); Path snapshotRegionDir = regionFs.getRegionDir(); new CopyRecoveredEditsTask(snapshot, monitor, fs, regionDir, snapshotRegionDir).call(); monitor.rethrowException(); status.setStatus("Completed copying recovered edits for offline snapshot of table: " + snapshotTable); // 2 reference all the files in the region new ReferenceRegionHFilesTask(snapshot, monitor, regionDir, fs, snapshotRegionDir).call(); monitor.rethrowException(); status.setStatus("Completed referencing HFiles for offline snapshot of table: " + snapshotTable); }
/** * Check that we don't get an exception if there is no recovered edits directory to copy * @throws Exception on failure */ @Test public void testNoEditsDir() throws Exception { SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("snapshot").build(); ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class); FileSystem fs = UTIL.getTestFileSystem(); Path root = UTIL.getDataTestDir(); String regionName = "regionA"; Path regionDir = new Path(root, regionName); Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, root); try { // doesn't really matter where the region's snapshot directory is, but this is pretty close Path snapshotRegionDir = new Path(workingDir, regionName); fs.mkdirs(snapshotRegionDir); Path regionEdits = HLogUtil.getRegionDirRecoveredEditsDir(regionDir); assertFalse("Edits dir exists already - it shouldn't", fs.exists(regionEdits)); CopyRecoveredEditsTask task = new CopyRecoveredEditsTask(snapshot, monitor, fs, regionDir, snapshotRegionDir); task.call(); } finally { // cleanup the working directory FSUtils.delete(fs, regionDir, true); FSUtils.delete(fs, workingDir, true); } }
@Test public void testCopyFiles() throws Exception { SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("snapshot").build(); ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class); FileSystem fs = UTIL.getTestFileSystem(); Path root = UTIL.getDataTestDir(); String regionName = "regionA"; Path regionDir = new Path(root, regionName); Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, root); try { // doesn't really matter where the region's snapshot directory is, but this is pretty close Path snapshotRegionDir = new Path(workingDir, regionName); fs.mkdirs(snapshotRegionDir); // put some stuff in the recovered.edits directory Path edits = HLogUtil.getRegionDirRecoveredEditsDir(regionDir); fs.mkdirs(edits); // make a file with some data Path file1 = new Path(edits, "0000000000000002352"); FSDataOutputStream out = fs.create(file1); byte[] data = new byte[] { 1, 2, 3, 4 }; out.write(data); out.close(); // make an empty file Path empty = new Path(edits, "empty"); fs.createNewFile(empty); CopyRecoveredEditsTask task = new CopyRecoveredEditsTask(snapshot, monitor, fs, regionDir, snapshotRegionDir); CopyRecoveredEditsTask taskSpy = Mockito.spy(task); taskSpy.call(); Path snapshotEdits = HLogUtil.getRegionDirRecoveredEditsDir(snapshotRegionDir); FileStatus[] snapshotEditFiles = FSUtils.listStatus(fs, snapshotEdits); assertEquals("Got wrong number of files in the snapshot edits", 1, snapshotEditFiles.length); FileStatus file = snapshotEditFiles[0]; assertEquals("Didn't copy expected file", file1.getName(), file.getPath().getName()); Mockito.verify(monitor, Mockito.never()).receive(Mockito.any(ForeignException.class)); Mockito.verify(taskSpy, Mockito.never()).snapshotFailure(Mockito.anyString(), Mockito.any(Exception.class)); } finally { // cleanup the working directory FSUtils.delete(fs, regionDir, true); FSUtils.delete(fs, workingDir, true); } }
@Override public void snapshotRegions(List<Pair<HRegionInfo, ServerName>> regionsAndLocations) throws IOException, KeeperException { try { timeoutInjector.start(); // 1. get all the regions hosting this table. // extract each pair to separate lists Set<String> serverNames = new HashSet<String>(); Set<HRegionInfo> regions = new HashSet<HRegionInfo>(); for (Pair<HRegionInfo, ServerName> p : regionsAndLocations) { regions.add(p.getFirst()); serverNames.add(p.getSecond().toString()); } // 2. for each region, write all the info to disk LOG.info("Starting to write region info and WALs for regions for offline snapshot:" + SnapshotDescriptionUtils.toString(snapshot)); for (HRegionInfo regionInfo : regions) { // 2.1 copy the regionInfo files to the snapshot Path snapshotRegionDir = TakeSnapshotUtils.getRegionSnapshotDirectory(snapshot, rootDir, regionInfo.getEncodedName()); HRegion.writeRegioninfoOnFilesystem(regionInfo, snapshotRegionDir, fs, conf); // check for error for each region monitor.rethrowException(); // 2.2 for each region, copy over its recovered.edits directory Path regionDir = HRegion.getRegionDir(rootDir, regionInfo); new CopyRecoveredEditsTask(snapshot, monitor, fs, regionDir, snapshotRegionDir).call(); monitor.rethrowException(); status.setStatus("Completed copying recovered edits for offline snapshot of table: " + snapshot.getTable()); // 2.3 reference all the files in the region new ReferenceRegionHFilesTask(snapshot, monitor, regionDir, fs, snapshotRegionDir).call(); monitor.rethrowException(); status.setStatus("Completed referencing HFiles for offline snapshot of table: " + snapshot.getTable()); } // 3. write the table info to disk LOG.info("Starting to copy tableinfo for offline snapshot: " + SnapshotDescriptionUtils.toString(snapshot)); TableInfoCopyTask tableInfoCopyTask = new TableInfoCopyTask(this.monitor, snapshot, fs, FSUtils.getRootDir(conf)); tableInfoCopyTask.call(); monitor.rethrowException(); status.setStatus("Finished copying tableinfo for snapshot of table: " + snapshot.getTable()); } catch (Exception e) { // make sure we capture the exception to propagate back to the client later String reason = "Failed snapshot " + SnapshotDescriptionUtils.toString(snapshot) + " due to exception:" + e.getMessage(); ForeignException ee = new ForeignException(reason, e); monitor.receive(ee); status.abort("Snapshot of table: "+ snapshot.getTable() +" failed because " + e.getMessage()); } finally { LOG.debug("Marking snapshot" + SnapshotDescriptionUtils.toString(snapshot) + " as finished."); // 6. mark the timer as finished - even if we got an exception, we don't need to time the // operation any further timeoutInjector.complete(); } }
