/** * Remove from the specified region the store files of the specified column family, * either by archiving them or outright deletion * @param fs the filesystem where the store files live * @param conf {@link Configuration} to examine to determine the archive directory * @param parent Parent region hosting the store files * @param tableDir {@link Path} to where the table is being stored (for building the archive path) * @param family the family hosting the store files * @throws IOException if the files could not be correctly disposed. */ public static void archiveFamily(FileSystem fs, Configuration conf, HRegionInfo parent, Path tableDir, byte[] family) throws IOException { Path familyDir = new Path(tableDir, new Path(parent.getEncodedName(), Bytes.toString(family))); FileStatus[] storeFiles = FSUtils.listStatus(fs, familyDir); if (storeFiles == null) { LOG.debug("No store files to dispose for region=" + parent.getRegionNameAsString() + ", family=" + Bytes.toString(family)); return; } FileStatusConverter getAsFile = new FileStatusConverter(fs); Collection<File> toArchive = Lists.transform(Arrays.asList(storeFiles), getAsFile); Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, parent, tableDir, family); // do the actual archive if (!resolveAndArchive(fs, storeArchiveDir, toArchive)) { throw new IOException("Failed to archive/delete all the files for region:" + Bytes.toString(parent.getRegionName()) + ", family:" + Bytes.toString(family) + " into " + storeArchiveDir + ". Something is probably awry on the filesystem."); } }
/** * Archive the store file * @param fs the filesystem where the store files live * @param regionInfo region hosting the store files * @param conf {@link Configuration} to examine to determine the archive directory * @param tableDir {@link Path} to where the table is being stored (for building the archive path) * @param family the family hosting the store files * @param storeFile file to be archived * @throws IOException if the files could not be correctly disposed. */ public static void archiveStoreFile(Configuration conf, FileSystem fs, HRegionInfo regionInfo, Path tableDir, byte[] family, Path storeFile) throws IOException { Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, tableDir, family); // make sure we don't archive if we can't and that the archive dir exists if (!fs.mkdirs(storeArchiveDir)) { throw new IOException("Could not make archive directory (" + storeArchiveDir + ") for store:" + Bytes.toString(family) + ", deleting compacted files instead."); } // do the actual archive long start = EnvironmentEdgeManager.currentTime(); File file = new FileablePath(fs, storeFile); if (!resolveAndArchiveFile(storeArchiveDir, file, Long.toString(start))) { throw new IOException("Failed to archive/delete the file for region:" + regionInfo.getRegionNameAsString() + ", family:" + Bytes.toString(family) + " into " + storeArchiveDir + ". Something is probably awry on the filesystem."); } }
/** * Remove from the specified region the store files of the specified column family, either by * archiving them or outright deletion * @param fs the filesystem where the store files live * @param conf {@link Configuration} to examine to determine the archive directory * @param parent Parent region hosting the store files * @param tableDir {@link Path} to where the table is being stored (for building the archive path) * @param family the family hosting the store files * @throws IOException if the files could not be correctly disposed. */ public static void archiveFamily(FileSystem fs, Configuration conf, HRegionInfo parent, Path tableDir, byte[] family) throws IOException { Path familyDir = new Path(tableDir, new Path(parent.getEncodedName(), Bytes.toString(family))); FileStatus[] storeFiles = FSUtils.listStatus(fs, familyDir, null); if (storeFiles == null) { LOG.debug("No store files to dispose for region=" + parent.getRegionNameAsString() + ", family=" + Bytes.toString(family)); return; } FileStatusConverter getAsFile = new FileStatusConverter(fs); Collection<File> toArchive = Lists.transform(Arrays.asList(storeFiles), getAsFile); Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, parent, tableDir, family); // do the actual archive if (!resolveAndArchive(fs, storeArchiveDir, toArchive)) { throw new IOException("Failed to archive/delete all the files for region:" + Bytes.toString(parent.getRegionName()) + ", family:" + Bytes.toString(family) + " into " + storeArchiveDir + ". Something is probably awry on the filesystem."); } }
/** * Archive the store file * @param fs the filesystem where the store files live * @param regionInfo region hosting the store files * @param conf {@link Configuration} to examine to determine the archive directory * @param tableDir {@link Path} to where the table is being stored (for building the archive path) * @param family the family hosting the store files * @param storeFile file to be archived * @throws IOException if the files could not be correctly disposed. */ public static void archiveStoreFile(FileSystem fs, HRegionInfo regionInfo, Configuration conf, Path tableDir, byte[] family, Path storeFile) throws IOException { Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, tableDir, family); // make sure we don't archive if we can't and that the archive dir exists if (!HBaseFileSystem.makeDirOnFileSystem(fs, storeArchiveDir)) { throw new IOException("Could not make archive directory (" + storeArchiveDir + ") for store:" + Bytes.toString(family) + ", deleting compacted files instead."); } // do the actual archive long start = EnvironmentEdgeManager.currentTimeMillis(); File file = new FileablePath(fs, storeFile); if (!resolveAndArchiveFile(storeArchiveDir, file, Long.toString(start))) { throw new IOException("Failed to archive/delete the file for region:" + regionInfo.getRegionNameAsString() + ", family:" + Bytes.toString(family) + " into " + storeArchiveDir + ". Something is probably awry on the filesystem."); } }
/** * Archive the store file * @param fs the filesystem where the store files live * @param regionInfo region hosting the store files * @param conf {@link Configuration} to examine to determine the archive directory * @param tableDir {@link Path} to where the table is being stored (for building the archive path) * @param family the family hosting the store files * @param storeFile file to be archived * @throws IOException if the files could not be correctly disposed. */ public static void archiveStoreFile(Configuration conf, FileSystem fs, HRegionInfo regionInfo, Path tableDir, byte[] family, Path storeFile) throws IOException { Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, tableDir, family); // make sure we don't archive if we can't and that the archive dir exists if (!fs.mkdirs(storeArchiveDir)) { throw new IOException("Could not make archive directory (" + storeArchiveDir + ") for store:" + Bytes.toString(family) + ", deleting compacted files instead."); } // do the actual archive long start = EnvironmentEdgeManager.currentTimeMillis(); File file = new FileablePath(fs, storeFile); if (!resolveAndArchiveFile(storeArchiveDir, file, Long.toString(start))) { throw new IOException("Failed to archive/delete the file for region:" + regionInfo.getRegionNameAsString() + ", family:" + Bytes.toString(family) + " into " + storeArchiveDir + ". Something is probably awry on the filesystem."); } }
/** * Remove from the specified region the store files of the specified column family, * either by archiving them or outright deletion * @param fs the filesystem where the store files live * @param conf {@link Configuration} to examine to determine the archive directory * @param parent Parent region hosting the store files * @param tableDir {@link Path} to where the table is being stored (for building the archive path) * @param family the family hosting the store files * @throws IOException if the files could not be correctly disposed. */ public static void archiveFamily(FileSystem fs, Configuration conf, HRegionInfo parent, Path tableDir, byte[] family) throws IOException { Path familyDir = new Path(tableDir, new Path(parent.getEncodedName(), Bytes.toString(family))); FileStatus[] storeFiles = FSUtils.listStatus(fs, familyDir, null); if (storeFiles == null) { LOG.debug("No store files to dispose for region=" + parent.getRegionNameAsString() + ", family=" + Bytes.toString(family)); return; } FileStatusConverter getAsFile = new FileStatusConverter(fs); Collection<File> toArchive = Lists.transform(Arrays.asList(storeFiles), getAsFile); Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, parent, tableDir, family); // do the actual archive if (!resolveAndArchive(fs, storeArchiveDir, toArchive)) { throw new IOException("Failed to archive/delete all the files for region:" + Bytes.toString(parent.getRegionName()) + ", family:" + Bytes.toString(family) + " into " + storeArchiveDir + ". Something is probably awry on the filesystem."); } }
/** * Removes from the specified region the store files of the specified column family, * either by archiving them or outright deletion * @param fs the filesystem where the store files live * @param conf {@link Configuration} to examine to determine the archive directory * @param parent Parent region hosting the store files * @param familyDir {@link Path} to where the family is being stored * @param family the family hosting the store files * @throws IOException if the files could not be correctly disposed. */ public static void archiveFamilyByFamilyDir(FileSystem fs, Configuration conf, RegionInfo parent, Path familyDir, byte[] family) throws IOException { FileStatus[] storeFiles = FSUtils.listStatus(fs, familyDir); if (storeFiles == null) { LOG.debug("No store files to dispose for region=" + parent.getRegionNameAsString() + ", family=" + Bytes.toString(family)); return; } FileStatusConverter getAsFile = new FileStatusConverter(fs); Collection<File> toArchive = Lists.transform(Arrays.asList(storeFiles), getAsFile); Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, parent, family); // do the actual archive List<File> failedArchive = resolveAndArchive(fs, storeArchiveDir, toArchive, EnvironmentEdgeManager.currentTime()); if (!failedArchive.isEmpty()){ throw new FailedArchiveException("Failed to archive/delete all the files for region:" + Bytes.toString(parent.getRegionName()) + ", family:" + Bytes.toString(family) + " into " + storeArchiveDir + ". Something is probably awry on the filesystem.", Collections2.transform(failedArchive, FUNC_FILE_TO_PATH)); } }
/** * Archive the store file * @param fs the filesystem where the store files live * @param regionInfo region hosting the store files * @param conf {@link Configuration} to examine to determine the archive directory * @param tableDir {@link Path} to where the table is being stored (for building the archive path) * @param family the family hosting the store files * @param storeFile file to be archived * @throws IOException if the files could not be correctly disposed. */ public static void archiveStoreFile(Configuration conf, FileSystem fs, RegionInfo regionInfo, Path tableDir, byte[] family, Path storeFile) throws IOException { Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, tableDir, family); // make sure we don't archive if we can't and that the archive dir exists if (!fs.mkdirs(storeArchiveDir)) { throw new IOException("Could not make archive directory (" + storeArchiveDir + ") for store:" + Bytes.toString(family) + ", deleting compacted files instead."); } // do the actual archive long start = EnvironmentEdgeManager.currentTime(); File file = new FileablePath(fs, storeFile); if (!resolveAndArchiveFile(storeArchiveDir, file, Long.toString(start))) { throw new IOException("Failed to archive/delete the file for region:" + regionInfo.getRegionNameAsString() + ", family:" + Bytes.toString(family) + " into " + storeArchiveDir + ". Something is probably awry on the filesystem."); } }
public HMobStore(final HRegion region, final ColumnFamilyDescriptor family, final Configuration confParam) throws IOException { super(region, family, confParam); this.family = family; this.mobCacheConfig = (MobCacheConfig) cacheConf; this.homePath = MobUtils.getMobHome(conf); this.mobFamilyPath = MobUtils.getMobFamilyPath(conf, this.getTableName(), family.getNameAsString()); List<Path> locations = new ArrayList<>(2); locations.add(mobFamilyPath); TableName tn = region.getTableDescriptor().getTableName(); locations.add(HFileArchiveUtil.getStoreArchivePath(conf, tn, MobUtils.getMobRegionInfo(tn) .getEncodedName(), family.getNameAsString())); map.put(Bytes.toString(tn.getName()), locations); List<Tag> tags = new ArrayList<>(2); tags.add(MobConstants.MOB_REF_TAG); Tag tableNameTag = new ArrayBackedTag(TagType.MOB_TABLE_NAME_TAG_TYPE, getTableName().getName()); tags.add(tableNameTag); this.refCellTags = TagUtil.fromList(tags); }
private void startServiceThreads() throws IOException{ // Start the executor service pools this.service.startExecutorService(ExecutorType.MASTER_OPEN_REGION, conf.getInt("hbase.master.executor.openregion.threads", 5)); this.service.startExecutorService(ExecutorType.MASTER_CLOSE_REGION, conf.getInt("hbase.master.executor.closeregion.threads", 5)); this.service.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS, conf.getInt("hbase.master.executor.serverops.threads", 5)); this.service.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS, conf.getInt("hbase.master.executor.serverops.threads", 5)); this.service.startExecutorService(ExecutorType.M_LOG_REPLAY_OPS, conf.getInt("hbase.master.executor.logreplayops.threads", 10)); // We depend on there being only one instance of this executor running // at a time. To do concurrency, would need fencing of enable/disable of // tables. // Any time changing this maxThreads to > 1, pls see the comment at // AccessController#postCreateTableHandler this.service.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, 1); startProcedureExecutor(); // Start log cleaner thread int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 60 * 1000); this.logCleaner = new LogCleaner(cleanerInterval, this, conf, getMasterFileSystem().getFileSystem(), getMasterFileSystem().getOldLogDir()); getChoreService().scheduleChore(logCleaner); //start the hfile archive cleaner thread Path archiveDir = HFileArchiveUtil.getArchivePath(conf); this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf, getMasterFileSystem() .getFileSystem(), archiveDir); getChoreService().scheduleChore(hfileCleaner); serviceStarted = true; if (LOG.isTraceEnabled()) { LOG.trace("Started service threads"); } }
@Test public void testRestoreSnapshotDoesNotCreateBackRefLinks() throws Exception { setupCluster(); TableName tableName = TableName.valueOf("testRestoreSnapshotDoesNotCreateBackRefLinks"); String snapshotName = "foo"; try { createTableAndSnapshot(UTIL, tableName, snapshotName, getStartRow(), getEndRow(), 1); Path tmpTableDir = UTIL.getRandomDir(); testRestoreSnapshotDoesNotCreateBackRefLinksInit(tableName, snapshotName,tmpTableDir); Path rootDir = FSUtils.getRootDir(UTIL.getConfiguration()); for (Path regionDir : FSUtils.getRegionDirs(fs, FSUtils.getTableDir(rootDir, tableName))) { for (Path storeDir : FSUtils.getFamilyDirs(fs, regionDir)) { for (FileStatus status : fs.listStatus(storeDir)) { System.out.println(status.getPath()); if (StoreFileInfo.isValid(status)) { Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(UTIL.getConfiguration(), tableName, regionDir.getName(), storeDir.getName()); Path path = HFileLink.getBackReferencesDir(storeDir, status.getPath().getName()); // assert back references directory is empty assertFalse("There is a back reference in " + path, fs.exists(path)); path = HFileLink.getBackReferencesDir(archiveStoreDir, status.getPath().getName()); // assert back references directory is empty assertFalse("There is a back reference in " + path, fs.exists(path)); } } } } } finally { UTIL.getHBaseAdmin().deleteSnapshot(snapshotName); UTIL.deleteTable(tableName); tearDownCluster(); } }
@Test (timeout=300000) public void testArchivingOnSingleTable() throws Exception { createArchiveDirectory(); FileSystem fs = UTIL.getTestFileSystem(); Path archiveDir = getArchiveDir(); Path tableDir = getTableDir(STRING_TABLE_NAME); toCleanup.add(archiveDir); toCleanup.add(tableDir); Configuration conf = UTIL.getConfiguration(); // setup the delegate Stoppable stop = new StoppableImplementation(); HFileCleaner cleaner = setupAndCreateCleaner(conf, fs, archiveDir, stop); List<BaseHFileCleanerDelegate> cleaners = turnOnArchiving(STRING_TABLE_NAME, cleaner); final LongTermArchivingHFileCleaner delegate = (LongTermArchivingHFileCleaner) cleaners.get(0); // create the region HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAM); Region region = UTIL.createTestRegion(STRING_TABLE_NAME, hcd); loadFlushAndCompact(region, TEST_FAM); // get the current hfiles in the archive directory List<Path> files = getAllFiles(fs, archiveDir); if (files == null) { FSUtils.logFileSystemState(fs, UTIL.getDataTestDir(), LOG); throw new RuntimeException("Didn't archive any files!"); } CountDownLatch finished = setupCleanerWatching(delegate, cleaners, files.size()); runCleaner(cleaner, finished, stop); // know the cleaner ran, so now check all the files again to make sure they are still there List<Path> archivedFiles = getAllFiles(fs, archiveDir); assertEquals("Archived files changed after running archive cleaner.", files, archivedFiles); // but we still have the archive directory assertTrue(fs.exists(HFileArchiveUtil.getArchivePath(UTIL.getConfiguration()))); }
private void assertArchiveFiles(FileSystem fs, List<String> storeFiles, long timeout) throws IOException { long end = System.currentTimeMillis() + timeout; Path archiveDir = HFileArchiveUtil.getArchivePath(UTIL.getConfiguration()); List<String> archivedFiles = new ArrayList<String>(); // We have to ensure that the DeleteTableHandler is finished. HBaseAdmin.deleteXXX() can return before all files // are archived. We should fix HBASE-5487 and fix synchronous operations from admin. while (System.currentTimeMillis() < end) { archivedFiles = getAllFileNames(fs, archiveDir); if (archivedFiles.size() >= storeFiles.size()) { break; } } Collections.sort(storeFiles); Collections.sort(archivedFiles); LOG.debug("Store files:"); for (int i = 0; i < storeFiles.size(); i++) { LOG.debug(i + " - " + storeFiles.get(i)); } LOG.debug("Archive files:"); for (int i = 0; i < archivedFiles.size(); i++) { LOG.debug(i + " - " + archivedFiles.get(i)); } assertTrue("Archived files are missing some of the store files!", archivedFiles.containsAll(storeFiles)); }
/** * Create a new HFileLink * * <p>It also adds a back-reference to the hfile back-reference directory * to simplify the reference-count and the cleaning process. * * @param conf {@link Configuration} to read for the archive directory name * @param fs {@link FileSystem} on which to write the HFileLink * @param dstFamilyPath - Destination path (table/region/cf/) * @param linkedTable - Linked Table Name * @param linkedRegion - Linked Region Name * @param hfileName - Linked HFile name * @return true if the file is created, otherwise the file exists. * @throws IOException on file or parent directory creation failure */ public static boolean create(final Configuration conf, final FileSystem fs, final Path dstFamilyPath, final String linkedTable, final String linkedRegion, final String hfileName) throws IOException { String familyName = dstFamilyPath.getName(); String regionName = dstFamilyPath.getParent().getName(); String tableName = dstFamilyPath.getParent().getParent().getName(); String name = createHFileLinkName(linkedTable, linkedRegion, hfileName); String refName = createBackReferenceName(tableName, regionName); // Make sure the destination directory exists fs.mkdirs(dstFamilyPath); // Make sure the FileLink reference directory exists Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf, linkedTable, linkedRegion, familyName); Path backRefssDir = getBackReferencesDir(archiveStoreDir, hfileName); fs.mkdirs(backRefssDir); // Create the reference for the link Path backRefPath = new Path(backRefssDir, refName); fs.createNewFile(backRefPath); try { // Create the link return fs.createNewFile(new Path(dstFamilyPath, name)); } catch (IOException e) { LOG.error("couldn't create the link=" + name + " for " + dstFamilyPath, e); // Revert the reference if the link creation failed fs.delete(backRefPath, false); throw e; } }
/** * Create a new HFileLink * * <p>It also adds a back-reference to the hfile back-reference directory * to simplify the reference-count and the cleaning process. * * @param conf {@link Configuration} to read for the archive directory name * @param fs {@link FileSystem} on which to write the HFileLink * @param dstFamilyPath - Destination path (table/region/cf/) * @param linkedTable - Linked Table Name * @param linkedRegion - Linked Region Name * @param hfileName - Linked HFile name * @return true if the file is created, otherwise the file exists. * @throws IOException on file or parent directory creation failure */ public static boolean create(final Configuration conf, final FileSystem fs, final Path dstFamilyPath, final TableName linkedTable, final String linkedRegion, final String hfileName) throws IOException { String familyName = dstFamilyPath.getName(); String regionName = dstFamilyPath.getParent().getName(); String tableName = FSUtils.getTableName(dstFamilyPath.getParent().getParent()) .getNameAsString(); String name = createHFileLinkName(linkedTable, linkedRegion, hfileName); String refName = createBackReferenceName(tableName, regionName); // Make sure the destination directory exists fs.mkdirs(dstFamilyPath); // Make sure the FileLink reference directory exists Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf, linkedTable, linkedRegion, familyName); Path backRefssDir = getBackReferencesDir(archiveStoreDir, hfileName); fs.mkdirs(backRefssDir); // Create the reference for the link Path backRefPath = new Path(backRefssDir, refName); fs.createNewFile(backRefPath); try { // Create the link return fs.createNewFile(new Path(dstFamilyPath, name)); } catch (IOException e) { LOG.error("couldn't create the link=" + name + " for " + dstFamilyPath, e); // Revert the reference if the link creation failed fs.delete(backRefPath, false); throw e; } }
@Test (timeout=300000) public void testArchivingOnSingleTable() throws Exception { createArchiveDirectory(); FileSystem fs = UTIL.getTestFileSystem(); Path archiveDir = getArchiveDir(); Path tableDir = getTableDir(STRING_TABLE_NAME); toCleanup.add(archiveDir); toCleanup.add(tableDir); Configuration conf = UTIL.getConfiguration(); // setup the delegate Stoppable stop = new StoppableImplementation(); HFileCleaner cleaner = setupAndCreateCleaner(conf, fs, archiveDir, stop); List<BaseHFileCleanerDelegate> cleaners = turnOnArchiving(STRING_TABLE_NAME, cleaner); final LongTermArchivingHFileCleaner delegate = (LongTermArchivingHFileCleaner) cleaners.get(0); // create the region HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAM); HRegion region = UTIL.createTestRegion(STRING_TABLE_NAME, hcd); loadFlushAndCompact(region, TEST_FAM); // get the current hfiles in the archive directory List<Path> files = getAllFiles(fs, archiveDir); if (files == null) { FSUtils.logFileSystemState(fs, UTIL.getDataTestDir(), LOG); throw new RuntimeException("Didn't archive any files!"); } CountDownLatch finished = setupCleanerWatching(delegate, cleaners, files.size()); runCleaner(cleaner, finished, stop); // know the cleaner ran, so now check all the files again to make sure they are still there List<Path> archivedFiles = getAllFiles(fs, archiveDir); assertEquals("Archived files changed after running archive cleaner.", files, archivedFiles); // but we still have the archive directory assertTrue(fs.exists(HFileArchiveUtil.getArchivePath(UTIL.getConfiguration()))); }
