/** * Remove the column family from the file system **/ public static void deleteColumnFamilyFromFileSystem( final MasterProcedureEnv env, final TableName tableName, List<HRegionInfo> regionInfoList, final byte[] familyName) throws IOException { final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); if (LOG.isDebugEnabled()) { LOG.debug("Removing family=" + Bytes.toString(familyName) + " from table=" + tableName); } if (regionInfoList == null) { regionInfoList = ProcedureSyncWait.getRegionsFromMeta(env, tableName); } for (HRegionInfo hri : regionInfoList) { // Delete the family directory in FS for all the regions one by one mfs.deleteFamilyFromFS(hri, familyName); } }
private void verifyHColumnDescriptor(int expected, final TableName tableName, final byte[]... families) throws IOException { Admin admin = TEST_UTIL.getHBaseAdmin(); // Verify descriptor from master HTableDescriptor htd = admin.getTableDescriptor(tableName); HColumnDescriptor[] hcds = htd.getColumnFamilies(); verifyHColumnDescriptor(expected, hcds, tableName, families); // Verify descriptor from HDFS MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem(); Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName); htd = FSTableDescriptors.getTableDescriptorFromFs(mfs.getFileSystem(), tableDir); hcds = htd.getColumnFamilies(); verifyHColumnDescriptor(expected, hcds, tableName, families); }
/** * This borrows code from MasterFileSystem.bootstrap() * * @return an open .META. HRegion */ private HRegion createNewRootAndMeta() throws IOException { Path rootdir = new Path(getConf().get(HConstants.HBASE_DIR)); Configuration c = getConf(); HRegionInfo rootHRI = new HRegionInfo(HRegionInfo.ROOT_REGIONINFO); MasterFileSystem.setInfoFamilyCachingForRoot(false); HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO); MasterFileSystem.setInfoFamilyCachingForMeta(false); HRegion root = HRegion.createHRegion(rootHRI, rootdir, c, HTableDescriptor.ROOT_TABLEDESC); HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c, HTableDescriptor.META_TABLEDESC); MasterFileSystem.setInfoFamilyCachingForRoot(true); MasterFileSystem.setInfoFamilyCachingForMeta(true); // Add first region from the META table to the ROOT region. HRegion.addRegionToMETA(root, meta); root.close(); root.getLog().closeAndDelete(); return meta; }
/** * Corrupt the specified snapshot by deleting some files. * * @param util {@link HBaseTestingUtility} * @param snapshotName name of the snapshot to corrupt * @return array of the corrupted HFiles * @throws IOException on unexecpted error reading the FS */ public static ArrayList corruptSnapshot(final HBaseTestingUtility util, final String snapshotName) throws IOException { final MasterFileSystem mfs = util.getHBaseCluster().getMaster().getMasterFileSystem(); final FileSystem fs = mfs.getFileSystem(); Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, mfs.getRootDir()); SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); final String table = snapshotDesc.getTable(); final ArrayList corruptedFiles = new ArrayList(); SnapshotReferenceUtil.visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() { public void storeFile (final String region, final String family, final String hfile) throws IOException { HFileLink link = HFileLink.create(util.getConfiguration(), table, region, family, hfile); if (corruptedFiles.size() % 2 == 0) { fs.delete(link.getAvailablePath(fs)); corruptedFiles.add(hfile); } } }); assertTrue(corruptedFiles.size() > 0); return corruptedFiles; }
/** * Removes from hdfs the families that are not longer present in the new table descriptor. */ private void deleteFamilyFromFS(final List<HRegionInfo> hris, final Set<byte[]> oldFamilies) { try { Set<byte[]> newFamilies = this.htd.getFamiliesKeys(); MasterFileSystem mfs = this.masterServices.getMasterFileSystem(); for (byte[] familyName: oldFamilies) { if (!newFamilies.contains(familyName)) { LOG.debug("Removing family=" + Bytes.toString(familyName) + " from table=" + this.tableName); for (HRegionInfo hri: hris) { // Delete the family directory in FS for all the regions one by one mfs.deleteFamilyFromFS(hri, familyName); } } } } catch (IOException e) { LOG.warn("Unable to remove on-disk directories for the removed families", e); } }
@Override protected void handleTableOperation(List<HRegionInfo> hris) throws IOException { MasterCoprocessorHost cpHost = ((HMaster) this.server) .getMasterCoprocessorHost(); if (cpHost != null) { cpHost.preDeleteColumnHandler(this.tableName, this.familyName); } // Update table descriptor this.masterServices.getMasterFileSystem().deleteColumn(tableName, familyName); // Remove the column family from the file system MasterFileSystem mfs = this.masterServices.getMasterFileSystem(); for (HRegionInfo hri : hris) { // Delete the family directory in FS for all the regions one by one mfs.deleteFamilyFromFS(hri, familyName); } if (cpHost != null) { cpHost.postDeleteColumnHandler(this.tableName, this.familyName); } }
@Override protected void handleTableOperation(List<HRegionInfo> hris) throws IOException { MasterCoprocessorHost cpHost = ((HMaster) this.server) .getCoprocessorHost(); if (cpHost != null) { cpHost.preDeleteColumnHandler(this.tableName, this.familyName); } // Update table descriptor this.masterServices.getMasterFileSystem().deleteColumn(tableName, familyName); // Remove the column family from the file system MasterFileSystem mfs = this.masterServices.getMasterFileSystem(); for (HRegionInfo hri : hris) { // Delete the family directory in FS for all the regions one by one mfs.deleteFamilyFromFS(hri, familyName); } if (cpHost != null) { cpHost.postDeleteColumnHandler(this.tableName, this.familyName); } }
public CreateTableHandler(Server server, MasterFileSystem fileSystemManager, HTableDescriptor hTableDescriptor, Configuration conf, HRegionInfo [] newRegions, MasterServices masterServices) { super(server, EventType.C_M_CREATE_TABLE); this.fileSystemManager = fileSystemManager; this.hTableDescriptor = hTableDescriptor; this.conf = conf; this.newRegions = newRegions; this.catalogTracker = masterServices.getCatalogTracker(); this.assignmentManager = masterServices.getAssignmentManager(); this.tableLockManager = masterServices.getTableLockManager(); this.tableLock = this.tableLockManager.writeLock(this.hTableDescriptor.getTableName() , EventType.C_M_CREATE_TABLE.toString()); }
private void verifyHColumnDescriptor(int expected, final TableName tableName, final byte[]... families) throws IOException { HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); // Verify descriptor from master HTableDescriptor htd = admin.getTableDescriptor(tableName); HColumnDescriptor[] hcds = htd.getColumnFamilies(); verifyHColumnDescriptor(expected, hcds, tableName, families); // Verify descriptor from HDFS MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem(); Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName); htd = FSTableDescriptors.getTableDescriptorFromFs(mfs.getFileSystem(), tableDir); hcds = htd.getColumnFamilies(); verifyHColumnDescriptor(expected, hcds, tableName, families); }
/** * Create merged region * @param env MasterProcedureEnv * @throws IOException */ private void createMergedRegion(final MasterProcedureEnv env) throws IOException { final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); final Path tabledir = FSUtils.getTableDir(mfs.getRootDir(), regionsToMerge[0].getTable()); final FileSystem fs = mfs.getFileSystem(); HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem( env.getMasterConfiguration(), fs, tabledir, regionsToMerge[0], false); regionFs.createMergesDir(); mergeStoreFiles(env, regionFs, regionFs.getMergesDir()); HRegionFileSystem regionFs2 = HRegionFileSystem.openRegionFromFileSystem( env.getMasterConfiguration(), fs, tabledir, regionsToMerge[1], false); mergeStoreFiles(env, regionFs2, regionFs.getMergesDir()); regionFs.commitMergedRegion(mergedRegion); //Prepare to create merged regions env.getAssignmentManager().getRegionStates(). getOrCreateRegionStateNode(mergedRegion).setState(State.MERGING_NEW); }
/** * Create reference file(s) of merging regions under the merges directory * @param env MasterProcedureEnv * @param regionFs region file system * @param mergedDir the temp directory of merged region * @throws IOException */ private void mergeStoreFiles( final MasterProcedureEnv env, final HRegionFileSystem regionFs, final Path mergedDir) throws IOException { final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); final Configuration conf = env.getMasterConfiguration(); final TableDescriptor htd = env.getMasterServices().getTableDescriptors().get(getTableName()); for (String family: regionFs.getFamilies()) { final ColumnFamilyDescriptor hcd = htd.getColumnFamily(Bytes.toBytes(family)); final Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(family); if (storeFiles != null && storeFiles.size() > 0) { final CacheConfig cacheConf = new CacheConfig(conf, hcd); for (StoreFileInfo storeFileInfo: storeFiles) { // Create reference file(s) of the region in mergedDir regionFs.mergeStoreFile(mergedRegion, family, new HStoreFile(mfs.getFileSystem(), storeFileInfo, conf, cacheConf, hcd.getBloomFilterType(), true), mergedDir); } } } }
/** * Create daughter regions * @param env MasterProcedureEnv * @throws IOException */ @VisibleForTesting public void createDaughterRegions(final MasterProcedureEnv env) throws IOException { final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); final Path tabledir = FSUtils.getTableDir(mfs.getRootDir(), getTableName()); final FileSystem fs = mfs.getFileSystem(); HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem( env.getMasterConfiguration(), fs, tabledir, getParentRegion(), false); regionFs.createSplitsDir(); Pair<Integer, Integer> expectedReferences = splitStoreFiles(env, regionFs); assertReferenceFileCount(fs, expectedReferences.getFirst(), regionFs.getSplitsDir(daughter_1_RI)); //Move the files from the temporary .splits to the final /table/region directory regionFs.commitDaughterRegion(daughter_1_RI); assertReferenceFileCount(fs, expectedReferences.getFirst(), new Path(tabledir, daughter_1_RI.getEncodedName())); assertReferenceFileCount(fs, expectedReferences.getSecond(), regionFs.getSplitsDir(daughter_2_RI)); regionFs.commitDaughterRegion(daughter_2_RI); assertReferenceFileCount(fs, expectedReferences.getSecond(), new Path(tabledir, daughter_2_RI.getEncodedName())); }
/** * Delete the namespace directories from the file system * @param env MasterProcedureEnv * @param namespaceName name of the namespace in string format * @throws IOException */ protected static void deleteDirectory( final MasterProcedureEnv env, final String namespaceName) throws IOException { MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); FileSystem fs = mfs.getFileSystem(); Path p = FSUtils.getNamespaceDir(mfs.getRootDir(), namespaceName); try { for(FileStatus status : fs.listStatus(p)) { if (!HConstants.HBASE_NON_TABLE_DIRS.contains(status.getPath().getName())) { throw new IOException("Namespace directory contains table dir: " + status.getPath()); } } if (!fs.delete(FSUtils.getNamespaceDir(mfs.getRootDir(), namespaceName), true)) { throw new IOException("Failed to remove namespace: " + namespaceName); } } catch (FileNotFoundException e) { // File already deleted, continue LOG.debug("deleteDirectory throws exception: " + e); } }
protected static List<RegionInfo> createFsLayout(final MasterProcedureEnv env, final TableDescriptor tableDescriptor, List<RegionInfo> newRegions, final CreateHdfsRegions hdfsRegionHandler) throws IOException { final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); final Path tempdir = mfs.getTempDir(); // 1. Create Table Descriptor // using a copy of descriptor, table will be created enabling first final Path tempTableDir = FSUtils.getTableDir(tempdir, tableDescriptor.getTableName()); ((FSTableDescriptors)(env.getMasterServices().getTableDescriptors())) .createTableDescriptorForTableDirectory( tempTableDir, tableDescriptor, false); // 2. Create Regions newRegions = hdfsRegionHandler.createHdfsRegions(env, tempdir, tableDescriptor.getTableName(), newRegions); // 3. Move Table temp directory to the hbase root location moveTempDirectoryToHBaseRoot(env, tableDescriptor, tempTableDir); return newRegions; }
/** * Remove the column family from the file system **/ public static void deleteColumnFamilyFromFileSystem( final MasterProcedureEnv env, final TableName tableName, final List<RegionInfo> regionInfoList, final byte[] familyName, final boolean hasMob) throws IOException { final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); if (LOG.isDebugEnabled()) { LOG.debug("Removing family=" + Bytes.toString(familyName) + " from table=" + tableName); } for (RegionInfo hri : regionInfoList) { // Delete the family directory in FS for all the regions one by one mfs.deleteFamilyFromFS(hri, familyName); } if (hasMob) { // Delete the mob region Path mobRootDir = new Path(mfs.getRootDir(), MobConstants.MOB_DIR_NAME); RegionInfo mobRegionInfo = MobUtils.getMobRegionInfo(tableName); mfs.deleteFamilyFromFS(mobRootDir, mobRegionInfo, familyName); } }
/** * Action before cloning from snapshot. * @param env MasterProcedureEnv * @throws IOException * @throws InterruptedException */ private void preCloneSnapshot(final MasterProcedureEnv env) throws IOException, InterruptedException { if (!getTableName().isSystemTable()) { // Check and update namespace quota final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); SnapshotManifest manifest = SnapshotManifest.open( env.getMasterConfiguration(), mfs.getFileSystem(), SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, mfs.getRootDir()), snapshot); ProcedureSyncWait.getMasterQuotaManager(env) .checkNamespaceTableAndRegionQuota(getTableName(), manifest.getRegionManifestsMap().size()); } final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); if (cpHost != null) { cpHost.preCreateTableAction(tableDescriptor, null, getUser()); } }
/** * Create region layout in file system. * @param env MasterProcedureEnv * @throws IOException */ private List<RegionInfo> createFsLayout( final MasterProcedureEnv env, final TableDescriptor tableDescriptor, List<RegionInfo> newRegions, final CreateHdfsRegions hdfsRegionHandler) throws IOException { final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); final Path tempdir = mfs.getTempDir(); // 1. Create Table Descriptor // using a copy of descriptor, table will be created enabling first final Path tempTableDir = FSUtils.getTableDir(tempdir, tableDescriptor.getTableName()); ((FSTableDescriptors)(env.getMasterServices().getTableDescriptors())) .createTableDescriptorForTableDirectory(tempTableDir, TableDescriptorBuilder.newBuilder(tableDescriptor).build(), false); // 2. Create Regions newRegions = hdfsRegionHandler.createHdfsRegions( env, tempdir, tableDescriptor.getTableName(), newRegions); // 3. Move Table temp directory to the hbase root location CreateTableProcedure.moveTempDirectoryToHBaseRoot(env, tableDescriptor, tempTableDir); return newRegions; }
/** * This borrows code from MasterFileSystem.bootstrap(). Explicitly creates it's own WAL, so be * sure to close it as well as the region when you're finished. * @param walFactoryID A unique identifier for WAL factory. Filesystem implementations will use * this ID to make a directory inside WAL directory path. * @return an open hbase:meta HRegion */ private HRegion createNewMeta(String walFactoryID) throws IOException { Path rootdir = FSUtils.getRootDir(getConf()); Configuration c = getConf(); RegionInfo metaHRI = RegionInfoBuilder.FIRST_META_REGIONINFO; TableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME); MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, false); // The WAL subsystem will use the default rootDir rather than the passed in rootDir // unless I pass along via the conf. Configuration confForWAL = new Configuration(c); confForWAL.set(HConstants.HBASE_DIR, rootdir.toString()); WAL wal = new WALFactory(confForWAL, walFactoryID).getWAL(metaHRI); HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c, metaDescriptor, wal); MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, true); return meta; }
private void verifyHColumnDescriptor(int expected, final TableName tableName, final byte[]... families) throws IOException { Admin admin = TEST_UTIL.getAdmin(); // Verify descriptor from master TableDescriptor htd = admin.getDescriptor(tableName); ColumnFamilyDescriptor[] hcds = htd.getColumnFamilies(); verifyHColumnDescriptor(expected, hcds, tableName, families); // Verify descriptor from HDFS MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem(); Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName); TableDescriptor td = FSTableDescriptors.getTableDescriptorFromFs(mfs.getFileSystem(), tableDir); hcds = td.getColumnFamilies(); verifyHColumnDescriptor(expected, hcds, tableName, families); }
@Test public void testRefreshRegionHFilesEndpoint() throws Exception { setUp(HRegion.class.getName()); MasterFileSystem mfs = HTU.getMiniHBaseCluster().getMaster().getMasterFileSystem(); Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), TABLE_NAME); for (Region region : cluster.getRegions(TABLE_NAME)) { Path regionDir = new Path(tableDir, region.getRegionInfo().getEncodedName()); Path familyDir = new Path(regionDir, Bytes.toString(FAMILY)); HFileTestUtil .createHFile(HTU.getConfiguration(), HTU.getTestFileSystem(), new Path(familyDir, HFILE_NAME), FAMILY, QUALIFIER, Bytes.toBytes("50"), Bytes.toBytes("60"), NUM_ROWS); } assertEquals(2, HTU.getNumHFiles(TABLE_NAME, FAMILY)); callRefreshRegionHFilesEndPoint(); assertEquals(4, HTU.getNumHFiles(TABLE_NAME, FAMILY)); }
/** * This borrows code from MasterFileSystem.bootstrap() * * @return an open .META. HRegion */ private HRegion createNewRootAndMeta() throws IOException { Path rootdir = new Path(conf.get(HConstants.HBASE_DIR)); Configuration c = conf; HRegionInfo rootHRI = new HRegionInfo(HRegionInfo.ROOT_REGIONINFO); MasterFileSystem.setInfoFamilyCachingForRoot(false); HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO); MasterFileSystem.setInfoFamilyCachingForMeta(false); HRegion root = HRegion.createHRegion(rootHRI, rootdir, c, HTableDescriptor.ROOT_TABLEDESC); HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c, HTableDescriptor.META_TABLEDESC); MasterFileSystem.setInfoFamilyCachingForRoot(true); MasterFileSystem.setInfoFamilyCachingForMeta(true); // Add first region from the META table to the ROOT region. HRegion.addRegionToMETA(root, meta); root.close(); root.getLog().closeAndDelete(); return meta; }
@Override protected void handleTableOperation(List<HRegionInfo> hris) throws IOException { MasterCoprocessorHost cpHost = ((HMaster) this.server) .getCoprocessorHost(); if (cpHost != null) { cpHost.preDeleteColumnHandler(this.tableName, this.familyName); } // Update table descriptor in HDFS HTableDescriptor htd = this.masterServices.getMasterFileSystem().deleteColumn(tableName, familyName); // Update in-memory descriptor cache this.masterServices.getTableDescriptors().add(htd); // Remove the column family from the file system MasterFileSystem mfs = this.masterServices.getMasterFileSystem(); for (HRegionInfo hri : hris) { // Delete the family directory in FS for all the regions one by one mfs.deleteFamilyFromFS(hri, familyName); } if (cpHost != null) { cpHost.postDeleteColumnHandler(this.tableName, this.familyName); } }
/** * This borrows code from MasterFileSystem.bootstrap() * * @return an open .META. HRegion */ private HRegion createNewRootAndMeta() throws IOException { Path rootdir = new Path(getConf().get(HConstants.HBASE_DIR)); Configuration c = getConf(); HRegionInfo rootHRI = new HRegionInfo(HRegionInfo.ROOT_REGIONINFO); MasterFileSystem.setInfoFamilyCachingForRoot(false); HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO); MasterFileSystem.setInfoFamilyCachingForMeta(false); HRegion root = HRegion.createHRegion(rootHRI, rootdir, c, HTableDescriptor.ROOT_TABLEDESC); HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c, HTableDescriptor.META_TABLEDESC); MasterFileSystem.setInfoFamilyCachingForRoot(true); MasterFileSystem.setInfoFamilyCachingForMeta(true); // Add first region from the META table to the ROOT region. HRegion.addRegionToMETA(root, meta); HRegion.closeHRegion(root); return meta; }
@Override protected void handleTableOperation(List<HRegionInfo> hris) throws IOException { MasterCoprocessorHost cpHost = ((HMaster) this.server).getCoprocessorHost(); if (cpHost != null) { cpHost.preDeleteColumnHandler(this.tableName, this.familyName); } MasterFileSystem mfs = this.masterServices.getMasterFileSystem(); // Update table descriptor mfs.deleteColumn(tableName, familyName); // Remove the column family from the file system for (HRegionInfo hri : hris) { // Delete the family directory in FS for all the regions one by one mfs.deleteFamilyFromFS(hri, familyName); } if (cpHost != null) { cpHost.postDeleteColumnHandler(this.tableName, this.familyName); } }
public CreateTableHandler(Server server, MasterFileSystem fileSystemManager, HTableDescriptor hTableDescriptor, Configuration conf, HRegionInfo [] newRegions, MasterServices masterServices) { super(server, EventType.C_M_CREATE_TABLE); this.fileSystemManager = fileSystemManager; this.hTableDescriptor = hTableDescriptor; this.conf = conf; this.newRegions = newRegions; this.assignmentManager = masterServices.getAssignmentManager(); this.tableLockManager = masterServices.getTableLockManager(); this.tableLock = this.tableLockManager.writeLock(this.hTableDescriptor.getTableName() , EventType.C_M_CREATE_TABLE.toString()); }