/** * Refresh all the region locations. * * @return true if user created regions got refreshed. */ private boolean scheduleFullRefresh() { // Protect from anything being null while starting up. if (services == null) { return false; } AssignmentManager am = services.getAssignmentManager(); if (am == null) { return false; } RegionStates regionStates = am.getRegionStates(); if (regionStates == null) { return false; } Set<HRegionInfo> regions = regionStates.getRegionAssignments().keySet(); boolean includesUserTables = false; for (final HRegionInfo hri : regions) { cache.refresh(hri); includesUserTables = includesUserTables || !hri.isSystemTable(); } return includesUserTables; }
@Override protected void populatePool(ExecutorService pool) { RegionStates regionStates = assignmentManager.getRegionStates(); for (HRegionInfo region: regions) { if (regionStates.isRegionInTransition(region) && !regionStates.isRegionInState(region, State.FAILED_CLOSE)) { continue; } final HRegionInfo hri = region; pool.execute(Trace.wrap("DisableTableHandler.BulkDisabler",new Runnable() { public void run() { assignmentManager.unassign(hri, true); } })); } }
/** * @param regionsInMeta * @return List of regions neither in transition nor assigned. * @throws IOException */ private Map<HRegionInfo, ServerName> regionsToAssignWithServerName( final List<Pair<HRegionInfo, ServerName>> regionsInMeta) throws IOException { Map<HRegionInfo, ServerName> regionsToAssign = new HashMap<HRegionInfo, ServerName>(regionsInMeta.size()); RegionStates regionStates = this.assignmentManager.getRegionStates(); for (Pair<HRegionInfo, ServerName> regionLocation : regionsInMeta) { HRegionInfo hri = regionLocation.getFirst(); ServerName sn = regionLocation.getSecond(); if (regionStates.isRegionOffline(hri)) { regionsToAssign.put(hri, sn); } else { if (LOG.isDebugEnabled()) { LOG.debug("Skipping assign for the region " + hri + " during enable table " + hri.getTable() + " because its already in tranition or assigned."); } } } return regionsToAssign; }
@Override protected void populatePool(ExecutorService pool) { RegionStates regionStates = assignmentManager.getRegionStates(); for (final HRegionInfo region : regions) { if (regionStates.isRegionInTransition(region) && !regionStates.isRegionInState(region, RegionState.State.FAILED_CLOSE)) { continue; } pool.execute(Trace.wrap("DisableTableHandler.BulkDisabler", new Runnable() { @Override public void run() { assignmentManager.unassign(region); } })); } }
protected static void waitRegionInTransition(final MasterProcedureEnv env, final List<HRegionInfo> regions) throws IOException, CoordinatedStateException { final AssignmentManager am = env.getMasterServices().getAssignmentManager(); final RegionStates states = am.getRegionStates(); for (final HRegionInfo region : regions) { ProcedureSyncWait.waitFor(env, "regions " + region.getRegionNameAsString() + " in transition", new ProcedureSyncWait.Predicate<Boolean>() { @Override public Boolean evaluate() throws IOException { if (states.isRegionInState(region, State.FAILED_OPEN)) { am.regionOffline(region); } return !states.isRegionInTransition(region); } }); } }
/** * @param regionsInMeta * @return List of regions neither in transition nor assigned. * @throws IOException */ private static Map<HRegionInfo, ServerName> regionsToAssignWithServerName( final MasterProcedureEnv env, final List<Pair<HRegionInfo, ServerName>> regionsInMeta) throws IOException { Map<HRegionInfo, ServerName> regionsToAssign = new HashMap<HRegionInfo, ServerName>(regionsInMeta.size()); RegionStates regionStates = env.getMasterServices().getAssignmentManager().getRegionStates(); for (Pair<HRegionInfo, ServerName> regionLocation : regionsInMeta) { HRegionInfo hri = regionLocation.getFirst(); ServerName sn = regionLocation.getSecond(); if (regionStates.isRegionOffline(hri)) { regionsToAssign.put(hri, sn); } else { if (LOG.isDebugEnabled()) { LOG.debug("Skipping assign for the region " + hri + " during enable table " + hri.getTable() + " because its already in tranition or assigned."); } } } return regionsToAssign; }
/** * Wait until all the regions are assigned. */ private void waitForAllRegionsAssigned() throws IOException { int totalRegions = HBaseTestingUtility.KEYS.length; while (UTIL.getMiniHBaseCluster().countServedRegions() < totalRegions) { // while (!cluster.getMaster().allRegionsAssigned()) { LOG.debug("Waiting for there to be "+ totalRegions +" regions, but there are " + UTIL.getMiniHBaseCluster().countServedRegions() + " right now."); try { Thread.sleep(200); } catch (InterruptedException e) {} } RegionStates regionStates = UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates(); while (!regionStates.getRegionsInTransition().isEmpty()) { Threads.sleep(100); } }
private boolean checkRegionInTransition(ObserverContext<MasterCoprocessorEnvironment> ctx, HRegionInfo hri) { MasterServices master = ctx.getEnvironment().getMasterServices(); RegionStates regionStates = master.getAssignmentManager().getRegionStates(); String tableName = hri.getTable().getNameAsString(); if (!IndexUtils.isIndexTable(tableName)) { if (regionStates.isRegionInTransition(hri)) { return true; } else { String indexTableName = IndexUtils.getIndexTableName(tableName); for (Entry<String, RegionState> region : regionStates.getRegionsInTransition().entrySet()) { HRegionInfo regionInfo = region.getValue().getRegion(); if (indexTableName.equals(regionInfo.getTable().getNameAsString())) { if (Bytes.compareTo(hri.getStartKey(), regionInfo.getStartKey()) == 0) { return true; } } } } } return false; }
/** * @param regionsInMeta * @return List of regions neither in transition nor assigned. * @throws IOException */ private List<HRegionInfo> regionsToAssignWithServerName( final List<Pair<HRegionInfo, ServerName>> regionsInMeta) throws IOException { ServerManager serverManager = ((HMaster) this.server).getServerManager(); List<HRegionInfo> regions = new ArrayList<HRegionInfo>(); RegionStates regionStates = this.assignmentManager.getRegionStates(); for (Pair<HRegionInfo, ServerName> regionLocation : regionsInMeta) { HRegionInfo hri = regionLocation.getFirst(); ServerName sn = regionLocation.getSecond(); if (regionStates.isRegionOffline(hri)) { if (sn != null && serverManager.isServerOnline(sn)) { this.assignmentManager.addPlan(hri.getEncodedName(), new RegionPlan(hri, null, sn)); } regions.add(hri); } else { if (LOG.isDebugEnabled()) { LOG.debug("Skipping assign for the region " + hri + " during enable table " + hri.getTable() + " because its already in tranition or assigned."); } } } return regions; }
/** * @param regionsInMeta * @return List of regions neither in transition nor assigned. * @throws IOException */ private List<HRegionInfo> regionsToAssignWithServerName( final List<Pair<HRegionInfo, ServerName>> regionsInMeta) throws IOException { ServerManager serverManager = ((HMaster) this.server).getServerManager(); List<HRegionInfo> regions = new ArrayList<HRegionInfo>(); RegionStates regionStates = this.assignmentManager.getRegionStates(); for (Pair<HRegionInfo, ServerName> regionLocation : regionsInMeta) { HRegionInfo hri = regionLocation.getFirst(); ServerName sn = regionLocation.getSecond(); if (!regionStates.isRegionInTransition(hri) && !regionStates.isRegionAssigned(hri)) { if (this.retainAssignment && sn != null && serverManager.isServerOnline(sn)) { this.assignmentManager.addPlan(hri.getEncodedName(), new RegionPlan(hri, null, sn)); } regions.add(hri); } else { if (LOG.isDebugEnabled()) { LOG.debug("Skipping assign for the region " + hri + " during enable table " + hri.getTableNameAsString() + " because its already in tranition or assigned."); } } } return regions; }
private void forceRegionsOffline(final List<HRegionInfo> hris) { AssignmentManager am = this.masterServices.getAssignmentManager(); RegionStates states = am.getRegionStates(); if (hris != null) { for (HRegionInfo hri: hris) { states.regionOffline(hri); } } }
@Test (timeout=300000) public void testCreateTableWithSplitRegion() throws Exception { final TableName tableName = TableName.valueOf("testCreateTableWithSplitRegion"); final MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); final HMaster m = cluster.getMaster(); final HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(new HColumnDescriptor(FAMILYNAME)); byte[] splitPoint = Bytes.toBytes("split-point"); long ts = System.currentTimeMillis(); HRegionInfo d1 = new HRegionInfo(desc.getTableName(), null, splitPoint, false, ts); HRegionInfo d2 = new HRegionInfo(desc.getTableName(), splitPoint, null, false, ts + 1); HRegionInfo parent = new HRegionInfo(desc.getTableName(), null, null, true, ts + 2); parent.setOffline(true); Path tempdir = m.getMasterFileSystem().getTempDir(); FileSystem fs = m.getMasterFileSystem().getFileSystem(); Path tempTableDir = FSUtils.getTableDir(tempdir, desc.getTableName()); fs.delete(tempTableDir, true); // Clean up temp table dir if exists final HRegionInfo[] hRegionInfos = new HRegionInfo[] {d1, d2, parent}; CreateTableHandler handler = new CreateTableHandler(m, m.getMasterFileSystem(), desc, cluster.getConfiguration(), hRegionInfos, m); handler.prepare(); handler.process(); for (int i = 0; i < 100; i++) { if (!TEST_UTIL.getHBaseAdmin().isTableAvailable(tableName)) { Thread.sleep(300); } } assertTrue(TEST_UTIL.getHBaseAdmin().isTableEnabled(tableName)); assertTrue(TEST_UTIL.getHBaseAdmin().isTableAvailable(tableName)); assertTrue(TEST_UTIL.getHBaseAdmin().isTableAvailable(tableName, new byte[][] { splitPoint })); RegionStates regionStates = m.getAssignmentManager().getRegionStates(); assertTrue("Parent should be in SPLIT state", regionStates.isRegionInState(parent, State.SPLIT)); }
protected void waitRegionInTransition(final List<HRegionInfo> regions) throws IOException, CoordinatedStateException { AssignmentManager am = this.masterServices.getAssignmentManager(); RegionStates states = am.getRegionStates(); long waitTime = server.getConfiguration(). getLong("hbase.master.wait.on.region", 5 * 60 * 1000); for (HRegionInfo region : regions) { long done = System.currentTimeMillis() + waitTime; while (System.currentTimeMillis() < done) { if (states.isRegionInState(region, State.FAILED_OPEN)) { am.regionOffline(region); } if (!states.isRegionInTransition(region)) break; try { Thread.sleep(waitingTimeForEvents); } catch (InterruptedException e) { LOG.warn("Interrupted while sleeping"); throw (InterruptedIOException)new InterruptedIOException().initCause(e); } LOG.debug("Waiting on region to clear regions in transition; " + am.getRegionStates().getRegionTransitionState(region)); } if (states.isRegionInTransition(region)) { throw new IOException("Waited hbase.master.wait.on.region (" + waitTime + "ms) for region to leave region " + region.getRegionNameAsString() + " in transitions"); } } }
/** * Returns a {@link Predicate} for checking that there are no regions in transition in master */ public Waiter.Predicate<Exception> predicateNoRegionsInTransition() { return new Waiter.Predicate<Exception>() { @Override public boolean evaluate() throws Exception { final RegionStates regionStates = getMiniHBaseCluster().getMaster() .getAssignmentManager().getRegionStates(); return !regionStates.isRegionsInTransition(); } }; }
@Test(timeout = 120000) public void testFailedSplit() throws Exception { TableName tableName = TableName.valueOf("testFailedSplit"); byte[] colFamily = Bytes.toBytes("info"); TESTING_UTIL.createTable(tableName, colFamily); Connection connection = ConnectionFactory.createConnection(TESTING_UTIL.getConfiguration()); HTable table = (HTable) connection.getTable(tableName); try { TESTING_UTIL.loadTable(table, colFamily); List<HRegionInfo> regions = TESTING_UTIL.getHBaseAdmin().getTableRegions(tableName); assertTrue(regions.size() == 1); final HRegion actualRegion = cluster.getRegions(tableName).get(0); actualRegion.getCoprocessorHost().load(FailingSplitRegionObserver.class, Coprocessor.PRIORITY_USER, actualRegion.getBaseConf()); // The following split would fail. admin.split(tableName); FailingSplitRegionObserver observer = (FailingSplitRegionObserver) actualRegion .getCoprocessorHost().findCoprocessor(FailingSplitRegionObserver.class.getName()); assertNotNull(observer); observer.latch.await(); observer.postSplit.await(); LOG.info("Waiting for region to come out of RIT"); TESTING_UTIL.waitFor(60000, 1000, new Waiter.Predicate<Exception>() { @Override public boolean evaluate() throws Exception { RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates(); Map<String, RegionState> rit = regionStates.getRegionsInTransition(); return (rit.size() == 0); } }); regions = TESTING_UTIL.getHBaseAdmin().getTableRegions(tableName); assertTrue(regions.size() == 1); assertTrue(admin.balancer()); } finally { table.close(); connection.close(); TESTING_UTIL.deleteTable(tableName); } }
@Test (timeout=300000) public void testCreateTableWithSplitRegion() throws Exception { final byte[] tableName = Bytes.toBytes("testCreateTableWithSplitRegion"); final MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); final HMaster m = cluster.getMaster(); final HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName)); desc.addFamily(new HColumnDescriptor(FAMILYNAME)); byte[] splitPoint = Bytes.toBytes("split-point"); long ts = System.currentTimeMillis(); HRegionInfo d1 = new HRegionInfo(desc.getTableName(), null, splitPoint, false, ts); HRegionInfo d2 = new HRegionInfo(desc.getTableName(), splitPoint, null, false, ts + 1); HRegionInfo parent = new HRegionInfo(desc.getTableName(), null, null, true, ts + 2); parent.setOffline(true); Path tempdir = m.getMasterFileSystem().getTempDir(); FileSystem fs = m.getMasterFileSystem().getFileSystem(); Path tempTableDir = FSUtils.getTableDir(tempdir, desc.getTableName()); fs.delete(tempTableDir, true); // Clean up temp table dir if exists final HRegionInfo[] hRegionInfos = new HRegionInfo[] {d1, d2, parent}; CreateTableHandler handler = new CreateTableHandler(m, m.getMasterFileSystem(), desc, cluster.getConfiguration(), hRegionInfos, m); handler.prepare(); handler.process(); for (int i = 0; i < 100; i++) { if (!TEST_UTIL.getHBaseAdmin().isTableAvailable(tableName)) { Thread.sleep(300); } } assertTrue(TEST_UTIL.getHBaseAdmin().isTableEnabled(tableName)); assertTrue(TEST_UTIL.getHBaseAdmin().isTableAvailable(tableName)); assertTrue(TEST_UTIL.getHBaseAdmin().isTableAvailable(tableName, new byte[][] { splitPoint })); RegionStates regionStates = m.getAssignmentManager().getRegionStates(); assertTrue("Parent should be in SPLIT state", regionStates.isRegionInState(parent, State.SPLIT)); }
@Override public void postAssign(ObserverContext<MasterCoprocessorEnvironment> ctx, HRegionInfo regionInfo) throws IOException { LOG.info("Entering into postAssign of region " + regionInfo.getRegionNameAsString() + '.'); if (!IndexUtils.isIndexTable(regionInfo.getTable().getName())) { MasterServices master = ctx.getEnvironment().getMasterServices(); LoadBalancer balancer = master.getAssignmentManager().getBalancer(); AssignmentManager am = master.getAssignmentManager(); RegionStates regionStates = am.getRegionStates(); // waiting until user region is removed from transition. long timeout = master.getConfiguration() .getLong("hbase.bulk.assignment.waiton.empty.rit", 5 * 60 * 1000); try { am.waitOnRegionToClearRegionsInTransition(regionInfo, timeout); } catch (InterruptedException e) { if (LOG.isDebugEnabled()) { LOG.debug("Interrupted while region in assignment."); } } ServerName sn = regionStates.getRegionServerOfRegion(regionInfo); TableName indexTableName = TableName.valueOf(IndexUtils.getIndexTableName(regionInfo.getTableName())); List<HRegionInfo> tableRegions = regionStates.getRegionsOfTable(indexTableName); for (HRegionInfo hRegionInfo : tableRegions) { if (0 == Bytes.compareTo(hRegionInfo.getStartKey(), regionInfo.getStartKey())) { am.addPlan(hRegionInfo.getEncodedName(), new RegionPlan(hRegionInfo, null, sn)); LOG.info("Assigning region " + hRegionInfo.getRegionNameAsString() + " to server " + sn + '.'); balancer.regionOnline(hRegionInfo, sn); am.assign(hRegionInfo, true, false); break; } } } LOG.info("Exiting from postAssign " + regionInfo.getRegionNameAsString() + '.'); }
/** * Returns a {@link Predicate} for checking that there is no regions in transition in master */ public Waiter.Predicate<Exception> predicateNoRegionsInTransition() { return new Waiter.Predicate<Exception>() { @Override public boolean evaluate() throws Exception { final RegionStates regionStates = getMiniHBaseCluster().getMaster() .getAssignmentManager().getRegionStates(); return !regionStates.isRegionsInTransition(); } }; }
@Override protected void populatePool(ExecutorService pool) { RegionStates regionStates = assignmentManager.getRegionStates(); for (HRegionInfo region: regions) { if (regionStates.isRegionInTransition(region)) continue; final HRegionInfo hri = region; pool.execute(Trace.wrap(new Runnable() { public void run() { assignmentManager.unassign(hri); } })); } }
/** * Wait until all regions for a table in hbase:meta have a non-empty * info:server, or until timeout. This means all regions have been deployed, * master has been informed and updated hbase:meta with the regions deployed * server. * @param tableName the table name * @param timeout timeout, in milliseconds * @throws IOException */ public void waitUntilAllRegionsAssigned(final TableName tableName, final long timeout) throws IOException { final Table meta = new HTable(getConfiguration(), TableName.META_TABLE_NAME); try { waitFor(timeout, 200, true, new Predicate<IOException>() { @Override public boolean evaluate() throws IOException { boolean allRegionsAssigned = true; Scan scan = new Scan(); scan.addFamily(HConstants.CATALOG_FAMILY); ResultScanner s = meta.getScanner(scan); try { Result r; while ((r = s.next()) != null) { byte[] b = r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); HRegionInfo info = HRegionInfo.parseFromOrNull(b); if (info != null && info.getTable().equals(tableName)) { b = r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER); allRegionsAssigned &= (b != null); } } } finally { s.close(); } return allRegionsAssigned; } }); } finally { meta.close(); } // check from the master state if we are using a mini cluster if (!getHBaseClusterInterface().isDistributedCluster()) { // So, all regions are in the meta table but make sure master knows of the assignments before // returing -- sometimes this can lag. HMaster master = getHBaseCluster().getMaster(); final RegionStates states = master.getAssignmentManager().getRegionStates(); waitFor(timeout, 200, new ExplainingPredicate<IOException>() { @Override public String explainFailure() throws IOException { return explainTableAvailability(tableName); } @Override public boolean evaluate() throws IOException { List<HRegionInfo> hris = states.getRegionsOfTable(tableName); return hris != null && !hris.isEmpty(); } }); } }
@Test public void testWholesomeMerge() throws Exception { LOG.info("Starting testWholesomeMerge"); final TableName tableName = TableName.valueOf("testWholesomeMerge"); // Create table and load data. Table table = createTableAndLoadData(master, tableName); // Merge 1st and 2nd region mergeRegionsAndVerifyRegionNum(master, tableName, 0, 1, INITIAL_REGION_NUM - 1); // Merge 2nd and 3th region PairOfSameType<HRegionInfo> mergedRegions = mergeRegionsAndVerifyRegionNum(master, tableName, 1, 2, INITIAL_REGION_NUM - 2); verifyRowCount(table, ROWSIZE); // Randomly choose one of the two merged regions HRegionInfo hri = RandomUtils.nextBoolean() ? mergedRegions.getFirst() : mergedRegions.getSecond(); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); AssignmentManager am = cluster.getMaster().getAssignmentManager(); RegionStates regionStates = am.getRegionStates(); long start = EnvironmentEdgeManager.currentTime(); while (!regionStates.isRegionInState(hri, State.MERGED)) { assertFalse("Timed out in waiting one merged region to be in state MERGED", EnvironmentEdgeManager.currentTime() - start > 60000); Thread.sleep(500); } // We should not be able to assign it again am.assign(hri, true, true); assertFalse("Merged region can't be assigned", regionStates.isRegionInTransition(hri)); assertTrue(regionStates.isRegionInState(hri, State.MERGED)); // We should not be able to unassign it either am.unassign(hri, true, null); assertFalse("Merged region can't be unassigned", regionStates.isRegionInTransition(hri)); assertTrue(regionStates.isRegionInState(hri, State.MERGED)); table.close(); }
@Test(timeout = 60000) public void testRITStateForRollback() throws Exception { final TableName tableName = TableName.valueOf("testRITStateForRollback"); try { // Create table then get the single region for our new table. Table t = createTableAndWait(tableName, Bytes.toBytes("cf")); final List<HRegion> regions = cluster.getRegions(tableName); final HRegionInfo hri = getAndCheckSingleTableRegion(regions); insertData(tableName, admin, t); t.close(); // Turn off balancer so it doesn't cut in and mess up our placements. this.admin.setBalancerRunning(false, true); // Turn off the meta scanner so it don't remove parent on us. cluster.getMaster().setCatalogJanitorEnabled(false); // find a splittable region final HRegion region = findSplittableRegion(regions); assertTrue("not able to find a splittable region", region != null); // install region co-processor to fail splits region.getCoprocessorHost().load(FailingSplitRegionObserver.class, Coprocessor.PRIORITY_USER, region.getBaseConf()); // split async this.admin.split(region.getRegionInfo().getRegionName(), new byte[] {42}); // we have to wait until the SPLITTING state is seen by the master FailingSplitRegionObserver observer = (FailingSplitRegionObserver) region .getCoprocessorHost().findCoprocessor(FailingSplitRegionObserver.class.getName()); assertNotNull(observer); observer.latch.await(); LOG.info("Waiting for region to come out of RIT"); TESTING_UTIL.waitFor(60000, 1000, new Waiter.Predicate<Exception>() { @Override public boolean evaluate() throws Exception { RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates(); Map<String, RegionState> rit = regionStates.getRegionsInTransition(); return (rit.size() == 0); } }); } finally { admin.setBalancerRunning(true, false); cluster.getMaster().setCatalogJanitorEnabled(true); TESTING_UTIL.deleteTable(tableName); } }
