/** * @param parent Region to split. * @param midkey Key to split around. * @return The Regions we created. * @throws IOException */ HRegion [] splitRegion(final HRegion parent, final byte [] midkey) throws IOException { PairOfSameType<HRegion> result = null; SplitTransaction st = new SplitTransaction(parent, midkey); // If prepare does not return true, for some reason -- logged inside in // the prepare call -- we are not ready to split just now. Just return. if (!st.prepare()) return null; try { result = st.execute(null, null); } catch (IOException ioe) { try { LOG.info("Running rollback of failed split of " + parent.getRegionNameAsString() + "; " + ioe.getMessage()); st.rollback(null, null); LOG.info("Successful rollback of failed split of " + parent.getRegionNameAsString()); return null; } catch (RuntimeException e) { // If failed rollback, kill this server to avoid having a hole in table. LOG.info("Failed rollback of failed split of " + parent.getRegionNameAsString() + " -- aborting server", e); } } return new HRegion [] {result.getFirst(), result.getSecond()}; }
/** * Run the transaction. * @param server Hosting server instance. Can be null when testing * @param services Used to online/offline regions. * @throws IOException If thrown, transaction failed. * Call {@link #rollback(Server, RegionServerServices)} * @return Regions created * @throws IOException * @see #rollback(Server, RegionServerServices) */ public PairOfSameType<HRegion> execute(final Server server, final RegionServerServices services) throws IOException { useZKForAssignment = server == null ? true : ConfigUtil.useZKForAssignment(server.getConfiguration()); if (useCoordinatedStateManager(server)) { std = ((BaseCoordinatedStateManager) server.getCoordinatedStateManager()) .getSplitTransactionCoordination().getDefaultDetails(); } PairOfSameType<HRegion> regions = createDaughters(server, services); if (this.parent.getCoprocessorHost() != null) { this.parent.getCoprocessorHost().preSplitAfterPONR(); } return stepsAfterPONR(server, services, regions); }
public PairOfSameType<HRegion> stepsAfterPONR(final Server server, final RegionServerServices services, PairOfSameType<HRegion> regions) throws IOException { openDaughters(server, services, regions.getFirst(), regions.getSecond()); if (useCoordinatedStateManager(server)) { ((BaseCoordinatedStateManager) server.getCoordinatedStateManager()) .getSplitTransactionCoordination().completeSplitTransaction(services, regions.getFirst(), regions.getSecond(), std, parent); } journal.add(new JournalEntry(JournalEntryType.BEFORE_POST_SPLIT_HOOK)); // Coprocessor callback if (parent.getCoprocessorHost() != null) { parent.getCoprocessorHost().postSplit(regions.getFirst(), regions.getSecond()); } journal.add(new JournalEntry(JournalEntryType.AFTER_POST_SPLIT_HOOK)); return regions; }
/** * @param parent * Region to split. * @param midkey * Key to split around. * @return The Regions we created. * @throws IOException */ HRegion[] splitRegion(final HRegion parent, final byte[] midkey) throws IOException { PairOfSameType<HRegion> result = null; SplitTransaction st = new SplitTransaction(parent, midkey); // If prepare does not return true, for some reason -- logged inside in // the prepare call -- we are not ready to split just now. Just return. if (!st.prepare()) return null; try { result = st.execute(null, null); } catch (IOException ioe) { try { LOG.info("Running rollback of failed split of " + parent.getRegionNameAsString() + "; " + ioe.getMessage()); st.rollback(null, null); LOG.info("Successful rollback of failed split of " + parent.getRegionNameAsString()); return null; } catch (RuntimeException e) { // If failed rollback, kill this server to avoid having a hole in table. LOG.info("Failed rollback of failed split of " + parent.getRegionNameAsString() + " -- aborting server", e); } } return new HRegion[] { result.getFirst(), result.getSecond() }; }
private List<HRegionLocation> splitRegion(final RegionInfo r) throws IOException, InterruptedException { List<HRegionLocation> locations = new ArrayList<>(); // Split this table in two. Admin admin = TEST_UTIL.getAdmin(); Connection connection = TEST_UTIL.getConnection(); admin.splitRegion(r.getEncodedNameAsBytes()); admin.close(); PairOfSameType<RegionInfo> regions = waitOnDaughters(r); if (regions != null) { try (RegionLocator rl = connection.getRegionLocator(r.getTable())) { locations.add(rl.getRegionLocation(regions.getFirst().getEncodedNameAsBytes())); locations.add(rl.getRegionLocation(regions.getSecond().getEncodedNameAsBytes())); } return locations; } return locations; }
/** * If daughters no longer hold reference to the parents, delete the parent. * @param parent HRegionInfo of split offlined parent * @param rowContent Content of <code>parent</code> row in * <code>metaRegionName</code> * @return True if we removed <code>parent</code> from meta table and from * the filesystem. * @throws IOException */ boolean cleanParent(final HRegionInfo parent, Result rowContent) throws IOException { boolean result = false; // Check whether it is a merged region and not clean reference // No necessary to check MERGEB_QUALIFIER because these two qualifiers will // be inserted/deleted together if (rowContent.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER) != null) { // wait cleaning merge region first return result; } // Run checks on each daughter split. PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(rowContent); Pair<Boolean, Boolean> a = checkDaughterInFs(parent, daughters.getFirst()); Pair<Boolean, Boolean> b = checkDaughterInFs(parent, daughters.getSecond()); if (hasNoReferences(a) && hasNoReferences(b)) { LOG.debug("Deleting region " + parent.getRegionNameAsString() + " because daughter splits no longer hold references"); FileSystem fs = this.services.getMasterFileSystem().getFileSystem(); if (LOG.isTraceEnabled()) LOG.trace("Archiving parent region: " + parent); HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, parent); MetaTableAccessor.deleteRegion(this.connection, parent); result = true; } return result; }
public PairOfSameType<Region> execute(final Server server, final RegionServerServices services) throws IOException { if (User.isHBaseSecurityEnabled(parent.getBaseConf())) { LOG.warn("Should use execute(Server, RegionServerServices, User)"); } return execute(server, services, null); }
/** * Run the transaction. * @param server Hosting server instance. Can be null when testing * @param services Used to online/offline regions. * @throws IOException If thrown, transaction failed. * Call {@link #rollback(Server, RegionServerServices)} * @return Regions created * @throws IOException * @see #rollback(Server, RegionServerServices) */ @Override public PairOfSameType<Region> execute(final Server server, final RegionServerServices services, User user) throws IOException { this.server = server; this.rsServices = services; useZKForAssignment = server == null ? true : ConfigUtil.useZKForAssignment(server.getConfiguration()); if (useCoordinatedStateManager(server)) { std = ((BaseCoordinatedStateManager) server.getCoordinatedStateManager()) .getSplitTransactionCoordination().getDefaultDetails(); } PairOfSameType<Region> regions = createDaughters(server, services, user); if (this.parent.getCoprocessorHost() != null) { if (user == null) { parent.getCoprocessorHost().preSplitAfterPONR(); } else { try { user.getUGI().doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { parent.getCoprocessorHost().preSplitAfterPONR(); return null; } }); } catch (InterruptedException ie) { InterruptedIOException iioe = new InterruptedIOException(); iioe.initCause(ie); throw iioe; } } } regions = stepsAfterPONR(server, services, regions, user); transition(SplitTransactionPhase.COMPLETED); return regions; }
public PairOfSameType<Region> stepsAfterPONR(final Server server, final RegionServerServices services, final PairOfSameType<Region> regions, User user) throws IOException { openDaughters(server, services, regions.getFirst(), regions.getSecond()); if (useCoordinatedStateManager(server)) { ((BaseCoordinatedStateManager) server.getCoordinatedStateManager()) .getSplitTransactionCoordination().completeSplitTransaction(services, regions.getFirst(), regions.getSecond(), std, parent); } transition(SplitTransactionPhase.BEFORE_POST_SPLIT_HOOK); // Coprocessor callback if (parent.getCoprocessorHost() != null) { if (user == null) { this.parent.getCoprocessorHost().postSplit(regions.getFirst(), regions.getSecond()); } else { try { user.getUGI().doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { parent.getCoprocessorHost().postSplit(regions.getFirst(), regions.getSecond()); return null; } }); } catch (InterruptedException ie) { InterruptedIOException iioe = new InterruptedIOException(); iioe.initCause(ie); throw iioe; } } } transition(SplitTransactionPhase.AFTER_POST_SPLIT_HOOK); return regions; }
private Region [] split(final Region r, final byte [] splitRow) throws IOException { Region[] regions = new Region[2]; SplitTransaction st = new SplitTransactionFactory(TEST_UTIL.getConfiguration()) .create(r, splitRow); int i = 0; if (!st.prepare()) { // test fails. assertTrue(false); } try { Server mockServer = Mockito.mock(Server.class); when(mockServer.getConfiguration()).thenReturn(TEST_UTIL.getConfiguration()); PairOfSameType<Region> daughters = st.execute(mockServer, null); for (Region each_daughter: daughters) { regions[i] = each_daughter; i++; } } catch (IOException ioe) { LOG.info("Split transaction of " + r.getRegionInfo().getRegionNameAsString() + " failed:" + ioe.getMessage()); assertTrue(false); } catch (RuntimeException e) { LOG.info("Failed rollback of failed split of " + r.getRegionInfo().getRegionNameAsString() + e.getMessage()); } assertTrue(i == 2); return regions; }
/** * @param parent * Region to split. * @param midkey * Key to split around. * @return The Regions we created. * @throws IOException */ HRegion[] splitRegion(final HRegion parent, final byte[] midkey) throws IOException { PairOfSameType<Region> result = null; SplitTransactionImpl st = new SplitTransactionImpl(parent, midkey); // If prepare does not return true, for some reason -- logged inside in // the prepare call -- we are not ready to split just now. Just return. if (!st.prepare()) { parent.clearSplit(); return null; } try { result = st.execute(null, null); } catch (IOException ioe) { try { LOG.info("Running rollback of failed split of " + parent.getRegionInfo().getRegionNameAsString() + "; " + ioe.getMessage()); st.rollback(null, null); LOG.info("Successful rollback of failed split of " + parent.getRegionInfo().getRegionNameAsString()); return null; } catch (RuntimeException e) { // If failed rollback, kill this server to avoid having a hole in table. LOG.info("Failed rollback of failed split of " + parent.getRegionInfo().getRegionNameAsString() + " -- aborting server", e); } } finally { parent.clearSplit(); } return new HRegion[] { (HRegion)result.getFirst(), (HRegion)result.getSecond() }; }
private PairOfSameType<HRegionInfo> mergeRegionsAndVerifyRegionNum( HMaster master, TableName tablename, int regionAnum, int regionBnum, int expectedRegionNum) throws Exception { PairOfSameType<HRegionInfo> mergedRegions = requestMergeRegion(master, tablename, regionAnum, regionBnum); waitAndVerifyRegionNum(master, tablename, expectedRegionNum); return mergedRegions; }
private PairOfSameType<HRegionInfo> requestMergeRegion( HMaster master, TableName tablename, int regionAnum, int regionBnum) throws Exception { List<Pair<HRegionInfo, ServerName>> tableRegions = MetaTableAccessor .getTableRegionsAndLocations(master.getZooKeeper(), master.getConnection(), tablename); HRegionInfo regionA = tableRegions.get(regionAnum).getFirst(); HRegionInfo regionB = tableRegions.get(regionBnum).getFirst(); TEST_UTIL.getHBaseAdmin().mergeRegions( regionA.getEncodedNameAsBytes(), regionB.getEncodedNameAsBytes(), false); return new PairOfSameType<HRegionInfo>(regionA, regionB); }
/** * Returns the daughter regions by reading the corresponding columns of the catalog table * Result. * @param data a Result object from the catalog table scan * @return a pair of HRegionInfo or PairOfSameType(null, null) if the region is not a split * parent */ public static PairOfSameType<HRegionInfo> getDaughterRegions(Result data) throws IOException { HRegionInfo splitA = Writables.getHRegionInfoOrNull(data.getValue(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER)); HRegionInfo splitB = Writables.getHRegionInfoOrNull(data.getValue(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER)); return new PairOfSameType<HRegionInfo>(splitA, splitB); }
/** * Returns the daughter regions by reading from the corresponding columns of the .META. table * Result. If the region is not a split parent region, it returns PairOfSameType(null, null). */ public static PairOfSameType<HRegionInfo> getDaughterRegions(Result data) throws IOException { HRegionInfo splitA = Writables.getHRegionInfoOrNull( data.getValue(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER)); HRegionInfo splitB = Writables.getHRegionInfoOrNull( data.getValue(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER)); return new PairOfSameType<HRegionInfo>(splitA, splitB); }
private HRegion [] split(final HRegion r, final byte [] splitRow) throws IOException { HRegion[] regions = new HRegion[2]; SplitTransaction st = new SplitTransaction(r, splitRow); int i = 0; if (!st.prepare()) { // test fails. assertTrue(false); } try { Server mockServer = Mockito.mock(Server.class); when(mockServer.getConfiguration()).thenReturn( TEST_UTIL.getConfiguration()); PairOfSameType<HRegion> daughters = st.execute(mockServer, null); for (HRegion each_daughter: daughters) { regions[i] = each_daughter; i++; } } catch (IOException ioe) { LOG.info("Split transaction of " + r.getRegionNameAsString() + " failed:" + ioe.getMessage()); assertTrue(false); } catch (RuntimeException e) { LOG.info("Failed rollback of failed split of " + r.getRegionNameAsString() + e.getMessage()); } assertTrue(i == 2); return regions; }
/** * @param parent * Region to split. * @param midkey * Key to split around. * @return The Regions we created. * @throws IOException */ HRegion[] splitRegion(final HRegion parent, final byte[] midkey) throws IOException { PairOfSameType<HRegion> result = null; SplitTransaction st = new SplitTransaction(parent, midkey); // If prepare does not return true, for some reason -- logged inside in // the prepare call -- we are not ready to split just now. Just return. if (!st.prepare()) { parent.clearSplit(); return null; } try { result = st.execute(null, null); } catch (IOException ioe) { try { LOG.info("Running rollback of failed split of " + parent.getRegionNameAsString() + "; " + ioe.getMessage()); st.rollback(null, null); LOG.info("Successful rollback of failed split of " + parent.getRegionNameAsString()); return null; } catch (RuntimeException e) { // If failed rollback, kill this server to avoid having a hole in table. LOG.info("Failed rollback of failed split of " + parent.getRegionNameAsString() + " -- aborting server", e); } } finally { parent.clearSplit(); } return new HRegion[] { result.getFirst(), result.getSecond() }; }
/** * If daughters no longer hold reference to the parents, delete the parent. * @param parent HRegionInfo of split offlined parent * @param rowContent Content of <code>parent</code> row in * <code>metaRegionName</code> * @return True if we removed <code>parent</code> from meta table and from * the filesystem. * @throws IOException */ boolean cleanParent(final HRegionInfo parent, Result rowContent) throws IOException { boolean result = false; // Check whether it is a merged region and not clean reference // No necessary to check MERGEB_QUALIFIER because these two qualifiers will // be inserted/deleted together if (rowContent.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER) != null) { // wait cleaning merge region first return result; } // Run checks on each daughter split. PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(rowContent); Pair<Boolean, Boolean> a = checkDaughterInFs(parent, daughters.getFirst()); Pair<Boolean, Boolean> b = checkDaughterInFs(parent, daughters.getSecond()); if (hasNoReferences(a) && hasNoReferences(b)) { LOG.debug("Deleting region " + parent.getRegionNameAsString() + " because daughter splits no longer hold references"); FileSystem fs = this.services.getMasterFileSystem().getFileSystem(); if (LOG.isTraceEnabled()) LOG.trace("Archiving parent region: " + parent); HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, parent); MetaEditor.deleteRegion(this.server.getCatalogTracker(), parent); result = true; } return result; }
public PairOfSameType<HRegion> stepsAfterPONR(final Server server, final RegionServerServices services, PairOfSameType<HRegion> regions) throws IOException { openDaughters(server, services, regions.getFirst(), regions.getSecond()); transitionZKNode(server, services, regions.getFirst(), regions.getSecond()); return regions; }
private PairOfSameType<HRegionInfo> requestMergeRegion( HMaster master, TableName tablename, int regionAnum, int regionBnum) throws Exception { List<Pair<HRegionInfo, ServerName>> tableRegions = MetaReader .getTableRegionsAndLocations(master.getCatalogTracker(), tablename); HRegionInfo regionA = tableRegions.get(regionAnum).getFirst(); HRegionInfo regionB = tableRegions.get(regionBnum).getFirst(); TEST_UTIL.getHBaseAdmin().mergeRegions( regionA.getEncodedNameAsBytes(), regionB.getEncodedNameAsBytes(), false); return new PairOfSameType<HRegionInfo>(regionA, regionB); }
@Override public void preSplitAfterPONR(ObserverContext<RegionCoprocessorEnvironment> e) throws IOException { RegionCoprocessorEnvironment environment = e.getEnvironment(); HRegionServer rs = (HRegionServer) environment.getRegionServerServices(); HRegion region = environment.getRegion(); String userTableName = region.getTableDesc().getNameAsString(); String indexTableName = IndexUtils.getIndexTableName(userTableName); if (IndexUtils.isIndexTable(userTableName)) { return; } LOG.trace("Entering postSplit for the table " + userTableName + " for the region " + region.getRegionInfo()); SplitTransaction splitTransaction = null; if (region.getTableDesc().getValue(Constants.INDEX_SPEC_KEY) != null) { try { SplitInfo splitInfo = splitThreadLocal.get(); if (splitInfo == null) return; splitTransaction = splitInfo.getSplitTransaction(); PairOfSameType<HRegion> daughters = splitInfo.getDaughters(); if (splitTransaction != null && daughters != null) { splitTransaction.stepsAfterPONR(rs, rs, daughters); LOG.info("Daughter regions are opened and split transaction finished for zknodes for index table " + indexTableName + " for the region " + region.getRegionInfo()); } } catch (Exception ex) { String msg = "Splitting of index region has failed in stepsAfterPONR stage so aborting the server"; LOG.error(msg, ex); rs.abort(msg); } } }
/** * If daughters no longer hold reference to the parents, delete the parent. * @param parent RegionInfo of split offlined parent * @param rowContent Content of <code>parent</code> row in * <code>metaRegionName</code> * @return True if we removed <code>parent</code> from meta table and from * the filesystem. * @throws IOException */ boolean cleanParent(final RegionInfo parent, Result rowContent) throws IOException { // Check whether it is a merged region and not clean reference // No necessary to check MERGEB_QUALIFIER because these two qualifiers will // be inserted/deleted together if (rowContent.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER) != null) { // wait cleaning merge region first return false; } // Run checks on each daughter split. PairOfSameType<RegionInfo> daughters = MetaTableAccessor.getDaughterRegions(rowContent); Pair<Boolean, Boolean> a = checkDaughterInFs(parent, daughters.getFirst()); Pair<Boolean, Boolean> b = checkDaughterInFs(parent, daughters.getSecond()); if (hasNoReferences(a) && hasNoReferences(b)) { String daughterA = daughters.getFirst() != null? daughters.getFirst().getShortNameToLog(): "null"; String daughterB = daughters.getSecond() != null? daughters.getSecond().getShortNameToLog(): "null"; LOG.debug("Deleting region " + parent.getShortNameToLog() + " because daughters -- " + daughterA + ", " + daughterB + " -- no longer hold references"); ProcedureExecutor<MasterProcedureEnv> pe = this.services.getMasterProcedureExecutor(); pe.submitProcedure(new GCRegionProcedure(pe.getEnvironment(), parent)); // Remove from in-memory states this.services.getAssignmentManager().getRegionStates().deleteRegion(parent); this.services.getServerManager().removeRegion(parent); return true; } return false; }
private PairOfSameType<RegionInfo> waitOnDaughters(final RegionInfo r) throws IOException { long start = System.currentTimeMillis(); PairOfSameType<RegionInfo> pair = null; try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()); Table metaTable = conn.getTable(TableName.META_TABLE_NAME)) { Result result = null; RegionInfo region = null; while ((System.currentTimeMillis() - start) < 60000) { result = metaTable.get(new Get(r.getRegionName())); if (result == null) { break; } region = MetaTableAccessor.getRegionInfo(result); if (region.isSplitParent()) { LOG.debug(region.toString() + " IS a parent!"); pair = MetaTableAccessor.getDaughterRegions(result); break; } Threads.sleep(100); } if (pair.getFirst() == null || pair.getSecond() == null) { throw new IOException("Failed to get daughters, for parent region: " + r); } return pair; } }