Java 类org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv 实例源码

private void startProcedureExecutor() throws IOException {
  final MasterProcedureEnv procEnv = new MasterProcedureEnv(this);
  final Path logDir = new Path(fileSystemManager.getRootDir(),
      MasterProcedureConstants.MASTER_PROCEDURE_LOGDIR);

  procedureStore = new WALProcedureStore(conf, fileSystemManager.getFileSystem(), logDir,
      new MasterProcedureEnv.WALStoreLeaseRecovery(this));
  procedureStore.registerListener(new MasterProcedureEnv.MasterProcedureStoreListener(this));
  procedureExecutor = new ProcedureExecutor(conf, procEnv, procedureStore,
      procEnv.getProcedureQueue());

  final int numThreads = conf.getInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS,
      Math.max(Runtime.getRuntime().availableProcessors(),
        MasterProcedureConstants.DEFAULT_MIN_MASTER_PROCEDURE_THREADS));
  final boolean abortOnCorruption = conf.getBoolean(
      MasterProcedureConstants.EXECUTOR_ABORT_ON_CORRUPTION,
      MasterProcedureConstants.DEFAULT_EXECUTOR_ABORT_ON_CORRUPTION);
  procedureStore.start(numThreads);
  procedureExecutor.start(numThreads, abortOnCorruption);
}

@Test
public void testAbortProcedure() throws Exception {
  final TableName tableName = TableName.valueOf("testAbortProcedure");
  final ProcedureExecutor<MasterProcedureEnv> procExec =
      TEST_UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
  Procedure proc = new TestTableDDLProcedure(procExec.getEnvironment(), tableName);
  proc.setOwner(USER_OWNER.getShortName());
  final long procId = procExec.submitProcedure(proc);

  AccessTestAction abortProcedureAction = new AccessTestAction() {
    @Override
    public Object run() throws Exception {
      ACCESS_CONTROLLER
      .preAbortProcedure(ObserverContext.createAndPrepare(CP_ENV, null), procExec, procId);
     return null;
    }
  };

  verifyAllowed(abortProcedureAction, SUPERUSER, USER_ADMIN, USER_GROUP_ADMIN);
  verifyAllowed(abortProcedureAction, USER_OWNER);
  verifyDenied(
    abortProcedureAction, USER_RW, USER_RO, USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE);
}

@Test
public void testGetProcedures() throws Exception {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  final ProcedureExecutor<MasterProcedureEnv> procExec =
      TEST_UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
  Procedure proc = new TestTableDDLProcedure(procExec.getEnvironment(), tableName);
  proc.setOwner(USER_OWNER);
  procExec.submitProcedure(proc);
  final List<Procedure<?>> procList = procExec.getProcedures();

  AccessTestAction getProceduresAction = new AccessTestAction() {
    @Override
    public Object run() throws Exception {
      ACCESS_CONTROLLER
      .postGetProcedures(ObserverContextImpl.createAndPrepare(CP_ENV));
     return null;
    }
  };

  verifyAllowed(getProceduresAction, SUPERUSER, USER_ADMIN, USER_GROUP_ADMIN);
  verifyAllowed(getProceduresAction, USER_OWNER);
  verifyIfNull(
    getProceduresAction, USER_RW, USER_RO, USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE);
}

@Override
protected synchronized Procedure<MasterProcedureEnv>[] execute(MasterProcedureEnv env)
    throws ProcedureYieldException, ProcedureSuspendedException, InterruptedException {
  if (dispatched) {
    if (succ) {
      return null;
    }
    // retry
    dispatched = false;
  }
  if (!env.getRemoteDispatcher().addOperationToNode(targetServer, this)) {
    LOG.info("Can not add remote operation for refreshing peer {} for {} to {}, " +
      "this usually because the server is already dead, " +
      "give up and mark the procedure as complete", peerId, type, targetServer);
    return null;
  }
  dispatched = true;
  event = new ProcedureEvent<>(this);
  event.suspendIfNotReady(this);
  throw new ProcedureSuspendedException();
}

@Override
protected Flow executeFromState(final MasterProcedureEnv env, final MoveRegionState state)
    throws InterruptedException {
  if (LOG.isTraceEnabled()) {
    LOG.trace(this + " execute state=" + state);
  }
  switch (state) {
    case MOVE_REGION_UNASSIGN:
      addChildProcedure(new UnassignProcedure(plan.getRegionInfo(), plan.getSource(),
          plan.getDestination(), true));
      setNextState(MoveRegionState.MOVE_REGION_ASSIGN);
      break;
    case MOVE_REGION_ASSIGN:
      AssignProcedure assignProcedure = plan.getDestination() == null ?
          new AssignProcedure(plan.getRegionInfo()):
          new AssignProcedure(plan.getRegionInfo(), plan.getDestination());
      addChildProcedure(assignProcedure);
      return Flow.NO_MORE_STATE;
    default:
      throw new UnsupportedOperationException("unhandled state=" + state);
  }
  return Flow.HAS_MORE_STATE;
}

@Test
public void testRollbackAndDoubleExecution() throws Exception {
  final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecution");
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

  List<RegionInfo> tableRegions = createTable(tableName);

  ProcedureTestingUtility.waitNoProcedureRunning(procExec);
  ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);

  RegionInfo[] regionsToMerge = new RegionInfo[2];
  regionsToMerge[0] = tableRegions.get(0);
  regionsToMerge[1] = tableRegions.get(1);

  long procId = procExec.submitProcedure(
    new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge, true));

  // Failing before MERGE_TABLE_REGIONS_CREATE_MERGED_REGION we should trigger the rollback
  // NOTE: the 5 (number before MERGE_TABLE_REGIONS_CREATE_MERGED_REGION step) is
  // hardcoded, so you have to look at this test at least once when you add a new step.
  int numberOfSteps = 5;
  MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, numberOfSteps,
      true);
}

private void startProcedureExecutor() throws IOException {
  final MasterProcedureEnv procEnv = new MasterProcedureEnv(this);

  procedureStore = new WALProcedureStore(conf,
      new MasterProcedureEnv.WALStoreLeaseRecovery(this));
  procedureStore.registerListener(new MasterProcedureEnv.MasterProcedureStoreListener(this));
  MasterProcedureScheduler procedureScheduler = procEnv.getProcedureScheduler();
  procedureExecutor = new ProcedureExecutor<>(conf, procEnv, procedureStore, procedureScheduler);
  configurationManager.registerObserver(procEnv);

  int cpus = Runtime.getRuntime().availableProcessors();
  final int numThreads = conf.getInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS,
      Math.max((cpus > 0? cpus/4: 0),
          MasterProcedureConstants.DEFAULT_MIN_MASTER_PROCEDURE_THREADS));
  final boolean abortOnCorruption = conf.getBoolean(
      MasterProcedureConstants.EXECUTOR_ABORT_ON_CORRUPTION,
      MasterProcedureConstants.DEFAULT_EXECUTOR_ABORT_ON_CORRUPTION);
  procedureStore.start(numThreads);
  procedureExecutor.start(numThreads, abortOnCorruption);
  procEnv.getRemoteDispatcher().start();
}

@Override
public void remoteCallFailed(final MasterProcedureEnv env,
    final ServerName serverName, final IOException exception) {
  final RegionStateNode regionNode = getRegionState(env);
  String msg = exception.getMessage() == null? exception.getClass().getSimpleName():
    exception.getMessage();
  LOG.warn("Remote call failed " + this + "; " + regionNode.toShortString() +
    "; exception=" + msg);
  if (remoteCallFailed(env, regionNode, exception)) {
    // NOTE: This call to wakeEvent puts this Procedure back on the scheduler.
    // Thereafter, another Worker can be in here so DO NOT MESS WITH STATE beyond
    // this method. Just get out of this current processing quickly.
    regionNode.getProcedureEvent().wake(env.getProcedureScheduler());
  }
  // else leave the procedure in suspended state; it is waiting on another call to this callback
}

/**
 * Be careful! At the end of this method, the procedure has either succeeded
 * and this procedure has been set into a suspended state OR, we failed and
 * this procedure has been put back on the scheduler ready for another worker
 * to pick it up. In both cases, we need to exit the current Worker processing
 * immediately!
 * @return True if we successfully dispatched the call and false if we failed;
 * if failed, we need to roll back any setup done for the dispatch.
 */
protected boolean addToRemoteDispatcher(final MasterProcedureEnv env,
    final ServerName targetServer) {
  assert targetServer == null || targetServer.equals(getRegionState(env).getRegionLocation()):
    "targetServer=" + targetServer + " getRegionLocation=" +
      getRegionState(env).getRegionLocation(); // TODO

  LOG.info("Dispatch " + this + "; " + getRegionState(env).toShortString());

  // Put this procedure into suspended mode to wait on report of state change
  // from remote regionserver. Means Procedure associated ProcedureEvent is marked not 'ready'.
  getRegionState(env).getProcedureEvent().suspend();

  // Tricky because the below call to addOperationToNode can fail. If it fails, we need to
  // backtrack on stuff like the 'suspend' done above -- tricky as the 'wake' requests us -- and
  // ditto up in the caller; it needs to undo state changes. Inside in remoteCallFailed, it does
  // wake to undo the above suspend.
  if (!env.getRemoteDispatcher().addOperationToNode(targetServer, this)) {
    remoteCallFailed(env, targetServer,
        new FailedRemoteDispatchException(this + " to " + targetServer));
    return false;
  }
  return true;
}

/**
 * An ugly utility to be removed when refactor TableNamespaceManager.
 * @throws TimeoutIOException
 */
private static void block(final MasterServices services, final long procId)
throws TimeoutIOException {
  int timeoutInMillis = services.getConfiguration().
      getInt(ClusterSchema.HBASE_MASTER_CLUSTER_SCHEMA_OPERATION_TIMEOUT_KEY,
          ClusterSchema.DEFAULT_HBASE_MASTER_CLUSTER_SCHEMA_OPERATION_TIMEOUT);
  long deadlineTs = EnvironmentEdgeManager.currentTime() + timeoutInMillis;
  ProcedureExecutor<MasterProcedureEnv> procedureExecutor =
      services.getMasterProcedureExecutor();
  while(EnvironmentEdgeManager.currentTime() < deadlineTs) {
    if (procedureExecutor.isFinished(procId)) return;
    // Sleep some
    Threads.sleep(10);
  }
  throw new TimeoutIOException("Procedure pid=" + procId + " is still running");
}

@Override
protected LockState acquireLock(final MasterProcedureEnv env) {
  // Unless we are assigning meta, wait for meta to be available and loaded.
  if (!isMeta() && (env.waitFailoverCleanup(this) ||
      env.getAssignmentManager().waitMetaInitialized(this, getRegionInfo()))) {
    return LockState.LOCK_EVENT_WAIT;
  }

  // TODO: Revisit this and move it to the executor
  if (env.getProcedureScheduler().waitRegion(this, getRegionInfo())) {
    try {
      LOG.debug(LockState.LOCK_EVENT_WAIT + " pid=" + getProcId() + " " +
        env.getProcedureScheduler().dumpLocks());
    } catch (IOException e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }
    return LockState.LOCK_EVENT_WAIT;
  }
  this.lock = true;
  return LockState.LOCK_ACQUIRED;
}

@Override
protected Procedure<MasterProcedureEnv>[] execute(final MasterProcedureEnv env)
throws ProcedureSuspendedException {
  // Local master locks don't store any state, so on recovery, simply finish this procedure
  // immediately.
  if (recoveredMasterLock) return null;
  if (lockAcquireLatch != null) {
    lockAcquireLatch.countDown();
  }
  if (unlock.get() || hasHeartbeatExpired()) {
    locked.set(false);
    LOG.debug((unlock.get()? "UNLOCKED " : "TIMED OUT ") + toString());
    return null;
  }
  synchronized (event) {
    event.suspend();
    event.suspendIfNotReady(this);
    setState(ProcedureProtos.ProcedureState.WAITING_TIMEOUT);
  }
  throw new ProcedureSuspendedException();
}

@Override
protected LockState acquireLock(final MasterProcedureEnv env) {
  if (env.waitInitialized(this)) return LockState.LOCK_EVENT_WAIT;
  if (env.getProcedureScheduler().waitRegions(this, getTableName(),
      mergedRegion, regionsToMerge[0], regionsToMerge[1])) {
    try {
      LOG.debug(LockState.LOCK_EVENT_WAIT + " " + env.getProcedureScheduler().dumpLocks());
    } catch (IOException e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }
    return LockState.LOCK_EVENT_WAIT;
  }
  this.lock = true;
  return LockState.LOCK_ACQUIRED;
}

@Override
public void preAbortProcedure(
    ObserverContext<MasterCoprocessorEnvironment> ctx,
    final ProcedureExecutor<MasterProcedureEnv> procEnv,
    final long procId) throws IOException {
  if (!procEnv.isProcedureOwner(procId, getActiveUser())) {
    // If the user is not the procedure owner, then we should further probe whether
    // he can abort the procedure.
    requirePermission("abortProcedure", Action.ADMIN);
  }
}

public boolean preAbortProcedure(
    final ProcedureExecutor<MasterProcedureEnv> procEnv,
    final long procId) throws IOException {
  return execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() {
    @Override
    public void call(MasterObserver oserver, ObserverContext<MasterCoprocessorEnvironment> ctx)
        throws IOException {
      oserver.preAbortProcedure(ctx, procEnv, procId);
    }
  });
}

@Override
protected Procedure[] execute(MasterProcedureEnv env) throws ProcedureYieldException,
    InterruptedException {
  // Not letting the procedure to complete until timed out
  setState(ProcedureState.WAITING_TIMEOUT);
  return null;
}

@Test
public void testListProcedures() throws Exception {
  final TableName tableName = TableName.valueOf("testAbortProcedure");
  final ProcedureExecutor<MasterProcedureEnv> procExec =
      TEST_UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
  Procedure proc = new TestTableDDLProcedure(procExec.getEnvironment(), tableName);
  proc.setOwner(USER_OWNER.getShortName());
  final long procId = procExec.submitProcedure(proc);
  final List<ProcedureInfo> procInfoList = procExec.listProcedures();

  AccessTestAction listProceduresAction = new AccessTestAction() {
    @Override
    public Object run() throws Exception {
      List<ProcedureInfo> procInfoListClone = new ArrayList<ProcedureInfo>(procInfoList.size());
      for(ProcedureInfo pi : procInfoList) {
        procInfoListClone.add(pi.clone());
      }
      ACCESS_CONTROLLER
      .postListProcedures(ObserverContext.createAndPrepare(CP_ENV, null), procInfoListClone);
     return null;
    }
  };

  verifyAllowed(listProceduresAction, SUPERUSER, USER_ADMIN, USER_GROUP_ADMIN);
  verifyAllowed(listProceduresAction, USER_OWNER);
  verifyIfNull(
    listProceduresAction, USER_RW, USER_RO, USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE);
}

@Override
public void preAbortProcedure(
    ObserverContext<MasterCoprocessorEnvironment> ctx,
    final ProcedureExecutor<MasterProcedureEnv> procEnv,
    final long procId) throws IOException {
  preAbortProcedureCalled = true;
}

@Override
public void remoteCallFailed(final MasterProcedureEnv env,
    final ServerName serverName, final IOException exception) {
  // Just skip this remoteCallFailed. Its too hard to mock. Assert it is called though.
  // Happens after the code we are testing has been called.
  this.remoteCallFailedWasCalled.set(true);
}

@Override
protected void rollbackState(MasterProcedureEnv env, PeerModificationState state)
    throws IOException, InterruptedException {
  if (state == PeerModificationState.PRE_PEER_MODIFICATION) {
    // actually the peer related operations has no rollback, but if we haven't done any
    // modifications on the peer storage yet, we can just return.
    return;
  }
  throw new UnsupportedOperationException();
}

@Override
protected void postPeerModification(MasterProcedureEnv env) throws IOException {
  LOG.info("Successfully enabled peer {}", peerId);
  MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
  if (cpHost != null) {
    cpHost.postEnableReplicationPeer(peerId);
  }
}

/**
 * Post split region actions
 * @param env MasterProcedureEnv
 **/
private void postSplitRegion(final MasterProcedureEnv env) throws IOException {
  final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
  if (cpHost != null) {
    cpHost.postCompletedSplitRegionAction(daughter_1_RI, daughter_2_RI, getUser());
  }
}

/**
 * Action after rollback a split table region action.
 * @param env MasterProcedureEnv
 * @throws IOException
 */
private void postRollBackSplitRegion(final MasterProcedureEnv env) throws IOException {
  final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
  if (cpHost != null) {
    cpHost.postRollBackSplitRegionAction(getUser());
  }
}

@Override
protected void postPeerModification(MasterProcedureEnv env)
    throws IOException, ReplicationException {
  env.getReplicationPeerManager().removeAllQueuesAndHFileRefs(peerId);
  LOG.info("Successfully removed peer {}", peerId);
  MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
  if (cpHost != null) {
    cpHost.postRemoveReplicationPeer(peerId);
  }
}

@Override
protected void prePeerModification(MasterProcedureEnv env)
    throws IOException, ReplicationException {
  MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
  if (cpHost != null) {
    cpHost.preAddReplicationPeer(peerId, peerConfig);
  }
  env.getReplicationPeerManager().preAddPeer(peerId, peerConfig);
}

@Override
protected boolean addToRemoteDispatcher(MasterProcedureEnv env, ServerName targetServer) {
  // So, mock the ServerCrashProcedure nulling out the targetServer AFTER updateTransition
  // has been called and BEFORE updateTransition gets to here.
  // We used to throw a NullPointerException. Now we just say the assign failed so it will
  // be rescheduled.
  boolean b = super.addToRemoteDispatcher(env, null);
  assertFalse(b);
  // Assert we were actually called.
  this.addToRemoteDispatcherWasCalled.set(true);
  return b;
}

/**
 * Verify if the restore of the specified table is in progress.
 *
 * @param tableName table under restore
 * @return <tt>true</tt> if there is a restore in progress of the specified table.
 */
private synchronized boolean isRestoringTable(final TableName tableName) {
  Long procId = this.restoreTableToProcIdMap.get(tableName);
  if (procId == null) {
    return false;
  }
  ProcedureExecutor<MasterProcedureEnv> procExec = master.getMasterProcedureExecutor();
  if (procExec.isRunning() && !procExec.isFinished(procId)) {
    return true;
  } else {
    this.restoreTableToProcIdMap.remove(tableName);
    return false;
  }

}

@Override
protected void prePeerModification(MasterProcedureEnv env) throws IOException {
  MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
  if (cpHost != null) {
    cpHost.preUpdateReplicationPeerConfig(peerId, peerConfig);
  }
  env.getReplicationPeerManager().preUpdatePeerConfig(peerId, peerConfig);
}

/**
 * Pre split region actions after the Point-of-No-Return step
 * @param env MasterProcedureEnv
 **/
private void preSplitRegionAfterMETA(final MasterProcedureEnv env)
    throws IOException, InterruptedException {
  final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
  if (cpHost != null) {
    cpHost.preSplitAfterMETAAction(getUser());
  }
}

@Override
protected LockState acquireLock(MasterProcedureEnv env) {
  if (env.getProcedureScheduler().waitPeerExclusiveLock(this, peerId)) {
    return LockState.LOCK_EVENT_WAIT;
  }
  locked = true;
  return LockState.LOCK_ACQUIRED;
}

@Test
public void testSplitTableRegionEmptyDaughter() throws Exception {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

  RegionInfo [] regions = MasterProcedureTestingUtility.createTable(
    procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
  insertData(tableName);
  // Split to two daughters with one of them only has 1 row
  int splitRowNum = startRowNum + rowCount;
  byte[] splitKey = Bytes.toBytes("" + splitRowNum);

  assertTrue("not able to find a splittable region", regions != null);
  assertTrue("not able to find a splittable region", regions.length == 1);

  // collect AM metrics before test
  collectAssignmentManagerMetrics();

  // Split region of the table
  long procId = procExec.submitProcedure(
    new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
  // Wait the completion
  ProcedureTestingUtility.waitProcedure(procExec, procId);
  ProcedureTestingUtility.assertProcNotFailed(procExec, procId);

  // Make sure one daughter has 0 rows.
  List<HRegion> daughters = UTIL.getMiniHBaseCluster().getRegions(tableName);
  assertTrue(daughters.size() == 2);
  assertTrue(UTIL.countRows(tableName) == rowCount);
  assertTrue(UTIL.countRows(daughters.get(0)) == 0 || UTIL.countRows(daughters.get(1)) == 0);

  assertEquals(splitSubmittedCount + 1,
      splitProcMetrics.getSubmittedCounter().getCount());
  assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
}

@Override
protected void reportTransition(final MasterProcedureEnv env, final RegionStateNode regionNode,
    final TransitionCode code, final long seqId) throws UnexpectedStateException {
  switch (code) {
    case CLOSED:
      setTransitionState(RegionTransitionState.REGION_TRANSITION_FINISH);
      break;
    default:
      throw new UnexpectedStateException(String.format(
        "Received report unexpected transition state=%s for region=%s server=%s, expected CLOSED.",
        code, regionNode.getRegionInfo(), regionNode.getRegionLocation()));
  }
}

@Override
public boolean acquireLock(final MasterProcedureEnv env) {
  // We invert return from waitNamespaceExclusiveLock; it returns true if you HAVE TO WAIT
  // to get the lock and false if you don't; i.e. you got the lock.
  return !env.getProcedureScheduler().waitNamespaceExclusiveLock(
      LockProcedure.this, namespace);
}

@Override
protected void periodicExecute(final MasterProcedureEnv env) {
  final AssignmentManager am = env.getAssignmentManager();

  final RegionInTransitionStat ritStat = am.computeRegionInTransitionStat();
  if (ritStat.hasRegionsOverThreshold()) {
    for (RegionState hri: ritStat.getRegionOverThreshold()) {
      am.handleRegionOverStuckWarningThreshold(hri.getRegion());
    }
  }

  // update metrics
  am.updateRegionsInTransitionMetrics(ritStat);
}

@Override
protected boolean abort(final MasterProcedureEnv env) {
  if (isRollbackSupported(transitionState)) {
    aborted.set(true);
    return true;
  }
  return false;
}

@Override
protected Procedure[] execute(MasterProcedureEnv env) throws ProcedureYieldException,
    InterruptedException {
  // Not letting the procedure to complete until timed out
  setState(ProcedureState.WAITING_TIMEOUT);
  return null;
}

public GCMergedRegionsProcedure(final MasterProcedureEnv env,
    final RegionInfo mergedChild,
    final RegionInfo father,
    final RegionInfo mother) {
  super(env);
  this.father = father;
  this.mother = mother;
  this.mergedChild = mergedChild;
}

@Override
protected Flow executeFromState(MasterProcedureEnv env, GCMergedRegionsState state)
throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException {
  if (LOG.isTraceEnabled()) {
    LOG.trace(this + " execute state=" + state);
  }
  try {
    switch (state) {
    case GC_MERGED_REGIONS_PREPARE:
      // Nothing to do to prepare.
      setNextState(GCMergedRegionsState.GC_MERGED_REGIONS_PURGE);
      break;
    case GC_MERGED_REGIONS_PURGE:
      addChildProcedure(createGCRegionProcedures(env));
      setNextState(GCMergedRegionsState.GC_REGION_EDIT_METADATA);
      break;
    case GC_REGION_EDIT_METADATA:
      MetaTableAccessor.deleteMergeQualifiers(env.getMasterServices().getConnection(), mergedChild);
      return Flow.NO_MORE_STATE;
    default:
      throw new UnsupportedOperationException(this + " unhandled state=" + state);
    }
  } catch (IOException ioe) {
    // TODO: This is going to spew log?
    LOG.warn("Error trying to GC merged regions " + this.father.getShortNameToLog() +
        " & " + this.mother.getShortNameToLog() + "; retrying...", ioe);
  }
  return Flow.HAS_MORE_STATE;
}

@Test
public void testRollbackAndDoubleExecution() throws Exception {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

  RegionInfo [] regions = MasterProcedureTestingUtility.createTable(
    procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
  insertData(tableName);
  int splitRowNum = startRowNum + rowCount / 2;
  byte[] splitKey = Bytes.toBytes("" + splitRowNum);

  assertTrue("not able to find a splittable region", regions != null);
  assertTrue("not able to find a splittable region", regions.length == 1);
  ProcedureTestingUtility.waitNoProcedureRunning(procExec);
  ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);

  // collect AM metrics before test
  collectAssignmentManagerMetrics();

  // Split region of the table
  long procId = procExec.submitProcedure(
    new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));

  // Failing before SPLIT_TABLE_REGION_CREATE_DAUGHTER_REGIONS we should trigger the
  // rollback
  // NOTE: the 3 (number before SPLIT_TABLE_REGION_CREATE_DAUGHTER_REGIONS step) is
  // hardcoded, so you have to look at this test at least once when you add a new step.
  int numberOfSteps = 3;
  MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, numberOfSteps,
      true);
  // check that we have only 1 region
  assertEquals(1, UTIL.getHBaseAdmin().getTableRegions(tableName).size());
  List<HRegion> daughters = UTIL.getMiniHBaseCluster().getRegions(tableName);
  assertEquals(1, daughters.size());
  verifyData(daughters.get(0), startRowNum, rowCount,
  Bytes.toBytes(ColumnFamilyName1), Bytes.toBytes(ColumnFamilyName2));

  assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
  assertEquals(splitFailedCount + 1, splitProcMetrics.getFailedCounter().getCount());
}

@Override
protected void finishTransition(final MasterProcedureEnv env, final RegionStateNode regionNode)
    throws IOException {
  env.getAssignmentManager().markRegionAsOpened(regionNode);
  // This success may have been after we failed open a few times. Be sure to cleanup any
  // failed open references. See #incrementAndCheckMaxAttempts and where it is called.
  env.getAssignmentManager().getRegionStates().removeFromFailedOpen(regionNode.getRegionInfo());
}