@Override public void perform() throws Exception { HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); Admin admin = util.getHBaseAdmin(); LOG.info("Performing action: Flush random region of table " + tableName); List<HRegionInfo> regions = admin.getTableRegions(tableName); if (regions == null || regions.isEmpty()) { LOG.info("Table " + tableName + " doesn't have regions to flush"); return; } HRegionInfo region = PolicyBasedChaosMonkey.selectRandomItem( regions.toArray(new HRegionInfo[regions.size()])); LOG.debug("Flushing region " + region.getRegionNameAsString()); try { admin.flushRegion(region.getRegionName()); } catch (Exception ex) { LOG.warn("Flush failed, might be caused by other chaos: " + ex.getMessage()); } if (sleepTime > 0) { Thread.sleep(sleepTime); } }
@Override public void perform() throws Exception { if (sleepTime > 0) { Thread.sleep(sleepTime); } HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); Admin admin = util.getHBaseAdmin(); LOG.info("Performing action: Move random region of table " + tableName); List<HRegionInfo> regions = admin.getTableRegions(tableName); if (regions == null || regions.isEmpty()) { LOG.info("Table " + tableName + " doesn't have regions to move"); return; } HRegionInfo region = PolicyBasedChaosMonkey.selectRandomItem( regions.toArray(new HRegionInfo[regions.size()])); LOG.debug("Unassigning region " + region.getRegionNameAsString()); admin.unassign(region.getRegionName(), false); if (sleepTime > 0) { Thread.sleep(sleepTime); } }
@Override protected void runOneIteration() { Action actionOne = PolicyBasedChaosMonkey.selectRandomItem(actionsOne); Action actionTwo = PolicyBasedChaosMonkey.selectRandomItem(actionsTwo); Future fOne = executor.submit(new ActionRunner(actionOne)); Future fTwo = executor.submit(new ActionRunner(actionTwo)); try { fOne.get(); fTwo.get(); } catch (InterruptedException e) { LOG.warn("Exception occurred during performing action: " + StringUtils.stringifyException(e)); } catch (ExecutionException ex) { LOG.warn("Exception occurred during performing action: " + StringUtils.stringifyException(ex)); } }
@Override public ChaosMonkey build() { loadProperties(); // Destructive actions to mess things around. Action[] actions1 = new Action[] { new RestartActiveMasterAction(restartActiveMasterSleepTime), }; // Action to log more info for debugging Action[] actions2 = new Action[] { new DumpClusterStatusAction() }; return new PolicyBasedChaosMonkey(util, new PeriodicRandomActionPolicy(action1Period, actions1), new PeriodicRandomActionPolicy(action2Period, actions2)); }
@Override public ChaosMonkey build() { // Destructive actions to mess things around. Cannot run batch restart Action[] actions1 = new Action[] { new RestartRandomRsExceptMetaAction(60000), new RestartActiveMasterAction(5000), new RollingBatchRestartRsExceptMetaAction(5000, 1.0f, 2), //only allow 2 servers to be dead new ForceBalancerAction() }; // Action to log more info for debugging Action[] actions2 = new Action[] { new DumpClusterStatusAction() }; return new PolicyBasedChaosMonkey(util, new CompositeSequentialPolicy( new DoActionsOncePolicy(60 * 1000, actions1), new PeriodicRandomActionPolicy(60 * 1000, actions1)), new PeriodicRandomActionPolicy(60 * 1000, actions2)); }
@Override public ChaosMonkey build() { // Destructive actions to mess things around. Cannot run batch restart. Action[] actions1 = new Action[]{ new RestartRandomRsExceptMetaAction(60000), new RestartActiveMasterAction(5000), new RollingBatchRestartRsExceptMetaAction(5000, 1.0f, 2), // only allow 2 servers to be dead. new ForceBalancerAction(), new RestartRandomDataNodeAction(60000), new RestartRandomZKNodeAction(60000) }; // Action to log more info for debugging Action[] actions2 = new Action[]{ new DumpClusterStatusAction() }; return new PolicyBasedChaosMonkey(util, new CompositeSequentialPolicy( new DoActionsOncePolicy(60 * 1000, actions1), new PeriodicRandomActionPolicy(60 * 1000, actions1)), new PeriodicRandomActionPolicy(60 * 1000, actions2)); }
@Override public void perform() throws Exception { HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); HBaseAdmin admin = util.getHBaseAdmin(); LOG.info("Performing action: Flush random region of table " + tableName); List<HRegionInfo> regions = admin.getTableRegions(tableNameBytes); if (regions == null || regions.isEmpty()) { LOG.info("Table " + tableName + " doesn't have regions to flush"); return; } HRegionInfo region = PolicyBasedChaosMonkey.selectRandomItem( regions.toArray(new HRegionInfo[regions.size()])); LOG.debug("Flushing region " + region.getRegionNameAsString()); try { admin.flush(region.getRegionName()); } catch (Exception ex) { LOG.warn("Flush failed, might be caused by other chaos: " + ex.getMessage()); } if (sleepTime > 0) { Thread.sleep(sleepTime); } }
@Override public void perform() throws Exception { HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); HBaseAdmin admin = util.getHBaseAdmin(); LOG.info("Performing action: Split random region of table " + tableName); List<HRegionInfo> regions = admin.getTableRegions(tableNameBytes); if (regions == null || regions.isEmpty()) { LOG.info("Table " + tableName + " doesn't have regions to split"); return; } HRegionInfo region = PolicyBasedChaosMonkey.selectRandomItem( regions.toArray(new HRegionInfo[regions.size()])); LOG.debug("Splitting region " + region.getRegionNameAsString()); try { admin.split(region.getRegionName()); } catch (Exception ex) { LOG.warn("Split failed, might be caused by other chaos: " + ex.getMessage()); } if (sleepTime > 0) { Thread.sleep(sleepTime); } }
@Override public void perform() throws Exception { if (sleepTime > 0) { Thread.sleep(sleepTime); } HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); HBaseAdmin admin = util.getHBaseAdmin(); LOG.info("Performing action: Move random region of table " + tableName); List<HRegionInfo> regions = admin.getTableRegions(tableNameBytes); if (regions == null || regions.isEmpty()) { LOG.info("Table " + tableName + " doesn't have regions to move"); return; } HRegionInfo region = PolicyBasedChaosMonkey.selectRandomItem( regions.toArray(new HRegionInfo[regions.size()])); LOG.debug("Unassigning region " + region.getRegionNameAsString()); admin.unassign(region.getRegionName(), false); if (sleepTime > 0) { Thread.sleep(sleepTime); } }
@Override public void perform() throws Exception { HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); Admin admin = util.getAdmin(); LOG.info("Performing action: Flush random region of table " + tableName); List<HRegionInfo> regions = admin.getTableRegions(tableName); if (regions == null || regions.isEmpty()) { LOG.info("Table " + tableName + " doesn't have regions to flush"); return; } HRegionInfo region = PolicyBasedChaosMonkey.selectRandomItem( regions.toArray(new HRegionInfo[regions.size()])); LOG.debug("Flushing region " + region.getRegionNameAsString()); try { admin.flushRegion(region.getRegionName()); } catch (Exception ex) { LOG.warn("Flush failed, might be caused by other chaos: " + ex.getMessage()); } if (sleepTime > 0) { Thread.sleep(sleepTime); } }
@Override public void perform() throws Exception { if (sleepTime > 0) { Thread.sleep(sleepTime); } HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); Admin admin = util.getAdmin(); LOG.info("Performing action: Move random region of table " + tableName); List<HRegionInfo> regions = admin.getTableRegions(tableName); if (regions == null || regions.isEmpty()) { LOG.info("Table " + tableName + " doesn't have regions to move"); return; } HRegionInfo region = PolicyBasedChaosMonkey.selectRandomItem( regions.toArray(new HRegionInfo[regions.size()])); LOG.debug("Unassigning region " + region.getRegionNameAsString()); admin.unassign(region.getRegionName(), false); if (sleepTime > 0) { Thread.sleep(sleepTime); } }
@Override public void perform() throws Exception { HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); Admin admin = util.getHBaseAdmin(); boolean major = RandomUtils.nextInt(100) < majorRatio; LOG.info("Performing action: Compact random region of table " + tableName + ", major=" + major); List<HRegionInfo> regions = admin.getTableRegions(tableName); if (regions == null || regions.isEmpty()) { LOG.info("Table " + tableName + " doesn't have regions to compact"); return; } HRegionInfo region = PolicyBasedChaosMonkey.selectRandomItem( regions.toArray(new HRegionInfo[regions.size()])); try { if (major) { LOG.debug("Major compacting region " + region.getRegionNameAsString()); admin.majorCompactRegion(region.getRegionName()); } else { LOG.debug("Compacting region " + region.getRegionNameAsString()); admin.compactRegion(region.getRegionName()); } } catch (Exception ex) { LOG.warn("Compaction failed, might be caused by other chaos: " + ex.getMessage()); } if (sleepTime > 0) { Thread.sleep(sleepTime); } }
@Override public void perform() throws Exception { int tries = 10; while (tries-- > 0 && getCurrentServers().length > 1) { ServerName server = PolicyBasedChaosMonkey.selectRandomItem(getCurrentServers()); ServerName metaServer = cluster.getServerHoldingMeta(); if (server != null && !server.equals(metaServer)) { restartRs(server, sleepTime); break; } } }
@Override public void perform() throws Exception { LOG.info("Performing action: Restart random zookeeper node"); ServerName server = PolicyBasedChaosMonkey.selectRandomItem( ZKServerTool.readZKNodes(getConf())); restartZKNode(server, sleepTime); }
@Override public void perform() throws Exception { LOG.info("Performing action: Restart random region server"); ServerName server = PolicyBasedChaosMonkey.selectRandomItem(getCurrentServers()); restartRs(server, sleepTime); }
@Override public void perform() throws Exception { HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); Admin admin = util.getHBaseAdmin(); LOG.info("Performing action: Split random region of table " + tableName); List<HRegionInfo> regions = admin.getTableRegions(tableName); if (regions == null || regions.isEmpty()) { LOG.info("Table " + tableName + " doesn't have regions to split"); return; } // Don't try the split if we're stopping if (context.isStopping()) { return; } HRegionInfo region = PolicyBasedChaosMonkey.selectRandomItem( regions.toArray(new HRegionInfo[regions.size()])); LOG.debug("Splitting region " + region.getRegionNameAsString()); try { admin.splitRegion(region.getRegionName()); } catch (Exception ex) { LOG.warn("Split failed, might be caused by other chaos: " + ex.getMessage()); } if (sleepTime > 0) { Thread.sleep(sleepTime); } }
@Override protected void runOneIteration() { Action action = PolicyBasedChaosMonkey.selectWeightedRandomItem(actions); try { action.perform(); } catch (Exception ex) { LOG.warn("Exception occured during performing action: " + StringUtils.stringifyException(ex)); } }
@Override public ChaosMonkey build() { Action[] actions1 = new Action[] { new CompactTableAction(tableName, MonkeyConstants.DEFAULT_PERIODIC_ACTION1_PERIOD), new CompactRandomRegionOfTableAction(tableName, MonkeyConstants.DEFAULT_COMPACT_RANDOM_REGION_RATIO), new FlushTableAction(tableName), new FlushRandomRegionOfTableAction(tableName), new MoveRandomRegionOfTableAction(tableName) }; Action[] actions2 = new Action[] { new SplitRandomRegionOfTableAction(tableName), new MergeRandomAdjacentRegionsOfTableAction(tableName), new SnapshotTableAction(tableName), new AddColumnAction(tableName), new RemoveColumnAction(tableName, columnFamilies), new ChangeEncodingAction(tableName), new ChangeCompressionAction(tableName), new ChangeBloomFilterAction(tableName), new ChangeVersionsAction(tableName) }; Action[] actions3 = new Action[] { new MoveRegionsOfTableAction(MonkeyConstants.DEFAULT_MOVE_REGIONS_SLEEP_TIME, MonkeyConstants.DEFAULT_MOVE_REGIONS_MAX_TIME, tableName), new MoveRandomRegionOfTableAction(MonkeyConstants.DEFAULT_RESTART_ACTIVE_MASTER_SLEEP_TIME, tableName), }; Action[] actions4 = new Action[] { new DumpClusterStatusAction() }; return new PolicyBasedChaosMonkey(util, new TwoConcurrentActionPolicy(MonkeyConstants.DEFAULT_PERIODIC_ACTION1_PERIOD, actions1, actions2), new PeriodicRandomActionPolicy(MonkeyConstants.DEFAULT_PERIODIC_ACTION2_PERIOD,actions3), new PeriodicRandomActionPolicy(MonkeyConstants.DEFAULT_PERIODIC_ACTION4_PERIOD,actions4)); }
@Override public ChaosMonkey build() { loadProperties(); Policy chaosPolicy = new PeriodicRandomActionPolicy(chaosEveryMilliSec, new UnbalanceKillAndRebalanceAction(waitForUnbalanceMilliSec, waitForKillMilliSec, waitAfterBalanceMilliSec)); return new PolicyBasedChaosMonkey(util, chaosPolicy); }
@Override public void setUpMonkey() throws Exception { Policy p = new PeriodicRandomActionPolicy(sleepTime, new RestartRandomRsExceptMetaAction(sleepTime), new MoveRandomRegionOfTableAction(tableName)); this.monkey = new PolicyBasedChaosMonkey(util, p); // don't start monkey right away }
@Override public void perform() throws Exception { HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); HBaseAdmin admin = util.getHBaseAdmin(); boolean major = RandomUtils.nextInt(100) < majorRatio; LOG.info("Performing action: Compact random region of table " + tableName + ", major=" + major); List<HRegionInfo> regions = admin.getTableRegions(tableNameBytes); if (regions == null || regions.isEmpty()) { LOG.info("Table " + tableName + " doesn't have regions to compact"); return; } HRegionInfo region = PolicyBasedChaosMonkey.selectRandomItem( regions.toArray(new HRegionInfo[regions.size()])); try { if (major) { LOG.debug("Major compacting region " + region.getRegionNameAsString()); admin.majorCompact(region.getRegionName()); } else { LOG.debug("Compacting region " + region.getRegionNameAsString()); admin.compact(region.getRegionName()); } } catch (Exception ex) { LOG.warn("Compaction failed, might be caused by other chaos: " + ex.getMessage()); } if (sleepTime > 0) { Thread.sleep(sleepTime); } }
protected void killRs(ServerName server) throws IOException { LOG.info("Killing region server:" + server); cluster.killRegionServer(server); cluster.waitForRegionServerToStop(server, PolicyBasedChaosMonkey.TIMEOUT); LOG.info("Killed region server:" + server + ". Reported num of rs:" + cluster.getClusterStatus().getServersSize()); }
protected void startRs(ServerName server) throws IOException { LOG.info("Starting region server:" + server.getHostname()); cluster.startRegionServer(server.getHostname()); cluster.waitForRegionServerToStart(server.getHostname(), PolicyBasedChaosMonkey.TIMEOUT); LOG.info("Started region server:" + server + ". Reported num of rs:" + cluster.getClusterStatus().getServersSize()); }
@Override public ChaosMonkey build() { Policy chaosPolicy = new PeriodicRandomActionPolicy( CHAOS_EVERY_MS, new UnbalanceKillAndRebalanceAction() ); return new PolicyBasedChaosMonkey(util, chaosPolicy); }
@Override public void perform() throws Exception { HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); Admin admin = util.getAdmin(); boolean major = RandomUtils.nextInt(0, 100) < majorRatio; LOG.info("Performing action: Compact random region of table " + tableName + ", major=" + major); List<HRegionInfo> regions = admin.getTableRegions(tableName); if (regions == null || regions.isEmpty()) { LOG.info("Table " + tableName + " doesn't have regions to compact"); return; } HRegionInfo region = PolicyBasedChaosMonkey.selectRandomItem( regions.toArray(new HRegionInfo[regions.size()])); try { if (major) { LOG.debug("Major compacting region " + region.getRegionNameAsString()); admin.majorCompactRegion(region.getRegionName()); } else { LOG.debug("Compacting region " + region.getRegionNameAsString()); admin.compactRegion(region.getRegionName()); } } catch (Exception ex) { LOG.warn("Compaction failed, might be caused by other chaos: " + ex.getMessage()); } if (sleepTime > 0) { Thread.sleep(sleepTime); } }
@Override public void perform() throws Exception { HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); Admin admin = util.getAdmin(); LOG.info("Performing action: Split random region of table " + tableName); List<HRegionInfo> regions = admin.getTableRegions(tableName); if (regions == null || regions.isEmpty()) { LOG.info("Table " + tableName + " doesn't have regions to split"); return; } // Don't try the split if we're stopping if (context.isStopping()) { return; } HRegionInfo region = PolicyBasedChaosMonkey.selectRandomItem( regions.toArray(new HRegionInfo[regions.size()])); LOG.debug("Splitting region " + region.getRegionNameAsString()); try { admin.splitRegion(region.getRegionName()); } catch (Exception ex) { LOG.warn("Split failed, might be caused by other chaos: " + ex.getMessage()); } if (sleepTime > 0) { Thread.sleep(sleepTime); } }
@Override protected void runOneIteration() { Action action = PolicyBasedChaosMonkey.selectWeightedRandomItem(actions); try { action.perform(); } catch (Exception ex) { LOG.warn("Exception occurred during performing action: " + StringUtils.stringifyException(ex)); } }