/** * Shutdown HBase mini cluster. Does not shutdown zk or dfs if running. * @throws IOException */ public void shutdownMiniHBaseCluster() throws IOException { if (hbaseAdmin != null) { hbaseAdmin.close0(); hbaseAdmin = null; } // unset the configuration for MIN and MAX RS to start conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1); conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, -1); if (this.hbaseCluster != null) { this.hbaseCluster.shutdown(); // Wait till hbase is down before going on to shutdown zk. this.hbaseCluster.waitUntilShutDown(); this.hbaseCluster = null; } if (zooKeeperWatcher != null) { zooKeeperWatcher.close(); zooKeeperWatcher = null; } }
/** * Blocks until there is an active master and that master has completed * initialization. * * @return true if an active master becomes available. false if there are no * masters left. * @throws InterruptedException */ public boolean waitForActiveAndReadyMaster(long timeout) throws IOException { List<JVMClusterUtil.MasterThread> mts; long start = System.currentTimeMillis(); while (!(mts = getMasterThreads()).isEmpty() && (System.currentTimeMillis() - start) < timeout) { for (JVMClusterUtil.MasterThread mt : mts) { ServerManager serverManager = mt.getMaster().getServerManager(); if (mt.getMaster().isActiveMaster() && mt.getMaster().isInitialized() && !serverManager.areDeadServersInProgress()) { return true; } } Threads.sleep(100); } return false; }
/** * Shutdown HBase mini cluster. Does not shutdown zk or dfs if running. * @throws IOException */ public void shutdownMiniHBaseCluster() throws IOException { if (hbaseAdmin != null) { hbaseAdmin.close(); hbaseAdmin = null; } // unset the configuration for MIN and MAX RS to start conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1); conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, -1); if (this.hbaseCluster != null) { this.hbaseCluster.shutdown(); // Wait till hbase is down before going on to shutdown zk. this.hbaseCluster.waitUntilShutDown(); this.hbaseCluster = null; } }
/** * Shutdown HBase mini cluster. Does not shutdown zk or dfs if running. * @throws IOException */ public void shutdownMiniHBaseCluster() throws IOException { if (hbaseAdmin != null) { hbaseAdmin.close(); hbaseAdmin = null; } // unset the configuration for MIN and MAX RS to start conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1); conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, -1); if (this.hbaseCluster != null) { this.hbaseCluster.shutdown(); // Wait till hbase is down before going on to shutdown zk. this.hbaseCluster.waitUntilShutDown(); this.hbaseCluster = null; } if (zooKeeperWatcher != null) { zooKeeperWatcher.close(); zooKeeperWatcher = null; } }
@BeforeClass public static void setUp() throws Exception { TEST_UTIL = new HBaseTestingUtility(); TEST_UTIL.getConfiguration().set( HConstants.HBASE_MASTER_LOADBALANCER_CLASS, RSGroupBasedLoadBalancer.class.getName()); TEST_UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, RSGroupAdminEndpoint.class.getName()); TEST_UTIL.getConfiguration().set( ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, "1"); TEST_UTIL.startMiniCluster(2, 3); cluster = TEST_UTIL.getHBaseCluster(); master = ((MiniHBaseCluster)cluster).getMaster(); master.balanceSwitch(false); hbaseAdmin = TEST_UTIL.getAdmin(); //wait till the balancer is in online mode TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { @Override public boolean evaluate() throws Exception { return master.isInitialized() && ((RSGroupBasedLoadBalancer) master.getLoadBalancer()).isOnline() && master.getServerManager().getOnlineServersList().size() >= 3; } }); }
/** * @param regionsInMeta * @return List of regions neither in transition nor assigned. * @throws IOException */ private List<HRegionInfo> regionsToAssignWithServerName( final List<Pair<HRegionInfo, ServerName>> regionsInMeta) throws IOException { ServerManager serverManager = ((HMaster) this.server).getServerManager(); List<HRegionInfo> regions = new ArrayList<HRegionInfo>(); RegionStates regionStates = this.assignmentManager.getRegionStates(); for (Pair<HRegionInfo, ServerName> regionLocation : regionsInMeta) { HRegionInfo hri = regionLocation.getFirst(); ServerName sn = regionLocation.getSecond(); if (regionStates.isRegionOffline(hri)) { if (sn != null && serverManager.isServerOnline(sn)) { this.assignmentManager.addPlan(hri.getEncodedName(), new RegionPlan(hri, null, sn)); } regions.add(hri); } else { if (LOG.isDebugEnabled()) { LOG.debug("Skipping assign for the region " + hri + " during enable table " + hri.getTable() + " because its already in tranition or assigned."); } } } return regions; }
/** * Shutdown HBase mini cluster. Does not shutdown zk or dfs if running. * @throws IOException */ public void shutdownMiniHBaseCluster() throws IOException { if (hbaseAdmin != null) { hbaseAdmin.close0(); hbaseAdmin = null; } if (zooKeeperWatcher != null) { zooKeeperWatcher.close(); zooKeeperWatcher = null; } // unset the configuration for MIN and MAX RS to start conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1); conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, -1); if (this.hbaseCluster != null) { this.hbaseCluster.shutdown(); // Wait till hbase is down before going on to shutdown zk. this.hbaseCluster.waitUntilShutDown(); this.hbaseCluster = null; } }
/** * @param regionsInMeta * @return List of regions neither in transition nor assigned. * @throws IOException */ private List<HRegionInfo> regionsToAssignWithServerName( final List<Pair<HRegionInfo, ServerName>> regionsInMeta) throws IOException { ServerManager serverManager = ((HMaster) this.server).getServerManager(); List<HRegionInfo> regions = new ArrayList<HRegionInfo>(); RegionStates regionStates = this.assignmentManager.getRegionStates(); for (Pair<HRegionInfo, ServerName> regionLocation : regionsInMeta) { HRegionInfo hri = regionLocation.getFirst(); ServerName sn = regionLocation.getSecond(); if (!regionStates.isRegionInTransition(hri) && !regionStates.isRegionAssigned(hri)) { if (this.retainAssignment && sn != null && serverManager.isServerOnline(sn)) { this.assignmentManager.addPlan(hri.getEncodedName(), new RegionPlan(hri, null, sn)); } regions.add(hri); } else { if (LOG.isDebugEnabled()) { LOG.debug("Skipping assign for the region " + hri + " during enable table " + hri.getTableNameAsString() + " because its already in tranition or assigned."); } } } return regions; }
/** * Shutdown HBase mini cluster. Does not shutdown zk or dfs if running. * @throws IOException */ public void shutdownMiniHBaseCluster() throws IOException { if (hbaseAdmin != null) { hbaseAdmin.close(); hbaseAdmin = null; } if (zooKeeperWatcher != null) { zooKeeperWatcher.close(); zooKeeperWatcher = null; } // unset the configuration for MIN and MAX RS to start conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1); conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, -1); if (this.hbaseCluster != null) { this.hbaseCluster.shutdown(); // Wait till hbase is down before going on to shutdown zk. this.hbaseCluster.waitUntilShutDown(); this.hbaseCluster = null; } }
private RegionLoad getRegionLoad(ServerName sn, HRegionInfo hri) { ServerManager serverManager = masterServices.getServerManager(); ServerLoad load = serverManager.getLoad(sn); if (load != null) { Map<byte[], RegionLoad> regionsLoad = load.getRegionsLoad(); if (regionsLoad != null) { return regionsLoad.get(hri.getRegionName()); } } return null; }
protected static void waitRegionServers(final MasterProcedureEnv env) throws IOException { final ServerManager sm = env.getMasterServices().getServerManager(); ProcedureSyncWait.waitFor(env, "server to assign region(s)", new ProcedureSyncWait.Predicate<Boolean>() { @Override public Boolean evaluate() throws IOException { List<ServerName> servers = sm.createDestinationServersList(); return servers != null && !servers.isEmpty(); } }); }
/** * Contacts a region server and waits up to hbase.hbck.close.timeout ms * (default 120s) to close the region. This bypasses the active hmaster. */ @SuppressWarnings("deprecation") public static void closeRegionSilentlyAndWait(HConnection connection, ServerName server, HRegionInfo region) throws IOException, InterruptedException { long timeout = connection.getConfiguration() .getLong("hbase.hbck.close.timeout", 120000); ServerManager.closeRegionSilentlyAndWait((ClusterConnection)connection, server, region, timeout); }
private void addServerToDrainedList(ServerName serverName, Map<ServerName, ServerLoad> onlineServers, ServerManager serverManager) { onlineServers.remove(serverName); List<ServerName> availableServers = new ArrayList<ServerName>(onlineServers.keySet()); Mockito.when(serverManager.createDestinationServersList()).thenReturn(availableServers); Mockito.when(serverManager.createDestinationServersList(null)).thenReturn(availableServers); }
/** * Starts up mini hbase cluster. Usually used after call to * {@link #startMiniCluster(int, int)} when doing stepped startup of clusters. * Usually you won't want this. You'll usually want {@link #startMiniCluster()}. * @param numMasters * @param numSlaves * @param create Whether to create a * root or data directory path or not; will overwrite if exists already. * @return Reference to the hbase mini hbase cluster. * @throws IOException * @throws InterruptedException * @see {@link #startMiniCluster()} */ public MiniHBaseCluster startMiniHBaseCluster(final int numMasters, final int numSlaves, Class<? extends HMaster> masterClass, Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass, boolean create) throws IOException, InterruptedException { // Now do the mini hbase cluster. Set the hbase.rootdir in config. createRootDir(create); // These settings will make the server waits until this exact number of // regions servers are connected. if (conf.getInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1) == -1) { conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, numSlaves); } if (conf.getInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, -1) == -1) { conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, numSlaves); } Configuration c = new Configuration(this.conf); this.hbaseCluster = new MiniHBaseCluster(c, numMasters, numSlaves, masterClass, regionserverClass); // Don't leave here till we've done a successful scan of the hbase:meta Table t = new HTable(c, TableName.META_TABLE_NAME); ResultScanner s = t.getScanner(new Scan()); while (s.next() != null) { continue; } s.close(); t.close(); getHBaseAdmin(); // create immediately the hbaseAdmin LOG.info("Minicluster is up"); // Set the hbase.fs.tmp.dir config to make sure that we have some default value. This is // for tests that do not read hbase-defaults.xml setHBaseFsTmpDir(); return (MiniHBaseCluster)this.hbaseCluster; }
private void decrementMinRegionServerCount(Configuration conf) { int currentCount = conf.getInt( ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1); if (currentCount != -1) { conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, Math.max(currentCount - 1, 1)); } }
/** * @param regionsInMeta This datastructure is edited by this method. * @return List of regions neither in transition nor assigned. * @throws IOException */ private List<HRegionInfo> regionsToAssignWithServerName( final List<Pair<HRegionInfo, ServerName>> regionsInMeta) throws IOException { ServerManager serverManager = ((HMaster) this.server).getServerManager(); List<HRegionInfo> regions = new ArrayList<HRegionInfo>(); List<HRegionInfo> enablingTableRegions = this.assignmentManager .getEnablingTableRegions(this.tableNameStr); final List<HRegionInfo> onlineRegions = this.assignmentManager.getRegionsOfTable(tableName); for (Pair<HRegionInfo, ServerName> regionLocation : regionsInMeta) { HRegionInfo hri = regionLocation.getFirst(); ServerName sn = regionLocation.getSecond(); if (this.skipTableStateCheck) { // Region may be available in enablingTableRegions during master startup only. if (enablingTableRegions != null && enablingTableRegions.contains(hri)) { regions.add(hri); if (sn != null && serverManager.isServerOnline(sn)) { this.assignmentManager.addPlan(hri.getEncodedName(), new RegionPlan(hri, null, sn)); } } } else if (onlineRegions.contains(hri)) { continue; } else { regions.add(hri); if (sn != null && serverManager.isServerOnline(sn)) { this.assignmentManager.addPlan(hri.getEncodedName(), new RegionPlan(hri, null, sn)); } } } return regions; }
/** * Starts up mini hbase cluster. Usually used after call to * {@link #startMiniCluster(int, int)} when doing stepped startup of clusters. * Usually you won't want this. You'll usually want {@link #startMiniCluster()}. * @param numMasters * @param numSlaves * @return Reference to the hbase mini hbase cluster. * @throws IOException * @throws InterruptedException * @see {@link #startMiniCluster()} */ public MiniHBaseCluster startMiniHBaseCluster(final int numMasters, final int numSlaves) throws IOException, InterruptedException { // Now do the mini hbase cluster. Set the hbase.rootdir in config. createRootDir(); // These settings will make the server waits until this exact number of // regions servers are connected. if (conf.getInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1) == -1) { conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, numSlaves); } if (conf.getInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, -1) == -1) { conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, numSlaves); } Configuration c = new Configuration(this.conf); this.hbaseCluster = new MiniHBaseCluster(c, numMasters, numSlaves); // Don't leave here till we've done a successful scan of the .META. HTable t = new HTable(c, HConstants.META_TABLE_NAME); ResultScanner s = t.getScanner(new Scan()); while (s.next() != null) { continue; } s.close(); t.close(); getHBaseAdmin(); // create immediately the hbaseAdmin LOG.info("Minicluster is up"); return (MiniHBaseCluster)this.hbaseCluster; }
@BeforeClass public static void setUpBeforeClass() throws Exception { // Set it so that this test runs with my custom master Configuration conf = TESTUTIL.getConfiguration(); conf.setClass(HConstants.MASTER_IMPL, TestingMaster.class, HMaster.class); conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 3); conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 5); // Start up the cluster. TESTUTIL.startMiniCluster(NUM_MASTERS, NUM_RS); }
/** * Starts up mini hbase cluster. Usually used after call to * {@link #startMiniCluster(int, int)} when doing stepped startup of clusters. * Usually you won't want this. You'll usually want {@link #startMiniCluster()}. * @param numMasters * @param numSlaves * @param create Whether to create a * root or data directory path or not; will overwrite if exists already. * @return Reference to the hbase mini hbase cluster. * @throws IOException * @throws InterruptedException * @see {@link #startMiniCluster()} */ public MiniHBaseCluster startMiniHBaseCluster(final int numMasters, final int numSlaves, Class<? extends HMaster> masterClass, Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass, boolean create) throws IOException, InterruptedException { // Now do the mini hbase cluster. Set the hbase.rootdir in config. createRootDir(create); // These settings will make the server waits until this exact number of // regions servers are connected. if (conf.getInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1) == -1) { conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, numSlaves); } if (conf.getInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, -1) == -1) { conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, numSlaves); } Configuration c = new Configuration(this.conf); this.hbaseCluster = new MiniHBaseCluster(c, numMasters, numSlaves, masterClass, regionserverClass); // Don't leave here till we've done a successful scan of the hbase:meta Table t = new HTable(c, TableName.META_TABLE_NAME); ResultScanner s = t.getScanner(new Scan()); while (s.next() != null) { continue; } s.close(); t.close(); getHBaseAdmin(); // create immediately the hbaseAdmin LOG.info("Minicluster is up"); return (MiniHBaseCluster)this.hbaseCluster; }
/** * Starts up mini hbase cluster. Usually used after call to * {@link #startMiniCluster(int, int)} when doing stepped startup of clusters. * Usually you won't want this. You'll usually want {@link #startMiniCluster()}. * @param numMasters * @param numSlaves * @return Reference to the hbase mini hbase cluster. * @throws IOException * @throws InterruptedException * @see {@link #startMiniCluster()} */ public MiniHBaseCluster startMiniHBaseCluster(final int numMasters, final int numSlaves, Class<? extends HMaster> masterClass, Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass) throws IOException, InterruptedException { // Now do the mini hbase cluster. Set the hbase.rootdir in config. createRootDir(); // These settings will make the server waits until this exact number of // regions servers are connected. if (conf.getInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1) == -1) { conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, numSlaves); } if (conf.getInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, -1) == -1) { conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, numSlaves); } Configuration c = new Configuration(this.conf); this.hbaseCluster = new MiniHBaseCluster(c, numMasters, numSlaves, masterClass, regionserverClass); // Don't leave here till we've done a successful scan of the hbase:meta HTable t = new HTable(c, TableName.META_TABLE_NAME); ResultScanner s = t.getScanner(new Scan()); while (s.next() != null) { continue; } s.close(); t.close(); getHBaseAdmin(); // create immediately the hbaseAdmin LOG.info("Minicluster is up"); return (MiniHBaseCluster)this.hbaseCluster; }