@SuppressWarnings("unchecked") public JVMClusterUtil.RegionServerThread addRegionServer( Configuration config, final int index) throws IOException { // Create each regionserver with its own Configuration instance so each has // its HConnection instance rather than share (see HBASE_INSTANCES down in // the guts of HConnectionManager. // Also, create separate CoordinatedStateManager instance per Server. // This is special case when we have to have more than 1 CoordinatedStateManager // within 1 process. CoordinatedStateManager cp = CoordinatedStateManagerFactory.getCoordinatedStateManager(conf); JVMClusterUtil.RegionServerThread rst = JVMClusterUtil.createRegionServerThread(config, cp, (Class<? extends HRegionServer>) conf .getClass(HConstants.REGION_SERVER_IMPL, this.regionServerClass), index); this.regionThreads.add(rst); return rst; }
public JVMClusterUtil.MasterThread addMaster(Configuration c, final int index) throws IOException { // Create each master with its own Configuration instance so each has // its HConnection instance rather than share (see HBASE_INSTANCES down in // the guts of HConnectionManager. // Also, create separate CoordinatedStateManager instance per Server. // This is special case when we have to have more than 1 CoordinatedStateManager // within 1 process. CoordinatedStateManager cp = CoordinatedStateManagerFactory.getCoordinatedStateManager(conf); JVMClusterUtil.MasterThread mt = JVMClusterUtil.createMasterThread(c, cp, (Class<? extends HMaster>) conf.getClass(HConstants.MASTER_IMPL, this.masterClass), index); this.masterThreads.add(mt); return mt; }
/** * Wait for the specified region server to stop * Removes this thread from list of running threads. * @param rst * @return Name of region server that just went down. */ public String waitOnRegionServer(JVMClusterUtil.RegionServerThread rst) { while (rst.isAlive()) { try { LOG.info("Waiting on " + rst.getRegionServer().toString()); rst.join(); } catch (InterruptedException e) { e.printStackTrace(); } } for (int i=0;i<regionThreads.size();i++) { if (regionThreads.get(i) == rst) { regionThreads.remove(i); break; } } return rst.getName(); }
/** * Wait for the specified master to stop * Removes this thread from list of running threads. * @param masterThread * @return Name of master that just went down. */ public String waitOnMaster(JVMClusterUtil.MasterThread masterThread) { while (masterThread.isAlive()) { try { LOG.info("Waiting on " + masterThread.getMaster().getServerName().toString()); masterThread.join(); } catch (InterruptedException e) { e.printStackTrace(); } } for (int i=0;i<masterThreads.size();i++) { if (masterThreads.get(i) == masterThread) { masterThreads.remove(i); break; } } return masterThread.getName(); }
/** * Starts a region server thread running * * @throws IOException * @return New RegionServerThread */ public JVMClusterUtil.RegionServerThread startRegionServer() throws IOException { final Configuration newConf = HBaseConfiguration.create(conf); User rsUser = HBaseTestingUtility.getDifferentUser(newConf, ".hfs."+index++); JVMClusterUtil.RegionServerThread t = null; try { t = hbaseCluster.addRegionServer( newConf, hbaseCluster.getRegionServers().size(), rsUser); t.start(); t.waitForServerOnline(); } catch (InterruptedException ie) { throw new IOException("Interrupted adding regionserver to cluster", ie); } return t; }
/** * Blocks until there is an active master and that master has completed * initialization. * * @return true if an active master becomes available. false if there are no * masters left. * @throws InterruptedException */ public boolean waitForActiveAndReadyMaster(long timeout) throws IOException { List<JVMClusterUtil.MasterThread> mts; long start = System.currentTimeMillis(); while (!(mts = getMasterThreads()).isEmpty() && (System.currentTimeMillis() - start) < timeout) { for (JVMClusterUtil.MasterThread mt : mts) { if (mt.getMaster().isActiveMaster() && mt.getMaster().isInitialized()) { return true; } } Threads.sleep(100); } return false; }
/** * Make sure that at least the specified number of region servers * are running. We don't count the ones that are currently stopping or are * stopped. * @param num minimum number of region servers that should be running * @return true if we started some servers * @throws IOException */ public boolean ensureSomeNonStoppedRegionServersAvailable(final int num) throws IOException { boolean startedServer = ensureSomeRegionServersAvailable(num); int nonStoppedServers = 0; for (JVMClusterUtil.RegionServerThread rst : getMiniHBaseCluster().getRegionServerThreads()) { HRegionServer hrs = rst.getRegionServer(); if (hrs.isStopping() || hrs.isStopped()) { LOG.info("A region server is stopped or stopping:"+hrs); } else { nonStoppedServers++; } } for (int i=nonStoppedServers; i<num; ++i) { LOG.info("Started new server=" + getMiniHBaseCluster().startRegionServer()); startedServer = true; } return startedServer; }
@Test public void testClusterId() throws Exception { TEST_UTIL.startMiniZKCluster(); TEST_UTIL.startMiniDFSCluster(1); Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); CoordinatedStateManager cp = CoordinatedStateManagerFactory.getCoordinatedStateManager(conf); //start region server, needs to be separate //so we get an unset clusterId rst = JVMClusterUtil.createRegionServerThread(conf,cp, HRegionServer.class, 0); rst.start(); //Make sure RS is in blocking state Thread.sleep(10000); TEST_UTIL.startMiniHBaseCluster(1, 1); rst.waitForServerOnline(); String clusterId = ZKClusterId.readClusterIdZNode(TEST_UTIL.getZooKeeperWatcher()); assertNotNull(clusterId); assertEquals(clusterId, rst.getRegionServer().getClusterId()); }
@Test public void testClusterId() throws Exception { TEST_UTIL.startMiniZKCluster(); TEST_UTIL.startMiniDFSCluster(1); Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); //start region server, needs to be separate //so we get an unset clusterId rst = JVMClusterUtil.createRegionServerThread(conf, HRegionServer.class, 0); rst.start(); //Make sure RS is in blocking state Thread.sleep(10000); TEST_UTIL.startMiniHBaseCluster(1, 0); rst.waitForServerOnline(); String clusterId = ZKClusterId.readClusterIdZNode(TEST_UTIL.getZooKeeperWatcher()); assertNotNull(clusterId); assertEquals(clusterId, rst.getRegionServer().getClusterId()); }
/** * Blocks until there is an active master and that master has completed * initialization. * * @return true if an active master becomes available. false if there are no * masters left. * @throws InterruptedException */ public boolean waitForActiveAndReadyMaster(long timeout) throws IOException { List<JVMClusterUtil.MasterThread> mts; long start = System.currentTimeMillis(); while (!(mts = getMasterThreads()).isEmpty() && (System.currentTimeMillis() - start) < timeout) { for (JVMClusterUtil.MasterThread mt : mts) { ServerManager serverManager = mt.getMaster().getServerManager(); if (mt.getMaster().isActiveMaster() && mt.getMaster().isInitialized() && !serverManager.areDeadServersInProgress()) { return true; } } Threads.sleep(100); } return false; }
private void verifyMethodResult(Class c, String methodName[], byte[] tableName, Object value[]) throws IOException { try { for (JVMClusterUtil.RegionServerThread t : cluster.getRegionServerThreads()) { for (HRegionInfo r : t.getRegionServer().getOnlineRegions()) { if (!Arrays.equals(r.getTableName(), tableName)) { continue; } RegionCoprocessorHost cph = t.getRegionServer().getOnlineRegion(r.getRegionName()). getCoprocessorHost(); Coprocessor cp = cph.findCoprocessor(c.getName()); assertNotNull(cp); for (int i = 0; i < methodName.length; ++i) { Method m = c.getMethod(methodName[i]); Object o = m.invoke(cp); assertTrue("Result of " + c.getName() + "." + methodName[i] + " is expected to be " + value[i].toString() + ", while we get " + o.toString(), o.equals(value[i])); } } } } catch (Exception e) { throw new IOException(e.toString()); } }
/** * Get the location of the specified region * @param regionName Name of the region in bytes * @return Index into List of {@link MiniHBaseCluster#getRegionServerThreads()} * of HRS carrying hbase:meta. Returns -1 if none found. */ public int getServerWith(byte[] regionName) { int index = -1; int count = 0; for (JVMClusterUtil.RegionServerThread rst: getRegionServerThreads()) { HRegionServer hrs = rst.getRegionServer(); HRegion metaRegion = hrs.getOnlineRegion(regionName); if (metaRegion != null) { index = count; break; } count++; } return index; }
public JVMClusterUtil.RegionServerThread addRegionServer( final Configuration config, final int index, User user) throws IOException, InterruptedException { return user.runAs( new PrivilegedExceptionAction<JVMClusterUtil.RegionServerThread>() { public JVMClusterUtil.RegionServerThread run() throws Exception { return addRegionServer(config, index); } }); }
public JVMClusterUtil.MasterThread addMaster( final Configuration c, final int index, User user) throws IOException, InterruptedException { return user.runAs( new PrivilegedExceptionAction<JVMClusterUtil.MasterThread>() { public JVMClusterUtil.MasterThread run() throws Exception { return addMaster(c, index); } }); }
/** * @return List of running servers (Some servers may have been killed or * aborted during lifetime of cluster; these servers are not included in this * list). */ public List<JVMClusterUtil.RegionServerThread> getLiveRegionServers() { List<JVMClusterUtil.RegionServerThread> liveServers = new ArrayList<JVMClusterUtil.RegionServerThread>(); List<RegionServerThread> list = getRegionServers(); for (JVMClusterUtil.RegionServerThread rst: list) { if (rst.isAlive()) liveServers.add(rst); else LOG.info("Not alive " + rst.getName()); } return liveServers; }
/** * Wait for the specified region server to stop * Removes this thread from list of running threads. * @param serverNumber * @return Name of region server that just went down. */ public String waitOnRegionServer(int serverNumber) { JVMClusterUtil.RegionServerThread regionServerThread = this.regionThreads.remove(serverNumber); while (regionServerThread.isAlive()) { try { LOG.info("Waiting on " + regionServerThread.getRegionServer().toString()); regionServerThread.join(); } catch (InterruptedException e) { e.printStackTrace(); } } return regionServerThread.getName(); }
/** * Gets the current active master, if available. If no active master, returns * null. * @return the HMaster for the active master */ public HMaster getActiveMaster() { for (JVMClusterUtil.MasterThread mt : masterThreads) { if (mt.getMaster().isActiveMaster()) { // Ensure that the current active master is not stopped. // We don't want to return a stopping master as an active master. if (mt.getMaster().isActiveMaster() && !mt.getMaster().isStopped()) { return mt.getMaster(); } } } return null; }
/** * @return List of running master servers (Some servers may have been killed * or aborted during lifetime of cluster; these servers are not included in * this list). */ public List<JVMClusterUtil.MasterThread> getLiveMasters() { List<JVMClusterUtil.MasterThread> liveServers = new ArrayList<JVMClusterUtil.MasterThread>(); List<JVMClusterUtil.MasterThread> list = getMasters(); for (JVMClusterUtil.MasterThread mt: list) { if (mt.isAlive()) { liveServers.add(mt); } } return liveServers; }
/** * Wait for the specified master to stop * Removes this thread from list of running threads. * @param serverNumber * @return Name of master that just went down. */ public String waitOnMaster(int serverNumber) { JVMClusterUtil.MasterThread masterThread = this.masterThreads.remove(serverNumber); while (masterThread.isAlive()) { try { LOG.info("Waiting on " + masterThread.getMaster().getServerName().toString()); masterThread.join(); } catch (InterruptedException e) { e.printStackTrace(); } } return masterThread.getName(); }
private void waitOnMasterThreads(LocalHBaseCluster cluster) throws InterruptedException{ List<JVMClusterUtil.MasterThread> masters = cluster.getMasters(); List<JVMClusterUtil.RegionServerThread> regionservers = cluster.getRegionServers(); if (masters != null) { for (JVMClusterUtil.MasterThread t : masters) { t.join(); if(t.getMaster().isAborted()) { closeAllRegionServerThreads(regionservers); throw new RuntimeException("HMaster Aborted"); } } } }
/** * @throws java.lang.Exception */ @Before public void setUp() throws Exception { // Starting and stopping replication can make us miss new logs, // rolling like this makes sure the most recent one gets added to the queue for ( JVMClusterUtil.RegionServerThread r : utility1.getHBaseCluster().getRegionServerThreads()) { utility1.getHBaseAdmin().rollWALWriter(r.getRegionServer().getServerName()); } utility1.deleteTableData(tableName); // truncating the table will send one Delete per row to the slave cluster // in an async fashion, which is why we cannot just call deleteTableData on // utility2 since late writes could make it to the slave in some way. // Instead, we truncate the first table and wait for all the Deletes to // make it to the slave. Scan scan = new Scan(); int lastCount = 0; for (int i = 0; i < NB_RETRIES; i++) { if (i==NB_RETRIES-1) { fail("Waited too much time for truncate"); } ResultScanner scanner = htable2.getScanner(scan); Result[] res = scanner.next(NB_ROWS_IN_BIG_BATCH); scanner.close(); if (res.length != 0) { if (res.length < lastCount) { i--; // Don't increment timeout if we make progress } lastCount = res.length; LOG.info("Still got " + res.length + " rows"); Thread.sleep(SLEEP_TIME); } else { break; } } }
/** * @throws java.lang.Exception */ @Before public void setUp() throws Exception { // Starting and stopping replication can make us miss new logs, // rolling like this makes sure the most recent one gets added to the queue for (JVMClusterUtil.RegionServerThread r : utility1.getHBaseCluster().getRegionServerThreads()) { utility1.getHBaseAdmin().rollWALWriter(r.getRegionServer().getServerName()); } utility1.deleteTableData(tableName); // truncating the table will send one Delete per row to the slave cluster // in an async fashion, which is why we cannot just call deleteTableData on // utility2 since late writes could make it to the slave in some way. // Instead, we truncate the first table and wait for all the Deletes to // make it to the slave. Scan scan = new Scan(); int lastCount = 0; for (int i = 0; i < NB_RETRIES; i++) { if (i == NB_RETRIES - 1) { fail("Waited too much time for truncate"); } ResultScanner scanner = htable2.getScanner(scan); Result[] res = scanner.next(NB_ROWS_IN_BIG_BATCH); scanner.close(); if (res.length != 0) { if (res.length < lastCount) { i--; // Don't increment timeout if we make progress } lastCount = res.length; LOG.info("Still got " + res.length + " rows"); Thread.sleep(SLEEP_TIME); } else { break; } } }
/** * Starts a master thread running * * @throws IOException * @return New RegionServerThread */ public JVMClusterUtil.MasterThread startMaster() throws IOException { Configuration c = HBaseConfiguration.create(conf); User user = HBaseTestingUtility.getDifferentUser(c, ".hfs."+index++); JVMClusterUtil.MasterThread t = null; try { t = hbaseCluster.addMaster(c, hbaseCluster.getMasters().size(), user); t.start(); } catch (InterruptedException ie) { throw new IOException("Interrupted adding master to cluster", ie); } return t; }
/** * Call flushCache on all regions on all participating regionservers. * @throws IOException */ public void flushcache() throws IOException { for (JVMClusterUtil.RegionServerThread t: this.hbaseCluster.getRegionServers()) { for(Region r: t.getRegionServer().getOnlineRegionsLocalContext()) { r.flush(true); } } }
/** * Call flushCache on all regions of the specified table. * @throws IOException */ public void flushcache(TableName tableName) throws IOException { for (JVMClusterUtil.RegionServerThread t: this.hbaseCluster.getRegionServers()) { for(Region r: t.getRegionServer().getOnlineRegionsLocalContext()) { if(r.getTableDesc().getTableName().equals(tableName)) { r.flush(true); } } } }
/** * Call flushCache on all regions on all participating regionservers. * @throws IOException */ public void compact(boolean major) throws IOException { for (JVMClusterUtil.RegionServerThread t: this.hbaseCluster.getRegionServers()) { for(Region r: t.getRegionServer().getOnlineRegionsLocalContext()) { r.compact(major); } } }
/** * Call flushCache on all regions of the specified table. * @throws IOException */ public void compact(TableName tableName, boolean major) throws IOException { for (JVMClusterUtil.RegionServerThread t: this.hbaseCluster.getRegionServers()) { for(Region r: t.getRegionServer().getOnlineRegionsLocalContext()) { if(r.getTableDesc().getTableName().equals(tableName)) { r.compact(major); } } } }
