@Override // Nothing is done w/ the 'master' parameter. It is ignored. public AdminService.BlockingInterface getAdmin(final ServerName serverName, final boolean master) throws IOException { if (isDeadServer(serverName)) { throw new RegionServerStoppedException(serverName + " is dead."); } String key = getStubKey(AdminService.BlockingInterface.class.getName(), serverName.getHostname(), serverName.getPort(), this.hostnamesCanChange); this.connectionLock.putIfAbsent(key, key); AdminService.BlockingInterface stub = null; synchronized (this.connectionLock.get(key)) { stub = (AdminService.BlockingInterface)this.stubs.get(key); if (stub == null) { BlockingRpcChannel channel = this.rpcClient.createBlockingRpcChannel(serverName, user, rpcTimeout); stub = AdminService.newBlockingStub(channel); this.stubs.put(key, stub); } } return stub; }
@Override public ClientService.BlockingInterface getClient(final ServerName sn) throws IOException { if (isDeadServer(sn)) { throw new RegionServerStoppedException(sn + " is dead."); } String key = getStubKey(ClientService.BlockingInterface.class.getName(), sn.getHostname(), sn.getPort(), this.hostnamesCanChange); this.connectionLock.putIfAbsent(key, key); ClientService.BlockingInterface stub = null; synchronized (this.connectionLock.get(key)) { stub = (ClientService.BlockingInterface)this.stubs.get(key); if (stub == null) { BlockingRpcChannel channel = this.rpcClient.createBlockingRpcChannel(sn, user, rpcTimeout); stub = ClientService.newBlockingStub(channel); // In old days, after getting stub/proxy, we'd make a call. We are not doing that here. // Just fail on first actual call rather than in here on setup. this.stubs.put(key, stub); } } return stub; }
ScanOpenNextThenExceptionThenRecoverConnection(Configuration conf, boolean managed, ExecutorService pool) throws IOException { super(conf, managed); // Mock up my stub so open scanner returns a scanner id and then on next, we throw // exceptions for three times and then after that, we return no more to scan. this.stub = Mockito.mock(ClientService.BlockingInterface.class); long sid = 12345L; try { Mockito.when(stub.scan((RpcController)Mockito.any(), (ClientProtos.ScanRequest)Mockito.any())). thenReturn(ClientProtos.ScanResponse.newBuilder().setScannerId(sid).build()). thenThrow(new ServiceException(new RegionServerStoppedException("From Mockito"))). thenReturn(ClientProtos.ScanResponse.newBuilder().setScannerId(sid). setMoreResults(false).build()); } catch (ServiceException e) { throw new IOException(e); } }
RegionServerStoppedOnScannerOpenConnection(Configuration conf, boolean managed, ExecutorService pool, User user) throws IOException { super(conf, managed); // Mock up my stub so open scanner returns a scanner id and then on next, we throw // exceptions for three times and then after that, we return no more to scan. this.stub = Mockito.mock(ClientService.BlockingInterface.class); long sid = 12345L; try { Mockito.when(stub.scan((RpcController)Mockito.any(), (ClientProtos.ScanRequest)Mockito.any())). thenReturn(ClientProtos.ScanResponse.newBuilder().setScannerId(sid).build()). thenThrow(new ServiceException(new RegionServerStoppedException("From Mockito"))). thenReturn(ClientProtos.ScanResponse.newBuilder().setScannerId(sid). setMoreResults(false).build()); } catch (ServiceException e) { throw new IOException(e); } }
@Override // Nothing is done w/ the 'master' parameter. It is ignored. public AdminService.BlockingInterface getAdmin(final ServerName serverName, final boolean master) throws IOException { if (isDeadServer(serverName)) { throw new RegionServerStoppedException(serverName + " is dead."); } String key = getStubKey(AdminService.BlockingInterface.class.getName(), serverName.getHostAndPort()); this.connectionLock.putIfAbsent(key, key); AdminService.BlockingInterface stub = null; synchronized (this.connectionLock.get(key)) { stub = (AdminService.BlockingInterface) this.stubs.get(key); if (stub == null) { BlockingRpcChannel channel = this.rpcClient.createBlockingRpcChannel(serverName, user, rpcTimeout); stub = AdminService.newBlockingStub(channel); this.stubs.put(key, stub); } } return stub; }
@Override public ClientService.BlockingInterface getClient(final ServerName sn) throws IOException { if (isDeadServer(sn)) { throw new RegionServerStoppedException(sn + " is dead."); } String key = getStubKey(ClientService.BlockingInterface.class.getName(), sn.getHostAndPort()); this.connectionLock.putIfAbsent(key, key); ClientService.BlockingInterface stub = null; synchronized (this.connectionLock.get(key)) { stub = (ClientService.BlockingInterface) this.stubs.get(key); if (stub == null) { BlockingRpcChannel channel = this.rpcClient.createBlockingRpcChannel(sn, user, rpcTimeout); stub = ClientService.newBlockingStub(channel); // In old days, after getting stub/proxy, we'd make a call. We are not doing that here. // Just fail on first actual call rather than in here on setup. this.stubs.put(key, stub); } } return stub; }
@Override // Nothing is done w/ the 'master' parameter. It is ignored. public AdminService.BlockingInterface getAdmin(final ServerName serverName, final boolean master) throws IOException { if (isDeadServer(serverName)) { throw new RegionServerStoppedException(serverName + " is dead."); } String key = getStubKey(AdminService.BlockingInterface.class.getName(), serverName.getHostAndPort()); this.connectionLock.putIfAbsent(key, key); AdminService.BlockingInterface stub = null; synchronized (this.connectionLock.get(key)) { stub = (AdminService.BlockingInterface)this.stubs.get(key); if (stub == null) { BlockingRpcChannel channel = this.rpcClient.createBlockingRpcChannel(serverName, user, this.rpcTimeout); stub = AdminService.newBlockingStub(channel); this.stubs.put(key, stub); } } return stub; }
@Override public ClientService.BlockingInterface getClient(final ServerName sn) throws IOException { if (isDeadServer(sn)) { throw new RegionServerStoppedException(sn + " is dead."); } String key = getStubKey(ClientService.BlockingInterface.class.getName(), sn.getHostAndPort()); this.connectionLock.putIfAbsent(key, key); ClientService.BlockingInterface stub = null; synchronized (this.connectionLock.get(key)) { stub = (ClientService.BlockingInterface)this.stubs.get(key); if (stub == null) { BlockingRpcChannel channel = this.rpcClient.createBlockingRpcChannel(sn, user, this.rpcTimeout); stub = ClientService.newBlockingStub(channel); // In old days, after getting stub/proxy, we'd make a call. We are not doing that here. // Just fail on first actual call rather than in here on setup. this.stubs.put(key, stub); } } return stub; }
@Override public void preGetOp(final ObserverContext<RegionCoprocessorEnvironment> e, final Get get, final List<Cell> results) throws IOException { int replicaId = e.getEnvironment().getRegion().getRegionInfo().getReplicaId(); // Fail for the primary replica, but not for meta if (throwException) { if (!e.getEnvironment().getRegion().getRegionInfo().isMetaRegion() && (replicaId == 0)) { LOG.info("Get, throw Region Server Stopped Exceptoin for region " + e.getEnvironment() .getRegion().getRegionInfo()); throw new RegionServerStoppedException("Server " + e.getEnvironment().getServerName() + " not running"); } } else { LOG.info("Get, We're replica region " + replicaId); } }
@Override protected CloseRegionResponse execCloseRegion(ServerName server, byte[] regionName) throws IOException { switch (this.invocations++) { case 0: throw new NotServingRegionException("Fake"); case 1: throw new RegionServerAbortedException("Fake!"); case 2: throw new RegionServerStoppedException("Fake!"); case 3: throw new ServerNotRunningYetException("Fake!"); case 4: LOG.info("Return null response from serverName=" + server + "; means STUCK...TODO timeout"); executor.schedule(new Runnable() { @Override public void run() { LOG.info("Sending in CRASH of " + server); doCrash(server); } }, 1, TimeUnit.SECONDS); return null; default: return super.execCloseRegion(server, regionName); } }
RegionServerStoppedOnScannerOpenConnection(Configuration conf, ExecutorService pool, User user) throws IOException { super(conf, pool, user); // Mock up my stub so open scanner returns a scanner id and then on next, we throw // exceptions for three times and then after that, we return no more to scan. this.stub = Mockito.mock(ClientService.BlockingInterface.class); long sid = 12345L; try { Mockito.when(stub.scan((RpcController)Mockito.any(), (ClientProtos.ScanRequest)Mockito.any())). thenReturn(ClientProtos.ScanResponse.newBuilder().setScannerId(sid).build()). thenThrow(new ServiceException(new RegionServerStoppedException("From Mockito"))). thenReturn(ClientProtos.ScanResponse.newBuilder().setScannerId(sid). setMoreResults(false).build()); } catch (ServiceException e) { throw new IOException(e); } }
@Override // Nothing is done w/ the 'master' parameter. It is ignored. public AdminService.BlockingInterface getAdmin(final ServerName serverName, final boolean master) throws IOException { if (isDeadServer(serverName)) { throw new RegionServerStoppedException(serverName + " is dead."); } String key = getStubKey(AdminService.BlockingInterface.class.getName(), serverName.getHostAndPort()); this.connectionLock.putIfAbsent(key, key); AdminService.BlockingInterface stub = null; synchronized (this.connectionLock.get(key)) { stub = (AdminService.BlockingInterface)this.stubs.get(key); if (stub == null) { BlockingRpcChannel channel = this.rpcClient.createBlockingRpcChannel(serverName, user, rpcTimeout); stub = AdminService.newBlockingStub(channel); this.stubs.put(key, stub); } } return stub; }
@Override public ClientService.BlockingInterface getClient(final ServerName sn) throws IOException { if (isDeadServer(sn)) { throw new RegionServerStoppedException(sn + " is dead."); } String key = getStubKey(ClientService.BlockingInterface.class.getName(), sn.getHostAndPort()); this.connectionLock.putIfAbsent(key, key); ClientService.BlockingInterface stub = null; synchronized (this.connectionLock.get(key)) { stub = (ClientService.BlockingInterface)this.stubs.get(key); if (stub == null) { BlockingRpcChannel channel = this.rpcClient.createBlockingRpcChannel(sn, user, rpcTimeout); stub = ClientService.newBlockingStub(channel); // In old days, after getting stub/proxy, we'd make a call. We are not doing that here. // Just fail on first actual call rather than in here on setup. this.stubs.put(key, stub); } } return stub; }
RpcTimeoutConnection(Configuration conf, boolean managed, ExecutorService pool, User user) throws IOException { super(conf, managed); // Mock up my stub so an exists call -- which turns into a get -- throws an exception this.stub = Mockito.mock(ClientService.BlockingInterface.class); try { Mockito.when(stub.get((RpcController)Mockito.any(), (ClientProtos.GetRequest)Mockito.any())). thenThrow(new ServiceException(new RegionServerStoppedException("From Mockito"))); } catch (ServiceException e) { throw new IOException(e); } }
@Override protected boolean remoteCallFailed(final MasterProcedureEnv env, final RegionStateNode regionNode, final IOException exception) { // TODO: Is there on-going rpc to cleanup? if (exception instanceof ServerCrashException) { // This exception comes from ServerCrashProcedure after log splitting. // SCP found this region as a RIT. Its call into here says it is ok to let this procedure go // on to a complete close now. This will release lock on this region so subsequent action on // region can succeed; e.g. the assign that follows this unassign when a move (w/o wait on SCP // the assign could run w/o logs being split so data loss). try { reportTransition(env, regionNode, TransitionCode.CLOSED, HConstants.NO_SEQNUM); } catch (UnexpectedStateException e) { // Should never happen. throw new RuntimeException(e); } } else if (exception instanceof RegionServerAbortedException || exception instanceof RegionServerStoppedException || exception instanceof ServerNotRunningYetException) { // TODO // RS is aborting, we cannot offline the region since the region may need to do WAL // recovery. Until we see the RS expiration, we should retry. // TODO: This should be suspend like the below where we call expire on server? LOG.info("Ignoring; waiting on ServerCrashProcedure", exception); } else if (exception instanceof NotServingRegionException) { LOG.info("IS THIS OK? ANY LOGS TO REPLAY; ACTING AS THOUGH ALL GOOD " + regionNode, exception); setTransitionState(RegionTransitionState.REGION_TRANSITION_FINISH); } else { LOG.warn("Expiring server " + this + "; " + regionNode.toShortString() + ", exception=" + exception); env.getMasterServices().getServerManager().expireServer(regionNode.getRegionLocation()); // Return false so this procedure stays in suspended state. It will be woken up by a // ServerCrashProcedure when it notices this RIT. // TODO: Add a SCP as a new subprocedure that we now come to depend on. return false; } return true; }
@Override public void preGetOp(final ObserverContext<RegionCoprocessorEnvironment> e, final Get get, final List<Cell> results) throws IOException { int replicaId = e.getEnvironment().getRegion().getRegionInfo().getReplicaId(); // Fail for the primary replica and replica 1 if (e.getEnvironment().getRegion().getRegionInfo().getReplicaId() <= 1) { LOG.info("Throw Region Server Stopped Exceptoin for replica id " + replicaId); throw new RegionServerStoppedException("Server " + e.getEnvironment().getServerName() + " not running"); } else { LOG.info("We're replica region " + replicaId); } }
@Override public void preScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> e, final Scan scan) throws IOException { int replicaId = e.getEnvironment().getRegion().getRegionInfo().getReplicaId(); // Fail for the primary replica and replica 1 if (e.getEnvironment().getRegion().getRegionInfo().getReplicaId() <= 1) { LOG.info("Throw Region Server Stopped Exceptoin for replica id " + replicaId); throw new RegionServerStoppedException("Server " + e.getEnvironment().getServerName() + " not running"); } else { LOG.info("We're replica region " + replicaId); } }
@Override public void preScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> e, final Scan scan) throws IOException { int replicaId = e.getEnvironment().getRegion().getRegionInfo().getReplicaId(); // Slow down with the primary meta region scan if (e.getEnvironment().getRegion().getRegionInfo().isMetaRegion() && (replicaId == 0)) { if (slowDownPrimaryMetaScan) { LOG.info("Scan with primary meta region, slow down a bit"); try { Thread.sleep(META_SCAN_TIMEOUT_IN_MILLISEC - 50); } catch (InterruptedException ie) { // Ingore } } // Fail for the primary replica if (throwException) { LOG.info("Scan, throw Region Server Stopped Exceptoin for replica " + e.getEnvironment() .getRegion().getRegionInfo()); throw new RegionServerStoppedException("Server " + e.getEnvironment().getServerName() + " not running"); } else { LOG.info("Scan, We're replica region " + replicaId); } } else { LOG.info("Scan, We're replica region " + replicaId); } }
@Override public AdminProtos.AdminService.BlockingInterface getAdmin(ServerName serverName) throws IOException { if (isDeadServer(serverName)) { throw new RegionServerStoppedException(serverName + " is dead."); } String key = getStubKey(AdminProtos.AdminService.BlockingInterface.class.getName(), serverName, this.hostnamesCanChange); return (AdminProtos.AdminService.BlockingInterface) computeIfAbsentEx(stubs, key, () -> { BlockingRpcChannel channel = this.rpcClient.createBlockingRpcChannel(serverName, user, rpcTimeout); return AdminProtos.AdminService.newBlockingStub(channel); }); }
@Override public BlockingInterface getClient(ServerName serverName) throws IOException { if (isDeadServer(serverName)) { throw new RegionServerStoppedException(serverName + " is dead."); } String key = getStubKey(ClientProtos.ClientService.BlockingInterface.class.getName(), serverName, this.hostnamesCanChange); return (ClientProtos.ClientService.BlockingInterface) computeIfAbsentEx(stubs, key, () -> { BlockingRpcChannel channel = this.rpcClient.createBlockingRpcChannel(serverName, user, rpcTimeout); return ClientProtos.ClientService.newBlockingStub(channel); }); }
RpcTimeoutConnection(Configuration conf, ExecutorService pool, User user) throws IOException { super(conf, pool, user); // Mock up my stub so an exists call -- which turns into a get -- throws an exception this.stub = Mockito.mock(ClientService.BlockingInterface.class); try { Mockito.when(stub.get((RpcController)Mockito.any(), (ClientProtos.GetRequest)Mockito.any())). thenThrow(new ServiceException(new RegionServerStoppedException("From Mockito"))); } catch (ServiceException e) { throw new IOException(e); } }
/** * @see java.util.concurrent.Callable#call() */ public Result [] call() throws IOException { if (scannerId != -1L && closed) { close(); } else if (scannerId == -1L && !closed) { this.scannerId = openScanner(); } else { Result [] rrs = null; try { rrs = server.next(scannerId, caching); } catch (IOException e) { IOException ioe = null; if (e instanceof RemoteException) { ioe = RemoteExceptionHandler.decodeRemoteException((RemoteException)e); } if (ioe == null) throw new IOException(e); if (ioe instanceof NotServingRegionException) { // Throw a DNRE so that we break out of cycle of calling NSRE // when what we need is to open scanner against new location. // Attach NSRE to signal client that it needs to resetup scanner. throw new DoNotRetryIOException("Reset scanner", ioe); } else if (ioe instanceof RegionServerStoppedException) { // Throw a DNRE so that we break out of cycle of calling RSSE // when what we need is to open scanner against new location. // Attach RSSE to signal client that it needs to resetup scanner. throw new DoNotRetryIOException("Reset scanner", ioe); } else { // The outer layers will retry throw ioe; } } return rrs; } return null; }
@Ignore @Test (expected = RegionServerStoppedException.class) public void testClusterStatus() throws Exception { TableName tn = TableName.valueOf("testClusterStatus"); byte[] cf = "cf".getBytes(); byte[] rk = "rk1".getBytes(); JVMClusterUtil.RegionServerThread rs = TEST_UTIL.getHBaseCluster().startRegionServer(); rs.waitForServerOnline(); final ServerName sn = rs.getRegionServer().getServerName(); HTable t = TEST_UTIL.createTable(tn, cf); TEST_UTIL.waitTableAvailable(tn); while(TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager(). getRegionStates().isRegionsInTransition()){ Thread.sleep(1); } final HConnectionImplementation hci = (HConnectionImplementation)t.getConnection(); while (t.getRegionLocation(rk).getPort() != sn.getPort()){ TEST_UTIL.getHBaseAdmin().move(t.getRegionLocation(rk).getRegionInfo(). getEncodedNameAsBytes(), Bytes.toBytes(sn.toString())); while(TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager(). getRegionStates().isRegionsInTransition()){ Thread.sleep(1); } hci.clearRegionCache(tn); } Assert.assertNotNull(hci.clusterStatusListener); TEST_UTIL.assertRegionOnServer(t.getRegionLocation(rk).getRegionInfo(), sn, 20000); Put p1 = new Put(rk); p1.add(cf, "qual".getBytes(), "val".getBytes()); t.put(p1); rs.getRegionServer().abort("I'm dead"); // We want the status to be updated. That's a least 10 second TEST_UTIL.waitFor(40000, 1000, true, new Waiter.Predicate<Exception>() { @Override public boolean evaluate() throws Exception { return TEST_UTIL.getHBaseCluster().getMaster().getServerManager(). getDeadServers().isDeadServer(sn); } }); TEST_UTIL.waitFor(40000, 1000, true, new Waiter.Predicate<Exception>() { @Override public boolean evaluate() throws Exception { return hci.clusterStatusListener.isDeadServer(sn); } }); t.close(); hci.getClient(sn); // will throw an exception: RegionServerStoppedException }
/** * @see java.util.concurrent.Callable#call() */ public Result[] call() throws IOException { if (scannerId != -1L && closed) { close(); } else if (scannerId == -1L && !closed) { this.scannerId = openScanner(); } else { Result[] rrs = null; try { incRPCcallsMetrics(); long timestamp = System.currentTimeMillis(); rrs = server.next(scannerId, caching); if (logScannerActivity) { long now = System.currentTimeMillis(); if (now - timestamp > logCutOffLatency) { int rows = rrs == null ? 0 : rrs.length; LOG.info("Took " + (now - timestamp) + "ms to fetch " + rows + " rows from scanner=" + scannerId); } } updateResultsMetrics(rrs); } catch (IOException e) { if (logScannerActivity) { LOG.info("Got exception in fetching from scanner=" + scannerId, e); } IOException ioe = null; if (e instanceof RemoteException) { ioe = RemoteExceptionHandler.decodeRemoteException((RemoteException) e); } if (ioe == null) throw new IOException(e); if (logScannerActivity && (ioe instanceof UnknownScannerException)) { try { HRegionLocation location = connection.relocateRegion(tableName, scan.getStartRow()); LOG.info("Scanner=" + scannerId + " expired, current region location is " + location.toString() + " ip:" + location.getServerAddress().getBindAddress()); } catch (Throwable t) { LOG.info("Failed to relocate region", t); } } if (ioe instanceof NotServingRegionException) { // Throw a DNRE so that we break out of cycle of calling NSRE // when what we need is to open scanner against new location. // Attach NSRE to signal client that it needs to resetup scanner. if (this.scanMetrics != null) { this.scanMetrics.countOfNSRE.inc(); } throw new DoNotRetryIOException("Reset scanner", ioe); } else if (ioe instanceof RegionServerStoppedException) { // Throw a DNRE so that we break out of cycle of calling RSSE // when what we need is to open scanner against new location. // Attach RSSE to signal client that it needs to resetup scanner. throw new DoNotRetryIOException("Reset scanner", ioe); } else { // The outer layers will retry throw ioe; } } return rrs; } return null; }
@Ignore @Test(expected = RegionServerStoppedException.class) public void testClusterStatus() throws Exception { if (!isJavaOk){ // This test requires jdk 1.7+ throw new RegionServerStoppedException("as expected by the test..."); } TableName tn = TableName.valueOf("testClusterStatus"); byte[] cf = "cf".getBytes(); byte[] rk = "rk1".getBytes(); JVMClusterUtil.RegionServerThread rs = TEST_UTIL.getHBaseCluster().startRegionServer(); rs.waitForServerOnline(); final ServerName sn = rs.getRegionServer().getServerName(); HTable t = TEST_UTIL.createTable(tn, cf); TEST_UTIL.waitTableAvailable(tn); while(TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager(). getRegionStates().isRegionsInTransition()){ Thread.sleep(1); } final HConnectionImplementation hci = (HConnectionImplementation)t.getConnection(); while (t.getRegionLocation(rk).getPort() != sn.getPort()){ TEST_UTIL.getHBaseAdmin().move(t.getRegionLocation(rk).getRegionInfo(). getEncodedNameAsBytes(), Bytes.toBytes(sn.toString())); while(TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager(). getRegionStates().isRegionsInTransition()){ Thread.sleep(1); } hci.clearRegionCache(tn); } Assert.assertNotNull(hci.clusterStatusListener); TEST_UTIL.assertRegionOnServer(t.getRegionLocation(rk).getRegionInfo(), sn, 20000); Put p1 = new Put(rk); p1.add(cf, "qual".getBytes(), "val".getBytes()); t.put(p1); rs.getRegionServer().abort("I'm dead"); // We want the status to be updated. That's a least 10 second TEST_UTIL.waitFor(40000, 1000, true, new Waiter.Predicate<Exception>() { @Override public boolean evaluate() throws Exception { return TEST_UTIL.getHBaseCluster().getMaster().getServerManager(). getDeadServers().isDeadServer(sn); } }); TEST_UTIL.waitFor(40000, 1000, true, new Waiter.Predicate<Exception>() { @Override public boolean evaluate() throws Exception { return hci.clusterStatusListener.isDeadServer(sn); } }); t.close(); hci.getClient(sn); // will throw an exception: RegionServerStoppedException }
@Ignore ("Fails in IDEs: HBASE-9042") @Test(expected = RegionServerStoppedException.class) public void testClusterStatus() throws Exception { TableName tn = TableName.valueOf("testClusterStatus"); byte[] cf = "cf".getBytes(); byte[] rk = "rk1".getBytes(); JVMClusterUtil.RegionServerThread rs = TEST_UTIL.getHBaseCluster().startRegionServer(); rs.waitForServerOnline(); final ServerName sn = rs.getRegionServer().getServerName(); HTable t = TEST_UTIL.createTable(tn, cf); TEST_UTIL.waitTableAvailable(tn.getName()); while(TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager(). getRegionStates().isRegionsInTransition()){ Thread.sleep(1); } final HConnectionImplementation hci = (HConnectionImplementation)t.getConnection(); while (t.getRegionLocation(rk).getPort() != sn.getPort()){ TEST_UTIL.getHBaseAdmin().move(t.getRegionLocation(rk).getRegionInfo(). getEncodedNameAsBytes(), Bytes.toBytes(sn.toString())); while(TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager(). getRegionStates().isRegionsInTransition()){ Thread.sleep(1); } hci.clearRegionCache(tn); } Assert.assertNotNull(hci.clusterStatusListener); TEST_UTIL.assertRegionOnServer(t.getRegionLocation(rk).getRegionInfo(), sn, 20000); Put p1 = new Put(rk); p1.add(cf, "qual".getBytes(), "val".getBytes()); t.put(p1); rs.getRegionServer().abort("I'm dead"); // We want the status to be updated. That's a least 10 second TEST_UTIL.waitFor(40000, 1000, true, new Waiter.Predicate<Exception>() { @Override public boolean evaluate() throws Exception { return TEST_UTIL.getHBaseCluster().getMaster().getServerManager(). getDeadServers().isDeadServer(sn); } }); TEST_UTIL.waitFor(40000, 1000, true, new Waiter.Predicate<Exception>() { @Override public boolean evaluate() throws Exception { return hci.clusterStatusListener.isDeadServer(sn); } }); t.close(); hci.getClient(sn); // will throw an exception: RegionServerStoppedException }
