@Override public void initialize(MasterServices master, MetricsMaster metricsMaster) throws IOException, UnsupportedOperationException { this.master = master; this.done = false; // setup the default procedure coordinator String name = master.getServerName().toString(); // get the configuration for the coordinator Configuration conf = master.getConfiguration(); long wakeFrequency = conf.getInt(BACKUP_WAKE_MILLIS_KEY, BACKUP_WAKE_MILLIS_DEFAULT); long timeoutMillis = conf.getLong(BACKUP_TIMEOUT_MILLIS_KEY,BACKUP_TIMEOUT_MILLIS_DEFAULT); int opThreads = conf.getInt(BACKUP_POOL_THREAD_NUMBER_KEY, BACKUP_POOL_THREAD_NUMBER_DEFAULT); // setup the default procedure coordinator ThreadPoolExecutor tpool = ProcedureCoordinator.defaultPool(name, opThreads); CoordinatedStateManager coordManager = new ZkCoordinatedStateManager(master); ProcedureCoordinatorRpcs comms = coordManager.getProcedureCoordinatorRpcs(getProcedureSignature(), name); this.coordinator = new ProcedureCoordinator(comms, tpool, timeoutMillis, wakeFrequency); }
@Override public void initialize(RegionServerServices rss) throws KeeperException { this.rss = rss; if (!BackupManager.isBackupEnabled(rss.getConfiguration())) { LOG.warn("Backup is not enabled. Check your " + BackupRestoreConstants.BACKUP_ENABLE_KEY + " setting"); return; } CoordinatedStateManager coordManager = new ZkCoordinatedStateManager(rss); this.memberRpcs = coordManager .getProcedureMemberRpcs(LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_SIGNATURE); // read in the backup handler configuration properties Configuration conf = rss.getConfiguration(); long keepAlive = conf.getLong(BACKUP_TIMEOUT_MILLIS_KEY, BACKUP_TIMEOUT_MILLIS_DEFAULT); int opThreads = conf.getInt(BACKUP_REQUEST_THREADS_KEY, BACKUP_REQUEST_THREADS_DEFAULT); // create the actual cohort member ThreadPoolExecutor pool = ProcedureMember.defaultPool(rss.getServerName().toString(), opThreads, keepAlive); this.member = new ProcedureMember(memberRpcs, pool, new BackupSubprocedureBuilder()); }
/** * Creates consensus provider from the given configuration. * @param conf Configuration * @return Implementation of {@link CoordinatedStateManager} */ public static CoordinatedStateManager getCoordinatedStateManager(Configuration conf) { Class<? extends CoordinatedStateManager> coordinatedStateMgrKlass = conf.getClass(HConstants.HBASE_COORDINATED_STATE_MANAGER_CLASS, ZkCoordinatedStateManager.class, CoordinatedStateManager.class); return ReflectionUtils.newInstance(coordinatedStateMgrKlass, conf); }
/** * Test if close region can handle ZK closing node version mismatch * @throws IOException * @throws NodeExistsException * @throws KeeperException * @throws DeserializationException */ @Test public void testZKClosingNodeVersionMismatch() throws IOException, NodeExistsException, KeeperException, DeserializationException { final Server server = new MockServer(HTU); final RegionServerServices rss = HTU.createMockRegionServerService(); HTableDescriptor htd = TEST_HTD; final HRegionInfo hri = TEST_HRI; ZkCoordinatedStateManager coordinationProvider = new ZkCoordinatedStateManager(); coordinationProvider.initialize(server); coordinationProvider.start(); // open a region first so that it can be closed later OpenRegion(server, rss, htd, hri, coordinationProvider.getOpenRegionCoordination()); // close the region // Create it CLOSING, which is what Master set before sending CLOSE RPC int versionOfClosingNode = ZKAssign.createNodeClosing(server.getZooKeeper(), hri, server.getServerName()); // The CloseRegionHandler will validate the expected version // Given it is set to invalid versionOfClosingNode+1, // CloseRegionHandler should be M_ZK_REGION_CLOSING ZkCloseRegionCoordination.ZkCloseRegionDetails zkCrd = new ZkCloseRegionCoordination.ZkCloseRegionDetails(); zkCrd.setPublishStatusInZk(true); zkCrd.setExpectedVersion(versionOfClosingNode+1); CloseRegionHandler handler = new CloseRegionHandler(server, rss, hri, false, coordinationProvider.getCloseRegionCoordination(), zkCrd); handler.process(); // Handler should remain in M_ZK_REGION_CLOSING RegionTransition rt = RegionTransition.parseFrom(ZKAssign.getData(server.getZooKeeper(), hri.getEncodedName())); assertTrue(rt.getEventType().equals(EventType.M_ZK_REGION_CLOSING )); }
/** * Test if the region can be closed properly * @throws IOException * @throws NodeExistsException * @throws KeeperException * @throws org.apache.hadoop.hbase.exceptions.DeserializationException */ @Test public void testCloseRegion() throws IOException, NodeExistsException, KeeperException, DeserializationException { final Server server = new MockServer(HTU); final RegionServerServices rss = HTU.createMockRegionServerService(); HTableDescriptor htd = TEST_HTD; HRegionInfo hri = TEST_HRI; ZkCoordinatedStateManager coordinationProvider = new ZkCoordinatedStateManager(); coordinationProvider.initialize(server); coordinationProvider.start(); // open a region first so that it can be closed later OpenRegion(server, rss, htd, hri, coordinationProvider.getOpenRegionCoordination()); // close the region // Create it CLOSING, which is what Master set before sending CLOSE RPC int versionOfClosingNode = ZKAssign.createNodeClosing(server.getZooKeeper(), hri, server.getServerName()); // The CloseRegionHandler will validate the expected version // Given it is set to correct versionOfClosingNode, // CloseRegionHandlerit should be RS_ZK_REGION_CLOSED ZkCloseRegionCoordination.ZkCloseRegionDetails zkCrd = new ZkCloseRegionCoordination.ZkCloseRegionDetails(); zkCrd.setPublishStatusInZk(true); zkCrd.setExpectedVersion(versionOfClosingNode); CloseRegionHandler handler = new CloseRegionHandler(server, rss, hri, false, coordinationProvider.getCloseRegionCoordination(), zkCrd); handler.process(); // Handler should have transitioned it to RS_ZK_REGION_CLOSED RegionTransition rt = RegionTransition.parseFrom( ZKAssign.getData(server.getZooKeeper(), hri.getEncodedName())); assertTrue(rt.getEventType().equals(EventType.RS_ZK_REGION_CLOSED)); }
@Test public void testFailedOpenRegion() throws Exception { Server server = new MockServer(HTU); RegionServerServices rsServices = HTU.createMockRegionServerService(); // Create it OFFLINE, which is what it expects ZKAssign.createNodeOffline(server.getZooKeeper(), TEST_HRI, server.getServerName()); ZkCoordinatedStateManager csm = new ZkCoordinatedStateManager(); csm.initialize(server); csm.start(); ZkOpenRegionCoordination.ZkOpenRegionDetails zkCrd = new ZkOpenRegionCoordination.ZkOpenRegionDetails(); zkCrd.setServerName(server.getServerName()); // Create the handler OpenRegionHandler handler = new OpenRegionHandler(server, rsServices, TEST_HRI, TEST_HTD, -1, csm.getOpenRegionCoordination(), zkCrd) { @Override HRegion openRegion() { // Fake failure of opening a region due to an IOE, which is caught return null; } }; rsServices.getRegionsInTransitionInRS().put( TEST_HRI.getEncodedNameAsBytes(), Boolean.TRUE); handler.process(); // Handler should have transitioned it to FAILED_OPEN RegionTransition rt = RegionTransition.parseFrom( ZKAssign.getData(server.getZooKeeper(), TEST_HRI.getEncodedName())); assertEquals(EventType.RS_ZK_REGION_FAILED_OPEN, rt.getEventType()); }
@Test public void testFailedUpdateMeta() throws Exception { Server server = new MockServer(HTU); RegionServerServices rsServices = HTU.createMockRegionServerService(); // Create it OFFLINE, which is what it expects ZKAssign.createNodeOffline(server.getZooKeeper(), TEST_HRI, server.getServerName()); // Create the handler ZkCoordinatedStateManager csm = new ZkCoordinatedStateManager(); csm.initialize(server); csm.start(); ZkOpenRegionCoordination.ZkOpenRegionDetails zkCrd = new ZkOpenRegionCoordination.ZkOpenRegionDetails(); zkCrd.setServerName(server.getServerName()); OpenRegionHandler handler = new OpenRegionHandler(server, rsServices, TEST_HRI, TEST_HTD, -1, csm.getOpenRegionCoordination(), zkCrd) { @Override boolean updateMeta(final HRegion r, long masterSystemTime) { // Fake failure of updating META return false; } }; rsServices.getRegionsInTransitionInRS().put( TEST_HRI.getEncodedNameAsBytes(), Boolean.TRUE); handler.process(); // Handler should have transitioned it to FAILED_OPEN RegionTransition rt = RegionTransition.parseFrom( ZKAssign.getData(server.getZooKeeper(), TEST_HRI.getEncodedName())); assertEquals(EventType.RS_ZK_REGION_FAILED_OPEN, rt.getEventType()); }
@Test public void testTransitionToFailedOpenEvenIfCleanupFails() throws Exception { Server server = new MockServer(HTU); RegionServerServices rsServices = HTU.createMockRegionServerService(); // Create it OFFLINE, which is what it expects ZKAssign.createNodeOffline(server.getZooKeeper(), TEST_HRI, server.getServerName()); // Create the handler ZkCoordinatedStateManager csm = new ZkCoordinatedStateManager(); csm.initialize(server); csm.start(); ZkOpenRegionCoordination.ZkOpenRegionDetails zkCrd = new ZkOpenRegionCoordination.ZkOpenRegionDetails(); zkCrd.setServerName(server.getServerName()); OpenRegionHandler handler = new OpenRegionHandler(server, rsServices, TEST_HRI, TEST_HTD, -1, csm.getOpenRegionCoordination(), zkCrd) { @Override boolean updateMeta(HRegion r, long masterSystemTime) { return false; }; @Override void cleanupFailedOpen(HRegion region) throws IOException { throw new IOException("FileSystem got closed."); } }; rsServices.getRegionsInTransitionInRS().put(TEST_HRI.getEncodedNameAsBytes(), Boolean.TRUE); try { handler.process(); } catch (Exception e) { // Ignore the IOException that we have thrown from cleanupFailedOpen } RegionTransition rt = RegionTransition.parseFrom(ZKAssign.getData(server.getZooKeeper(), TEST_HRI.getEncodedName())); assertEquals(EventType.RS_ZK_REGION_FAILED_OPEN, rt.getEventType()); }
@Test public void testTransitionToFailedOpenFromOffline() throws Exception { Server server = new MockServer(HTU); RegionServerServices rsServices = HTU.createMockRegionServerService(server.getServerName()); // Create it OFFLINE, which is what it expects ZKAssign.createNodeOffline(server.getZooKeeper(), TEST_HRI, server.getServerName()); // Create the handler ZkCoordinatedStateManager csm = new ZkCoordinatedStateManager(); csm.initialize(server); csm.start(); ZkOpenRegionCoordination.ZkOpenRegionDetails zkCrd = new ZkOpenRegionCoordination.ZkOpenRegionDetails(); zkCrd.setServerName(server.getServerName()); ZkOpenRegionCoordination openRegionCoordination = new ZkOpenRegionCoordination(csm, server.getZooKeeper()) { @Override public boolean transitionFromOfflineToOpening(HRegionInfo regionInfo, OpenRegionDetails ord) { return false; } }; OpenRegionHandler handler = new OpenRegionHandler(server, rsServices, TEST_HRI, TEST_HTD, -1, openRegionCoordination, zkCrd); rsServices.getRegionsInTransitionInRS().put(TEST_HRI.getEncodedNameAsBytes(), Boolean.TRUE); handler.process(); RegionTransition rt = RegionTransition.parseFrom(ZKAssign.getData(server.getZooKeeper(), TEST_HRI.getEncodedName())); assertEquals(EventType.RS_ZK_REGION_FAILED_OPEN, rt.getEventType()); }
/** * Test the scenario when the master is in failover and trying to process a * region which is in Opening state on a dead RS. Master will force offline the * region and put it in transition. AM relies on SSH to reassign it. */ @Test(timeout = 60000) public void testRegionInOpeningStateOnDeadRSWhileMasterFailover() throws IOException, KeeperException, ServiceException, CoordinatedStateException, InterruptedException { AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager( this.server, this.serverManager); ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A); int version = ZKAssign.getVersion(this.watcher, REGIONINFO); ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE, EventType.RS_ZK_REGION_OPENING, version); RegionTransition rt = RegionTransition.createRegionTransition(EventType.RS_ZK_REGION_OPENING, REGIONINFO.getRegionName(), SERVERNAME_A, HConstants.EMPTY_BYTE_ARRAY); version = ZKAssign.getVersion(this.watcher, REGIONINFO); Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(false); am.getRegionStates().logSplit(SERVERNAME_A); // Assume log splitting is done am.getRegionStates().createRegionState(REGIONINFO); am.gate.set(false); BaseCoordinatedStateManager cp = new ZkCoordinatedStateManager(); cp.initialize(server); cp.start(); OpenRegionCoordination orc = cp.getOpenRegionCoordination(); ZkOpenRegionCoordination.ZkOpenRegionDetails zkOrd = new ZkOpenRegionCoordination.ZkOpenRegionDetails(); zkOrd.setServerName(server.getServerName()); zkOrd.setVersion(version); assertFalse(am.processRegionsInTransition(rt, REGIONINFO, orc, zkOrd)); am.getTableStateManager().setTableState(REGIONINFO.getTable(), Table.State.ENABLED); processServerShutdownHandler(am, false); // Waiting for the assignment to get completed. while (!am.gate.get()) { Thread.sleep(10); } assertTrue("The region should be assigned immediately.", null != am.regionPlans.get(REGIONINFO .getEncodedName())); am.shutdown(); }
@Test public void testFailedOpenRegion() throws Exception { Server server = new MockServer(HTU); RegionServerServices rsServices = HTU.createMockRegionServerService(); // Create it OFFLINE, which is what it expects ZKAssign.createNodeOffline(server.getZooKeeper(), TEST_HRI, server.getServerName()); ZkCoordinatedStateManager csm = new ZkCoordinatedStateManager(); csm.initialize(server); csm.start(); ZkOpenRegionCoordination.ZkOpenRegionDetails zkCrd = new ZkOpenRegionCoordination.ZkOpenRegionDetails(); zkCrd.setServerName(server.getServerName()); // Create the handler OpenRegionHandler handler = new OpenRegionHandler(server, rsServices, TEST_HRI, TEST_HTD, csm.getOpenRegionCoordination(), zkCrd) { @Override HRegion openRegion() { // Fake failure of opening a region due to an IOE, which is caught return null; } }; rsServices.getRegionsInTransitionInRS().put( TEST_HRI.getEncodedNameAsBytes(), Boolean.TRUE); handler.process(); // Handler should have transitioned it to FAILED_OPEN RegionTransition rt = RegionTransition.parseFrom( ZKAssign.getData(server.getZooKeeper(), TEST_HRI.getEncodedName())); assertEquals(EventType.RS_ZK_REGION_FAILED_OPEN, rt.getEventType()); }
@Test public void testFailedUpdateMeta() throws Exception { Server server = new MockServer(HTU); RegionServerServices rsServices = HTU.createMockRegionServerService(); // Create it OFFLINE, which is what it expects ZKAssign.createNodeOffline(server.getZooKeeper(), TEST_HRI, server.getServerName()); // Create the handler ZkCoordinatedStateManager csm = new ZkCoordinatedStateManager(); csm.initialize(server); csm.start(); ZkOpenRegionCoordination.ZkOpenRegionDetails zkCrd = new ZkOpenRegionCoordination.ZkOpenRegionDetails(); zkCrd.setServerName(server.getServerName()); OpenRegionHandler handler = new OpenRegionHandler(server, rsServices, TEST_HRI, TEST_HTD, csm.getOpenRegionCoordination(), zkCrd) { @Override boolean updateMeta(final HRegion r) { // Fake failure of updating META return false; } }; rsServices.getRegionsInTransitionInRS().put( TEST_HRI.getEncodedNameAsBytes(), Boolean.TRUE); handler.process(); // Handler should have transitioned it to FAILED_OPEN RegionTransition rt = RegionTransition.parseFrom( ZKAssign.getData(server.getZooKeeper(), TEST_HRI.getEncodedName())); assertEquals(EventType.RS_ZK_REGION_FAILED_OPEN, rt.getEventType()); }
@Test public void testTransitionToFailedOpenEvenIfCleanupFails() throws Exception { Server server = new MockServer(HTU); RegionServerServices rsServices = HTU.createMockRegionServerService(); // Create it OFFLINE, which is what it expects ZKAssign.createNodeOffline(server.getZooKeeper(), TEST_HRI, server.getServerName()); // Create the handler ZkCoordinatedStateManager csm = new ZkCoordinatedStateManager(); csm.initialize(server); csm.start(); ZkOpenRegionCoordination.ZkOpenRegionDetails zkCrd = new ZkOpenRegionCoordination.ZkOpenRegionDetails(); zkCrd.setServerName(server.getServerName()); OpenRegionHandler handler = new OpenRegionHandler(server, rsServices, TEST_HRI, TEST_HTD, csm.getOpenRegionCoordination(), zkCrd) { @Override boolean updateMeta(HRegion r) { return false; }; @Override void cleanupFailedOpen(HRegion region) throws IOException { throw new IOException("FileSystem got closed."); } }; rsServices.getRegionsInTransitionInRS().put(TEST_HRI.getEncodedNameAsBytes(), Boolean.TRUE); try { handler.process(); } catch (Exception e) { // Ignore the IOException that we have thrown from cleanupFailedOpen } RegionTransition rt = RegionTransition.parseFrom(ZKAssign.getData(server.getZooKeeper(), TEST_HRI.getEncodedName())); assertEquals(EventType.RS_ZK_REGION_FAILED_OPEN, rt.getEventType()); }
@Test public void testTransitionToFailedOpenFromOffline() throws Exception { Server server = new MockServer(HTU); RegionServerServices rsServices = HTU.createMockRegionServerService(server.getServerName()); // Create it OFFLINE, which is what it expects ZKAssign.createNodeOffline(server.getZooKeeper(), TEST_HRI, server.getServerName()); // Create the handler ZkCoordinatedStateManager csm = new ZkCoordinatedStateManager(); csm.initialize(server); csm.start(); ZkOpenRegionCoordination.ZkOpenRegionDetails zkCrd = new ZkOpenRegionCoordination.ZkOpenRegionDetails(); zkCrd.setServerName(server.getServerName()); ZkOpenRegionCoordination openRegionCoordination = new ZkOpenRegionCoordination(csm, server.getZooKeeper()) { @Override public boolean transitionFromOfflineToOpening(HRegionInfo regionInfo, OpenRegionDetails ord) { return false; } }; OpenRegionHandler handler = new OpenRegionHandler(server, rsServices, TEST_HRI, TEST_HTD, openRegionCoordination, zkCrd); rsServices.getRegionsInTransitionInRS().put(TEST_HRI.getEncodedNameAsBytes(), Boolean.TRUE); handler.process(); RegionTransition rt = RegionTransition.parseFrom(ZKAssign.getData(server.getZooKeeper(), TEST_HRI.getEncodedName())); assertEquals(EventType.RS_ZK_REGION_FAILED_OPEN, rt.getEventType()); }
@Test public void testShouldNotCompeleteOpenedRegionSuccessfullyIfVersionMismatches() throws Exception { HRegion region = null; try { int testIndex = 0; TEST_UTIL.startMiniZKCluster(); final Server server = new MockServer(TEST_UTIL); HTableDescriptor htd = new HTableDescriptor( TableName.valueOf("testShouldNotCompeleteOpenedRegionSuccessfullyIfVersionMismatches")); HRegionInfo hri = new HRegionInfo(htd.getTableName(), Bytes.toBytes(testIndex), Bytes.toBytes(testIndex + 1)); region = HRegion.createHRegion(hri, TEST_UTIL.getDataTestDir(), TEST_UTIL.getConfiguration(), htd); assertNotNull(region); AssignmentManager am = Mockito.mock(AssignmentManager.class); RegionStates rsm = Mockito.mock(RegionStates.class); Mockito.doReturn(rsm).when(am).getRegionStates(); when(rsm.isRegionInTransition(hri)).thenReturn(false); when(rsm.getRegionState(hri)).thenReturn( new RegionState(region.getRegionInfo(), RegionState.State.OPEN, System.currentTimeMillis(), server.getServerName())); // create a node with OPENED state zkw = HBaseTestingUtility.createAndForceNodeToOpenedState(TEST_UTIL, region, server.getServerName()); when(am.getTableStateManager()).thenReturn(new ZKTableStateManager(zkw)); Stat stat = new Stat(); String nodeName = ZKAssign.getNodeName(zkw, region.getRegionInfo() .getEncodedName()); ZKUtil.getDataAndWatch(zkw, nodeName, stat); // use the version for the OpenedRegionHandler BaseCoordinatedStateManager csm = new ZkCoordinatedStateManager(); csm.initialize(server); csm.start(); OpenRegionCoordination orc = csm.getOpenRegionCoordination(); ZkOpenRegionCoordination.ZkOpenRegionDetails zkOrd = new ZkOpenRegionCoordination.ZkOpenRegionDetails(); zkOrd.setServerName(server.getServerName()); zkOrd.setVersion(stat.getVersion()); OpenedRegionHandler handler = new OpenedRegionHandler(server, am, region .getRegionInfo(), orc, zkOrd); // Once again overwrite the same znode so that the version changes. ZKAssign.transitionNode(zkw, region.getRegionInfo(), server .getServerName(), EventType.RS_ZK_REGION_OPENED, EventType.RS_ZK_REGION_OPENED, stat.getVersion()); // Should not invoke assignmentmanager.regionOnline. If it is // invoked as per current mocking it will throw null pointer exception. boolean expectedException = false; try { handler.process(); } catch (Exception e) { expectedException = true; } assertFalse("The process method should not throw any exception.", expectedException); List<String> znodes = ZKUtil.listChildrenAndWatchForNewChildren(zkw, zkw.assignmentZNode); String regionName = znodes.get(0); assertEquals("The region should not be opened successfully.", regionName, region.getRegionInfo().getEncodedName()); } finally { HRegion.closeHRegion(region); TEST_UTIL.shutdownMiniZKCluster(); } }
/** * Test concurrent updates to meta when meta is not on master * @throws Exception */ @Test(timeout = 30000) public void testUpdatesRemoteMeta() throws Exception { // Not for zk less assignment if (conf.getBoolean("hbase.assignment.usezk", true)) { return; } conf.setInt("hbase.regionstatestore.meta.connection", 3); final RegionStateStore rss = new RegionStateStore(new MyRegionServer(conf, new ZkCoordinatedStateManager())); rss.start(); // Create 10 threads and make each do 10 puts related to region state update Thread[] th = new Thread[10]; List<String> nameList = new ArrayList<String>(); List<TableName> tableNameList = new ArrayList<TableName>(); for (int i = 0; i < th.length; i++) { th[i] = new Thread() { @Override public void run() { HRegionInfo[] hri = new HRegionInfo[10]; ServerName serverName = ServerName.valueOf("dummyhost", 1000, 1234); for (int i = 0; i < 10; i++) { hri[i] = new HRegionInfo(TableName.valueOf(Thread.currentThread().getName() + "_" + i)); RegionState newState = new RegionState(hri[i], RegionState.State.OPEN, serverName); RegionState oldState = new RegionState(hri[i], RegionState.State.PENDING_OPEN, serverName); rss.updateRegionState(1, newState, oldState); } } }; th[i].start(); nameList.add(th[i].getName()); } for (int i = 0; i < th.length; i++) { th[i].join(); } // Add all the expected table names in meta to tableNameList for (String name : nameList) { for (int i = 0; i < 10; i++) { tableNameList.add(TableName.valueOf(name + "_" + i)); } } List<Result> metaRows = MetaTableAccessor.fullScanOfMeta(admin.getConnection()); int count = 0; // Check all 100 rows are in meta for (Result result : metaRows) { if (tableNameList.contains(HRegionInfo.getTable(result.getRow()))) { count++; if (count == 100) { break; } } } assertTrue(count == 100); rss.stop(); }
/** * Test that if we do a close while opening it stops the opening. */ @Test(timeout = 60000) public void testCancelOpeningWithoutZK() throws Exception { // We close closeRegionNoZK(); checkRegionIsClosed(HTU, getRS(), hri); // Let do the initial steps, without having a handler ZKAssign.createNodeOffline(HTU.getZooKeeperWatcher(), hri, getRS().getServerName()); getRS().getRegionsInTransitionInRS().put(hri.getEncodedNameAsBytes(), Boolean.TRUE); // That's a close without ZK. AdminProtos.CloseRegionRequest crr = RequestConverter.buildCloseRegionRequest(getRS().getServerName(), regionName, false); try { getRS().rpcServices.closeRegion(null, crr); Assert.assertTrue(false); } catch (ServiceException expected) { } // The state in RIT should have changed to close Assert.assertEquals(Boolean.FALSE, getRS().getRegionsInTransitionInRS().get( hri.getEncodedNameAsBytes())); // Let's start the open handler HTableDescriptor htd = getRS().tableDescriptors.get(hri.getTable()); BaseCoordinatedStateManager csm = new ZkCoordinatedStateManager(); csm.initialize(getRS()); csm.start(); ZkOpenRegionCoordination.ZkOpenRegionDetails zkCrd = new ZkOpenRegionCoordination.ZkOpenRegionDetails(); zkCrd.setServerName(getRS().getServerName()); zkCrd.setVersionOfOfflineNode(0); getRS().service.submit(new OpenRegionHandler(getRS(), getRS(), hri, htd, -1, csm.getOpenRegionCoordination(), zkCrd)); // The open handler should have removed the region from RIT but kept the region closed checkRegionIsClosed(HTU, getRS(), hri); // The open handler should have updated the value in ZK. Assert.assertTrue(ZKAssign.deleteNode( getRS().getZooKeeper(), hri.getEncodedName(), EventType.RS_ZK_REGION_FAILED_OPEN, 1) ); openRegion(HTU, getRS(), hri); }
/** * Test an open then a close with ZK. This is going to mess-up the ZK states, so * the opening will fail as well because it doesn't find what it expects in ZK. */ @Test(timeout = 60000) public void testCancelOpeningWithZK() throws Exception { // We close closeRegionNoZK(); checkRegionIsClosed(HTU, getRS(), hri); // Let do the initial steps, without having a handler getRS().getRegionsInTransitionInRS().put(hri.getEncodedNameAsBytes(), Boolean.TRUE); // That's a close without ZK. ZKAssign.createNodeClosing(HTU.getZooKeeperWatcher(), hri, getRS().getServerName()); AdminProtos.CloseRegionRequest crr = RequestConverter.buildCloseRegionRequest(getRS().getServerName(), regionName, false); try { getRS().rpcServices.closeRegion(null, crr); Assert.assertTrue(false); } catch (ServiceException expected) { Assert.assertTrue(expected.getCause() instanceof RegionAlreadyInTransitionException); } // The close should have left the ZK state as it is: it's the job the AM to delete it Assert.assertTrue(ZKAssign.deleteNode( getRS().getZooKeeper(), hri.getEncodedName(), EventType.M_ZK_REGION_CLOSING, 0) ); // The state in RIT should have changed to close Assert.assertEquals(Boolean.FALSE, getRS().getRegionsInTransitionInRS().get( hri.getEncodedNameAsBytes())); // Let's start the open handler // It should not succeed for two reasons: // 1) There is no ZK node // 2) The region in RIT was changed. // The order is more or less implementation dependant. HTableDescriptor htd = getRS().tableDescriptors.get(hri.getTable()); BaseCoordinatedStateManager csm = new ZkCoordinatedStateManager(); csm.initialize(getRS()); csm.start(); ZkOpenRegionCoordination.ZkOpenRegionDetails zkCrd = new ZkOpenRegionCoordination.ZkOpenRegionDetails(); zkCrd.setServerName(getRS().getServerName()); zkCrd.setVersionOfOfflineNode(0); getRS().service.submit(new OpenRegionHandler(getRS(), getRS(), hri, htd, -1, csm.getOpenRegionCoordination(), zkCrd)); // The open handler should have removed the region from RIT but kept the region closed checkRegionIsClosed(HTU, getRS(), hri); // We should not find any znode here. Assert.assertEquals(-1, ZKAssign.getVersion(HTU.getZooKeeperWatcher(), hri)); openRegion(HTU, getRS(), hri); }
/** * Test that if we fail a flush, abort gets set on close. * @see <a href="https://issues.apache.org/jira/browse/HBASE-4270">HBASE-4270</a> * @throws IOException * @throws NodeExistsException * @throws KeeperException */ @Test public void testFailedFlushAborts() throws IOException, NodeExistsException, KeeperException { final Server server = new MockServer(HTU, false); final RegionServerServices rss = HTU.createMockRegionServerService(); HTableDescriptor htd = TEST_HTD; final HRegionInfo hri = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_END_ROW, HConstants.EMPTY_END_ROW); HRegion region = HTU.createLocalHRegion(hri, htd); try { assertNotNull(region); // Spy on the region so can throw exception when close is called. HRegion spy = Mockito.spy(region); final boolean abort = false; Mockito.when(spy.close(abort)). thenThrow(new IOException("Mocked failed close!")); // The CloseRegionHandler will try to get an HRegion that corresponds // to the passed hri -- so insert the region into the online region Set. rss.addToOnlineRegions(spy); // Assert the Server is NOT stopped before we call close region. assertFalse(server.isStopped()); ZkCoordinatedStateManager consensusProvider = new ZkCoordinatedStateManager(); consensusProvider.initialize(server); consensusProvider.start(); ZkCloseRegionCoordination.ZkCloseRegionDetails zkCrd = new ZkCloseRegionCoordination.ZkCloseRegionDetails(); zkCrd.setPublishStatusInZk(false); zkCrd.setExpectedVersion(-1); CloseRegionHandler handler = new CloseRegionHandler(server, rss, hri, false, consensusProvider.getCloseRegionCoordination(), zkCrd); boolean throwable = false; try { handler.process(); } catch (Throwable t) { throwable = true; } finally { assertTrue(throwable); // Abort calls stop so stopped flag should be set. assertTrue(server.isStopped()); } } finally { HRegion.closeHRegion(region); } }
/** * Test the openregionhandler can deal with perceived failure of transitioning to OPENED state * due to intermittent zookeeper malfunctioning. * @see <a href="https://issues.apache.org/jira/browse/HBASE-9387">HBASE-9387</a> * @throws IOException * @throws NodeExistsException * @throws KeeperException */ @Test public void testRegionServerAbortionDueToFailureTransitioningToOpened() throws IOException, NodeExistsException, KeeperException { final Server server = new MockServer(HTU); final RegionServerServices rss = HTU.createMockRegionServerService(); HTableDescriptor htd = TEST_HTD; final HRegionInfo hri = TEST_HRI; HRegion region = HRegion.createHRegion(hri, HTU.getDataTestDir(), HTU .getConfiguration(), htd); assertNotNull(region); try { ZkCoordinatedStateManager csm = new ZkCoordinatedStateManager(); csm.initialize(server); csm.start(); ZkOpenRegionCoordination.ZkOpenRegionDetails zkCrd = new ZkOpenRegionCoordination.ZkOpenRegionDetails(); zkCrd.setServerName(server.getServerName()); ZkOpenRegionCoordination openRegionCoordination = new ZkOpenRegionCoordination(csm, server.getZooKeeper()) { @Override public boolean transitionToOpened(final HRegion r, OpenRegionDetails ord) throws IOException { // remove znode simulating intermittent zookeeper connection issue ZooKeeperWatcher zkw = server.getZooKeeper(); String node = ZKAssign.getNodeName(zkw, hri.getEncodedName()); try { ZKUtil.deleteNodeFailSilent(zkw, node); } catch (KeeperException e) { throw new RuntimeException("Ugh failed delete of " + node, e); } // then try to transition to OPENED return super.transitionToOpened(r, ord); } }; OpenRegionHandler handler = new OpenRegionHandler(server, rss, hri, htd, -1, openRegionCoordination, zkCrd); rss.getRegionsInTransitionInRS().put( hri.getEncodedNameAsBytes(), Boolean.TRUE); // Call process without first creating OFFLINE region in zk, see if // exception or just quiet return (expected). handler.process(); rss.getRegionsInTransitionInRS().put( hri.getEncodedNameAsBytes(), Boolean.TRUE); ZKAssign.createNodeOffline(server.getZooKeeper(), hri, server.getServerName()); // Call process again but this time yank the zk znode out from under it // post OPENING; again will expect it to come back w/o NPE or exception. handler.process(); } catch (IOException ioe) { } finally { HRegion.closeHRegion(region); } // Region server is expected to abort due to OpenRegionHandler perceiving transitioning // to OPENED as failed // This was corresponding to the second handler.process() call above. assertTrue("region server should have aborted", server.isAborted()); }
@Before public void before() throws ZooKeeperConnectionException, IOException { // TODO: Make generic versions of what we do below and put up in a mocking // utility class or move up into HBaseTestingUtility. // Mock a Server. Have it return a legit Configuration and ZooKeeperWatcher. // If abort is called, be sure to fail the test (don't just swallow it // silently as is mockito default). this.server = Mockito.mock(Server.class); Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("master,1,1")); Mockito.when(server.getConfiguration()).thenReturn(HTU.getConfiguration()); this.watcher = new ZooKeeperWatcher(HTU.getConfiguration(), "mockedServer", this.server, true); Mockito.when(server.getZooKeeper()).thenReturn(this.watcher); Mockito.doThrow(new RuntimeException("Aborted")). when(server).abort(Mockito.anyString(), (Throwable)Mockito.anyObject()); cp = new ZkCoordinatedStateManager(); cp.initialize(this.server); cp.start(); mtl = Mockito.mock(MetaTableLocator.class); Mockito.when(server.getCoordinatedStateManager()).thenReturn(cp); Mockito.when(server.getMetaTableLocator()).thenReturn(mtl); // Get a connection w/ mocked up common methods. this.connection = (ClusterConnection)HConnectionTestingUtility.getMockedConnection(HTU.getConfiguration()); // Make it so we can get a catalogtracker from servermanager.. .needed // down in guts of server shutdown handler. Mockito.when(server.getConnection()).thenReturn(connection); // Mock a ServerManager. Say server SERVERNAME_{A,B} are online. Also // make it so if close or open, we return 'success'. this.serverManager = Mockito.mock(ServerManager.class); Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true); Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(true); Mockito.when(this.serverManager.getDeadServers()).thenReturn(new DeadServer()); final Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>(); onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD); onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD); Mockito.when(this.serverManager.getOnlineServersList()).thenReturn( new ArrayList<ServerName>(onlineServers.keySet())); Mockito.when(this.serverManager.getOnlineServers()).thenReturn(onlineServers); List<ServerName> avServers = new ArrayList<ServerName>(); avServers.addAll(onlineServers.keySet()); Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(avServers); Mockito.when(this.serverManager.createDestinationServersList(null)).thenReturn(avServers); Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, REGIONINFO, -1)). thenReturn(true); Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_B, REGIONINFO, -1)). thenReturn(true); // Ditto on open. Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_A, REGIONINFO, -1, null)). thenReturn(RegionOpeningState.OPENED); Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_B, REGIONINFO, -1, null)). thenReturn(RegionOpeningState.OPENED); this.master = Mockito.mock(HMaster.class); Mockito.when(this.master.getServerManager()).thenReturn(serverManager); }
/** * Test that if we do a close while opening it stops the opening. */ @Test(timeout = 60000) public void testCancelOpeningWithoutZK() throws Exception { // We close closeNoZK(); checkRegionIsClosed(); // Let do the initial steps, without having a handler ZKAssign.createNodeOffline(HTU.getZooKeeperWatcher(), hri, getRS().getServerName()); getRS().getRegionsInTransitionInRS().put(hri.getEncodedNameAsBytes(), Boolean.TRUE); // That's a close without ZK. AdminProtos.CloseRegionRequest crr = RequestConverter.buildCloseRegionRequest(getRS().getServerName(), regionName, false); try { getRS().rpcServices.closeRegion(null, crr); Assert.assertTrue(false); } catch (ServiceException expected) { } // The state in RIT should have changed to close Assert.assertEquals(Boolean.FALSE, getRS().getRegionsInTransitionInRS().get( hri.getEncodedNameAsBytes())); // Let's start the open handler HTableDescriptor htd = getRS().tableDescriptors.get(hri.getTable()); BaseCoordinatedStateManager csm = new ZkCoordinatedStateManager(); csm.initialize(getRS()); csm.start(); ZkOpenRegionCoordination.ZkOpenRegionDetails zkCrd = new ZkOpenRegionCoordination.ZkOpenRegionDetails(); zkCrd.setServerName(getRS().getServerName()); zkCrd.setVersionOfOfflineNode(0); getRS().service.submit(new OpenRegionHandler(getRS(), getRS(), hri, htd, csm.getOpenRegionCoordination(), zkCrd)); // The open handler should have removed the region from RIT but kept the region closed checkRegionIsClosed(); // The open handler should have updated the value in ZK. Assert.assertTrue(ZKAssign.deleteNode( getRS().getZooKeeper(), hri.getEncodedName(), EventType.RS_ZK_REGION_FAILED_OPEN, 1) ); reopenRegion(); }
/** * Test an open then a close with ZK. This is going to mess-up the ZK states, so * the opening will fail as well because it doesn't find what it expects in ZK. */ @Test(timeout = 60000) public void testCancelOpeningWithZK() throws Exception { // We close closeNoZK(); checkRegionIsClosed(); // Let do the initial steps, without having a handler getRS().getRegionsInTransitionInRS().put(hri.getEncodedNameAsBytes(), Boolean.TRUE); // That's a close without ZK. ZKAssign.createNodeClosing(HTU.getZooKeeperWatcher(), hri, getRS().getServerName()); AdminProtos.CloseRegionRequest crr = RequestConverter.buildCloseRegionRequest(getRS().getServerName(), regionName, false); try { getRS().rpcServices.closeRegion(null, crr); Assert.assertTrue(false); } catch (ServiceException expected) { Assert.assertTrue(expected.getCause() instanceof RegionAlreadyInTransitionException); } // The close should have left the ZK state as it is: it's the job the AM to delete it Assert.assertTrue(ZKAssign.deleteNode( getRS().getZooKeeper(), hri.getEncodedName(), EventType.M_ZK_REGION_CLOSING, 0) ); // The state in RIT should have changed to close Assert.assertEquals(Boolean.FALSE, getRS().getRegionsInTransitionInRS().get( hri.getEncodedNameAsBytes())); // Let's start the open handler // It should not succeed for two reasons: // 1) There is no ZK node // 2) The region in RIT was changed. // The order is more or less implementation dependant. HTableDescriptor htd = getRS().tableDescriptors.get(hri.getTable()); BaseCoordinatedStateManager csm = new ZkCoordinatedStateManager(); csm.initialize(getRS()); csm.start(); ZkOpenRegionCoordination.ZkOpenRegionDetails zkCrd = new ZkOpenRegionCoordination.ZkOpenRegionDetails(); zkCrd.setServerName(getRS().getServerName()); zkCrd.setVersionOfOfflineNode(0); getRS().service.submit(new OpenRegionHandler(getRS(), getRS(), hri, htd, csm.getOpenRegionCoordination(), zkCrd)); // The open handler should have removed the region from RIT but kept the region closed checkRegionIsClosed(); // We should not find any znode here. Assert.assertEquals(-1, ZKAssign.getVersion(HTU.getZooKeeperWatcher(), hri)); reopenRegion(); }
/** * Test the openregionhandler can deal with perceived failure of transitioning to OPENED state * due to intermittent zookeeper malfunctioning. * @see <a href="https://issues.apache.org/jira/browse/HBASE-9387">HBASE-9387</a> * @throws IOException * @throws NodeExistsException * @throws KeeperException */ @Test public void testRegionServerAbortionDueToFailureTransitioningToOpened() throws IOException, NodeExistsException, KeeperException { final Server server = new MockServer(HTU); final RegionServerServices rss = HTU.createMockRegionServerService(); HTableDescriptor htd = TEST_HTD; final HRegionInfo hri = TEST_HRI; HRegion region = HRegion.createHRegion(hri, HTU.getDataTestDir(), HTU .getConfiguration(), htd); assertNotNull(region); try { ZkCoordinatedStateManager csm = new ZkCoordinatedStateManager(); csm.initialize(server); csm.start(); ZkOpenRegionCoordination.ZkOpenRegionDetails zkCrd = new ZkOpenRegionCoordination.ZkOpenRegionDetails(); zkCrd.setServerName(server.getServerName()); ZkOpenRegionCoordination openRegionCoordination = new ZkOpenRegionCoordination(csm, server.getZooKeeper()) { @Override public boolean transitionToOpened(final HRegion r, OpenRegionDetails ord) throws IOException { // remove znode simulating intermittent zookeeper connection issue ZooKeeperWatcher zkw = server.getZooKeeper(); String node = ZKAssign.getNodeName(zkw, hri.getEncodedName()); try { ZKUtil.deleteNodeFailSilent(zkw, node); } catch (KeeperException e) { throw new RuntimeException("Ugh failed delete of " + node, e); } // then try to transition to OPENED return super.transitionToOpened(r, ord); } }; OpenRegionHandler handler = new OpenRegionHandler(server, rss, hri, htd, openRegionCoordination, zkCrd); rss.getRegionsInTransitionInRS().put( hri.getEncodedNameAsBytes(), Boolean.TRUE); // Call process without first creating OFFLINE region in zk, see if // exception or just quiet return (expected). handler.process(); rss.getRegionsInTransitionInRS().put( hri.getEncodedNameAsBytes(), Boolean.TRUE); ZKAssign.createNodeOffline(server.getZooKeeper(), hri, server.getServerName()); // Call process again but this time yank the zk znode out from under it // post OPENING; again will expect it to come back w/o NPE or exception. handler.process(); } catch (IOException ioe) { } finally { HRegion.closeHRegion(region); } // Region server is expected to abort due to OpenRegionHandler perceiving transitioning // to OPENED as failed // This was corresponding to the second handler.process() call above. assertTrue("region server should have aborted", server.isAborted()); }
public DummyMasterServices(ZKWatcher zkw, Configuration conf) { super(conf); this.zkw = zkw; cm = new ZkCoordinatedStateManager(this); }
public DummyServer(ZKWatcher zkw, Configuration conf) { this.zkw = zkw; this.conf = conf; cm = new ZkCoordinatedStateManager(this); }