Java 类org.apache.hadoop.hbase.zookeeper.MasterAddressTracker 实例源码

/**
 * Unit tests that uses ZooKeeper but does not use the master-side methods
 * but rather acts directly on ZK.
 * @throws Exception
 */
@Test
public void testMasterAddressTrackerFromZK() throws Exception {
  // Create the master node with a dummy address
  final int infoPort = 1235;
  final ServerName sn = ServerName.valueOf("localhost", 1234, System.currentTimeMillis());
  final MasterAddressTracker addressTracker = setupMasterTracker(sn, infoPort);
  try {
    assertTrue(addressTracker.hasMaster());
    ServerName pulledAddress = addressTracker.getMasterAddress();
    assertTrue(pulledAddress.equals(sn));
    assertEquals(infoPort, addressTracker.getMasterInfoPort());
  } finally {
    assertTrue("Couldn't clean up master",
        MasterAddressTracker.deleteIfEquals(addressTracker.getWatcher(), sn.toString()));
  }
}

@Before
public void setupBasicMocks() throws IOException, ServiceException {
  rs = Mockito.mock(HRegionServer.class);
  Mockito.doReturn(HBaseConfiguration.create())
    .when(rs).getConfiguration();
  Mockito.doReturn(fakeResponse).when(rs).getServerInfo(
    (RpcController)Mockito.any(), (GetServerInfoRequest)Mockito.any());
  // Fake ZKW
  ZooKeeperWatcher zkw = Mockito.mock(ZooKeeperWatcher.class);
  Mockito.doReturn("fakequorum").when(zkw).getQuorum();
  Mockito.doReturn(zkw).when(rs).getZooKeeper();

  // Fake MasterAddressTracker
  MasterAddressTracker mat = Mockito.mock(MasterAddressTracker.class);
  Mockito.doReturn(fakeMasterAddress).when(mat).getMasterAddress();
  Mockito.doReturn(mat).when(rs).getMasterAddressTracker();

  MetricsRegionServer rms = Mockito.mock(MetricsRegionServer.class);
  Mockito.doReturn(new MetricsRegionServerWrapperStub()).when(rms).getRegionServerWrapper();
  Mockito.doReturn(rms).when(rs).getMetrics();
}

/**
 * Unit tests that uses ZooKeeper but does not use the master-side methods
 * but rather acts directly on ZK.
 * @throws Exception
 */
@Test
public void testMasterAddressTrackerFromZK() throws Exception {
  // Create the master node with a dummy address
  final int infoPort = 1235;
  final ServerName sn = ServerName.valueOf("localhost", 1234, System.currentTimeMillis());
  final MasterAddressTracker addressTracker = setupMasterTracker(sn, infoPort);
  try {
    assertTrue(addressTracker.hasMaster());
    ServerName pulledAddress = addressTracker.getMasterAddress();
    assertTrue(pulledAddress.equals(sn));
    assertEquals(infoPort, addressTracker.getMasterInfoPort());
  } finally {
    assertTrue("Couldn't clean up master",
        MasterAddressTracker.deleteIfEquals(addressTracker.getWatcher(), sn.toString()));
  }
}

@Before
public void setupBasicMocks() throws IOException, ServiceException {
  rs = Mockito.mock(HRegionServer.class);
  rpcServices = Mockito.mock(RSRpcServices.class);
  Mockito.doReturn(HBaseConfiguration.create())
    .when(rs).getConfiguration();
  Mockito.doReturn(rpcServices).when(rs).getRSRpcServices();
  Mockito.doReturn(fakeResponse).when(rpcServices).getServerInfo(
    (RpcController)Mockito.any(), (GetServerInfoRequest)Mockito.any());
  // Fake ZKW
  ZooKeeperWatcher zkw = Mockito.mock(ZooKeeperWatcher.class);
  Mockito.doReturn("fakequorum").when(zkw).getQuorum();
  Mockito.doReturn(zkw).when(rs).getZooKeeper();

  // Fake MasterAddressTracker
  MasterAddressTracker mat = Mockito.mock(MasterAddressTracker.class);
  Mockito.doReturn(fakeMasterAddress).when(mat).getMasterAddress();
  Mockito.doReturn(mat).when(rs).getMasterAddressTracker();

  MetricsRegionServer rms = Mockito.mock(MetricsRegionServer.class);
  Mockito.doReturn(new MetricsRegionServerWrapperStub()).when(rms).getRegionServerWrapper();
  Mockito.doReturn(rms).when(rs).getRegionServerMetrics();
}

@Before
public void setupBasicMocks() throws IOException, ServiceException {
  rs = Mockito.mock(HRegionServer.class);
  Mockito.doReturn(HBaseConfiguration.create())
    .when(rs).getConfiguration();
  Mockito.doReturn(fakeResponse).when(rs).getServerInfo(
    (RpcController)Mockito.any(), (GetServerInfoRequest)Mockito.any());
  // Fake ZKW
  ZooKeeperWatcher zkw = Mockito.mock(ZooKeeperWatcher.class);
  Mockito.doReturn("fakequorum").when(zkw).getQuorum();
  Mockito.doReturn(zkw).when(rs).getZooKeeper();

  // Fake MasterAddressTracker
  MasterAddressTracker mat = Mockito.mock(MasterAddressTracker.class);
  Mockito.doReturn(fakeMasterAddress).when(mat).getMasterAddress();
  Mockito.doReturn(mat).when(rs).getMasterAddressManager();

  MetricsRegionServer rms = Mockito.mock(MetricsRegionServer.class);
  Mockito.doReturn(new MetricsRegionServerWrapperStub()).when(rms).getRegionServerWrapper();
  Mockito.doReturn(rms).when(rs).getMetrics();
}

@Before
public void setupBasicMocks() throws IOException, ServiceException {
  rs = Mockito.mock(HRegionServer.class);
  Mockito.doReturn(HBaseConfiguration.create())
    .when(rs).getConfiguration();
  Mockito.doReturn(fakeResponse).when(rs).getServerInfo(
    (RpcController)Mockito.any(), (GetServerInfoRequest)Mockito.any());
  // Fake ZKW
  ZooKeeperWatcher zkw = Mockito.mock(ZooKeeperWatcher.class);
  Mockito.doReturn("fakequorum").when(zkw).getQuorum();
  Mockito.doReturn(zkw).when(rs).getZooKeeper();

  // Fake MasterAddressTracker
  MasterAddressTracker mat = Mockito.mock(MasterAddressTracker.class);
  Mockito.doReturn(fakeMasterAddress).when(mat).getMasterAddress();
  Mockito.doReturn(mat).when(rs).getMasterAddressManager();

  MetricsRegionServer rms = Mockito.mock(MetricsRegionServer.class);
  Mockito.doReturn(new MetricsRegionServerWrapperStub()).when(rms).getRegionServerWrapper();
  Mockito.doReturn(rms).when(rs).getMetrics();
}

/**
 * Assert there is an active master and that it has the specified address.
 * @param zk
 * @param thisMasterAddress
 * @throws KeeperException
 * @throws IOException 
 */
private void assertMaster(ZooKeeperWatcher zk,
    ServerName expectedAddress)
throws KeeperException, IOException {
  ServerName readAddress = MasterAddressTracker.getMasterAddress(zk);
  assertNotNull(readAddress);
  assertTrue(expectedAddress.equals(readAddress));
}

@Before
public void setupBasicMocks() throws IOException, ServiceException {
  rs = Mockito.mock(HRegionServer.class);
  rpcServices = Mockito.mock(RSRpcServices.class);
  rpcServer = Mockito.mock(RpcServerInterface.class);
  Mockito.doReturn(HBaseConfiguration.create())
    .when(rs).getConfiguration();
  Mockito.doReturn(rpcServices).when(rs).getRSRpcServices();
  Mockito.doReturn(rpcServer).when(rs).getRpcServer();
  Mockito.doReturn(fakeResponse).when(rpcServices).getServerInfo(
    (RpcController)Mockito.any(), (GetServerInfoRequest)Mockito.any());
  // Fake ZKW
  ZooKeeperWatcher zkw = Mockito.mock(ZooKeeperWatcher.class);
  Mockito.doReturn("fakequorum").when(zkw).getQuorum();
  Mockito.doReturn(zkw).when(rs).getZooKeeper();

  // Fake CacheConfig
  LOG.warn("The " + HConstants.HFILE_BLOCK_CACHE_SIZE_KEY + " is set to 0");
  CacheConfig cacheConf = Mockito.mock(CacheConfig.class);
  Mockito.doReturn(null).when(cacheConf).getBlockCache();
  Mockito.doReturn(cacheConf).when(rs).getCacheConfig();

  // Fake MasterAddressTracker
  MasterAddressTracker mat = Mockito.mock(MasterAddressTracker.class);
  Mockito.doReturn(fakeMasterAddress).when(mat).getMasterAddress();
  Mockito.doReturn(mat).when(rs).getMasterAddressTracker();

  MetricsRegionServer rms = Mockito.mock(MetricsRegionServer.class);
  Mockito.doReturn(new MetricsRegionServerWrapperStub()).when(rms).getRegionServerWrapper();
  Mockito.doReturn(rms).when(rs).getRegionServerMetrics();

  MetricsHBaseServer ms = Mockito.mock(MetricsHBaseServer.class);
  Mockito.doReturn(new MetricsHBaseServerWrapperStub()).when(ms).getHBaseServerWrapper();
  Mockito.doReturn(ms).when(rpcServer).getMetrics();
}

@Test
public void testDeleteIfEquals() throws Exception {
  final ServerName sn = ServerName.valueOf("localhost", 1234, System.currentTimeMillis());
  final MasterAddressTracker addressTracker = setupMasterTracker(sn, 1772);
  try {
    assertFalse("shouldn't have deleted wrong master server.",
        MasterAddressTracker.deleteIfEquals(addressTracker.getWatcher(), "some other string."));
  } finally {
    assertTrue("Couldn't clean up master",
        MasterAddressTracker.deleteIfEquals(addressTracker.getWatcher(), sn.toString()));
  }
}

/**
 * create an address tracker instance
 * @param sn if not-null set the active master
 * @param infoPort if there is an active master, set its info port.
 */
private MasterAddressTracker setupMasterTracker(final ServerName sn, final int infoPort)
    throws Exception {
  ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
      name.getMethodName(), null);
  ZKUtil.createAndFailSilent(zk, zk.baseZNode);

  // Should not have a master yet
  MasterAddressTracker addressTracker = new MasterAddressTracker(zk, null);
  addressTracker.start();
  assertFalse(addressTracker.hasMaster());
  zk.registerListener(addressTracker);

  // Use a listener to capture when the node is actually created
  NodeCreationListener listener = new NodeCreationListener(zk, zk.getMasterAddressZNode());
  zk.registerListener(listener);

  if (sn != null) {
    LOG.info("Creating master node");
    MasterAddressTracker.setMasterAddress(zk, zk.getMasterAddressZNode(), sn, infoPort);

    // Wait for the node to be created
    LOG.info("Waiting for master address manager to be notified");
    listener.waitForCreation();
    LOG.info("Master node created");
  }
  return addressTracker;
}

@Test
public void testNoBackups() throws Exception {
  final ServerName sn = ServerName.valueOf("localhost", 1234, System.currentTimeMillis());
  final MasterAddressTracker addressTracker = setupMasterTracker(sn, 1772);
  try {
    assertEquals("Should receive 0 for backup not found.", 0,
        addressTracker.getBackupMasterInfoPort(
            ServerName.valueOf("doesnotexist.example.com", 1234, System.currentTimeMillis())));
  } finally {
    assertTrue("Couldn't clean up master",
        MasterAddressTracker.deleteIfEquals(addressTracker.getWatcher(), sn.toString()));
  }
}

@Test
public void testNoMaster() throws Exception {
  final MasterAddressTracker addressTracker = setupMasterTracker(null, 1772);
  assertFalse(addressTracker.hasMaster());
  assertNull("should get null master when none active.", addressTracker.getMasterAddress());
  assertEquals("Should receive 0 for backup not found.", 0, addressTracker.getMasterInfoPort());
}

@Override
public int getMasterInfoPort() throws IOException {
  // TODO: Fix!  Reaching into internal implementation!!!!
  ConnectionManager.HConnectionImplementation connection =
      (ConnectionManager.HConnectionImplementation)this.connection;
  ZooKeeperKeepAliveConnection zkw = connection.getKeepAliveZooKeeperWatcher();
  try {
    return MasterAddressTracker.getMasterInfoPort(zkw);
  } catch (KeeperException e) {
    throw new IOException("Failed to get master info port from MasterAddressTracker", e);
  }
}

/**
 * Create a stub. Try once only.  It is not typed because there is no common type to
 * protobuf services nor their interfaces.  Let the caller do appropriate casting.
 * @return A stub for master services.
 * @throws IOException
 * @throws KeeperException
 * @throws ServiceException
 */
private Object makeStubNoRetries() throws IOException, KeeperException, ServiceException {
  ZooKeeperKeepAliveConnection zkw;
  try {
    zkw = getKeepAliveZooKeeperWatcher();
  } catch (IOException e) {
    ExceptionUtil.rethrowIfInterrupt(e);
    throw new ZooKeeperConnectionException("Can't connect to ZooKeeper", e);
  }
  try {
    checkIfBaseNodeAvailable(zkw);
    ServerName sn = MasterAddressTracker.getMasterAddress(zkw);
    if (sn == null) {
      String msg = "ZooKeeper available but no active master location found";
      LOG.info(msg);
      throw new MasterNotRunningException(msg);
    }
    if (isDeadServer(sn)) {
      throw new MasterNotRunningException(sn + " is dead.");
    }
    // Use the security info interface name as our stub key
    String key = getStubKey(getServiceName(),
        sn.getHostname(), sn.getPort(), hostnamesCanChange);
    connectionLock.putIfAbsent(key, key);
    Object stub = null;
    synchronized (connectionLock.get(key)) {
      stub = stubs.get(key);
      if (stub == null) {
        BlockingRpcChannel channel = rpcClient.createBlockingRpcChannel(sn, user, rpcTimeout);
        stub = makeStub(channel);
        isMasterRunning();
        stubs.put(key, stub);
      }
    }
    return stub;
  } finally {
    zkw.close();
  }
}

/**
 * Assert there is an active master and that it has the specified address.
 * @param zk
 * @param thisMasterAddress
 * @throws KeeperException
 * @throws IOException 
 */
private void assertMaster(ZooKeeperWatcher zk,
    ServerName expectedAddress)
throws KeeperException, IOException {
  ServerName readAddress = MasterAddressTracker.getMasterAddress(zk);
  assertNotNull(readAddress);
  assertTrue(expectedAddress.equals(readAddress));
}

@Before
public void setupBasicMocks() throws IOException, ServiceException {
  rs = Mockito.mock(HRegionServer.class);
  rpcServices = Mockito.mock(RSRpcServices.class);
  Mockito.doReturn(HBaseConfiguration.create())
    .when(rs).getConfiguration();
  Mockito.doReturn(rpcServices).when(rs).getRSRpcServices();
  Mockito.doReturn(fakeResponse).when(rpcServices).getServerInfo(
    (RpcController)Mockito.any(), (GetServerInfoRequest)Mockito.any());
  // Fake ZKW
  ZooKeeperWatcher zkw = Mockito.mock(ZooKeeperWatcher.class);
  Mockito.doReturn("fakequorum").when(zkw).getQuorum();
  Mockito.doReturn(zkw).when(rs).getZooKeeper();

  // Fake CacheConfig
  LOG.warn("The " + HConstants.HFILE_BLOCK_CACHE_SIZE_KEY + " is set to 0");
  CacheConfig cacheConf = Mockito.mock(CacheConfig.class);
  Mockito.doReturn(null).when(cacheConf).getBlockCache();
  Mockito.doReturn(cacheConf).when(rs).getCacheConfig();

  // Fake MasterAddressTracker
  MasterAddressTracker mat = Mockito.mock(MasterAddressTracker.class);
  Mockito.doReturn(fakeMasterAddress).when(mat).getMasterAddress();
  Mockito.doReturn(mat).when(rs).getMasterAddressTracker();

  MetricsRegionServer rms = Mockito.mock(MetricsRegionServer.class);
  Mockito.doReturn(new MetricsRegionServerWrapperStub()).when(rms).getRegionServerWrapper();
  Mockito.doReturn(rms).when(rs).getRegionServerMetrics();
}

/**
 * Unit tests that uses ZooKeeper but does not use the master-side methods
 * but rather acts directly on ZK.
 * @throws Exception
 */
@Test
public void testMasterAddressTrackerFromZK() throws Exception {

  ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
      "testMasterAddressTrackerFromZK", null);
  ZKUtil.createAndFailSilent(zk, zk.baseZNode);

  // Should not have a master yet
  MasterAddressTracker addressTracker = new MasterAddressTracker(zk, null);
  addressTracker.start();
  assertFalse(addressTracker.hasMaster());
  zk.registerListener(addressTracker);

  // Use a listener to capture when the node is actually created
  NodeCreationListener listener = new NodeCreationListener(zk, zk.getMasterAddressZNode());
  zk.registerListener(listener);

  // Create the master node with a dummy address
  String host = "localhost";
  int port = 1234;
  int infoPort = 1235;
  ServerName sn = ServerName.valueOf(host, port, System.currentTimeMillis());
  LOG.info("Creating master node");
  MasterAddressTracker.setMasterAddress(zk, zk.getMasterAddressZNode(), sn, infoPort);

  // Wait for the node to be created
  LOG.info("Waiting for master address manager to be notified");
  listener.waitForCreation();
  LOG.info("Master node created");
  assertTrue(addressTracker.hasMaster());
  ServerName pulledAddress = addressTracker.getMasterAddress();
  assertTrue(pulledAddress.equals(sn));
  assertEquals(infoPort, addressTracker.getMasterInfoPort());
}

@Override
public int getMasterInfoPort() throws IOException {
  // TODO: Fix!  Reaching into internal implementation!!!!
  ConnectionManager.HConnectionImplementation connection =
      (ConnectionManager.HConnectionImplementation)this.connection;
  ZooKeeperKeepAliveConnection zkw = connection.getKeepAliveZooKeeperWatcher();
  try {
    return MasterAddressTracker.getMasterInfoPort(zkw);
  } catch (KeeperException e) {
    throw new IOException("Failed to get master info port from MasterAddressTracker", e);
  }
}

/**
 * Create a stub. Try once only.  It is not typed because there is no common type to
 * protobuf services nor their interfaces.  Let the caller do appropriate casting.
 *
 * @return A stub for master services.
 * @throws IOException
 * @throws KeeperException
 * @throws ServiceException
 */
private Object makeStubNoRetries() throws IOException, KeeperException, ServiceException {
    ZooKeeperKeepAliveConnection zkw;
    try {
        zkw = getKeepAliveZooKeeperWatcher();
    } catch (IOException e) {
        ExceptionUtil.rethrowIfInterrupt(e);
        throw new ZooKeeperConnectionException("Can't connect to ZooKeeper", e);
    }
    try {
        checkIfBaseNodeAvailable(zkw);
        ServerName sn = MasterAddressTracker.getMasterAddress(zkw);
        if (sn == null) {
            String msg = "ZooKeeper available but no active master location found";
            LOG.info(msg);
            throw new MasterNotRunningException(msg);
        }
        if (isDeadServer(sn)) {
            throw new MasterNotRunningException(sn + " is dead.");
        }
        // Use the security info interface name as our stub key
        String key = getStubKey(getServiceName(), sn.getHostAndPort());
        connectionLock.putIfAbsent(key, key);
        Object stub = null;
        synchronized (connectionLock.get(key)) {
            stub = stubs.get(key);
            if (stub == null) {
                BlockingRpcChannel channel = rpcClient.createBlockingRpcChannel(sn, user, rpcTimeout);
                stub = makeStub(channel);
                isMasterRunning();
                stubs.put(key, stub);
            }
        }
        return stub;
    } finally {
        zkw.close();
    }
}

/**
 * Assert there is an active master and that it has the specified address.
 * @param zk
 * @param thisMasterAddress
 * @throws KeeperException
 * @throws IOException 
 */
private void assertMaster(ZooKeeperWatcher zk,
    ServerName expectedAddress)
throws KeeperException, IOException {
  ServerName readAddress = MasterAddressTracker.getMasterAddress(zk);
  assertNotNull(readAddress);
  assertTrue(expectedAddress.equals(readAddress));
}

/**
 * Unit tests that uses ZooKeeper but does not use the master-side methods
 * but rather acts directly on ZK.
 * @throws Exception
 */
@Test
public void testMasterAddressTrackerFromZK() throws Exception {

  ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
      "testMasterAddressTrackerFromZK", null);
  ZKUtil.createAndFailSilent(zk, zk.baseZNode);

  // Should not have a master yet
  MasterAddressTracker addressTracker = new MasterAddressTracker(zk, null);
  addressTracker.start();
  assertFalse(addressTracker.hasMaster());
  zk.registerListener(addressTracker);

  // Use a listener to capture when the node is actually created
  NodeCreationListener listener = new NodeCreationListener(zk, zk.getMasterAddressZNode());
  zk.registerListener(listener);

  // Create the master node with a dummy address
  String host = "localhost";
  int port = 1234;
  ServerName sn = ServerName.valueOf(host, port, System.currentTimeMillis());
  LOG.info("Creating master node");
  MasterAddressTracker.setMasterAddress(zk, zk.getMasterAddressZNode(), sn);

  // Wait for the node to be created
  LOG.info("Waiting for master address manager to be notified");
  listener.waitForCreation();
  LOG.info("Master node created");
  assertTrue(addressTracker.hasMaster());
  ServerName pulledAddress = addressTracker.getMasterAddress();
  assertTrue(pulledAddress.equals(sn));

}

/**
 * Create a stub. Try once only.  It is not typed because there is no common type to
 * protobuf services nor their interfaces.  Let the caller do appropriate casting.
 * @return A stub for master services.
 * @throws IOException
 * @throws KeeperException
 * @throws ServiceException
 */
private Object makeStubNoRetries() throws IOException, KeeperException, ServiceException {
  ZooKeeperKeepAliveConnection zkw;
  try {
    zkw = getKeepAliveZooKeeperWatcher();
  } catch (IOException e) {
    ExceptionUtil.rethrowIfInterrupt(e);
    throw new ZooKeeperConnectionException("Can't connect to ZooKeeper", e);
  }
  try {
    checkIfBaseNodeAvailable(zkw);
    ServerName sn = MasterAddressTracker.getMasterAddress(zkw);
    if (sn == null) {
      String msg = "ZooKeeper available but no active master location found";
      LOG.info(msg);
      throw new MasterNotRunningException(msg);
    }
    if (isDeadServer(sn)) {
      throw new MasterNotRunningException(sn + " is dead.");
    }
    // Use the security info interface name as our stub key
    String key = getStubKey(getServiceName(), sn.getHostAndPort());
    connectionLock.putIfAbsent(key, key);
    Object stub = null;
    synchronized (connectionLock.get(key)) {
      stub = stubs.get(key);
      if (stub == null) {
        BlockingRpcChannel channel = rpcClient.createBlockingRpcChannel(sn,
          user, rpcTimeout);
        stub = makeStub(channel);
        isMasterRunning();
        stubs.put(key, stub);
      }
    }
    return stub;
  } finally {
    zkw.close();
  }
}

private String getHBaseMasterUrl() throws IOException, KeeperException {
    String host = conf.get("hbase.master.info.bindAddress");
    if (host.equals("0.0.0.0")) {
        host = MasterAddressTracker.getMasterAddress(new ZooKeeperWatcher(conf, null, null)).getHostname();
    }

    String port = conf.get("hbase.master.info.port");
    return "http://" + host + ":" + port + "/";
}

/**
 * Assert there is an active master and that it has the specified address.
 * @param zk
 * @param thisMasterAddress
 * @throws KeeperException
 * @throws IOException
 */
private void assertMaster(ZKWatcher zk,
    ServerName expectedAddress)
throws KeeperException, IOException {
  ServerName readAddress = MasterAddressTracker.getMasterAddress(zk);
  assertNotNull(readAddress);
  assertTrue(expectedAddress.equals(readAddress));
}

@Before
public void setupBasicMocks() throws IOException, ServiceException {
  rs = Mockito.mock(HRegionServer.class);
  rpcServices = Mockito.mock(RSRpcServices.class);
  rpcServer = Mockito.mock(RpcServerInterface.class);
  Mockito.doReturn(HBaseConfiguration.create())
    .when(rs).getConfiguration();
  Mockito.doReturn(rpcServices).when(rs).getRSRpcServices();
  Mockito.doReturn(rpcServer).when(rs).getRpcServer();
  Mockito.doReturn(fakeResponse).when(rpcServices).getServerInfo(
    (RpcController)Mockito.any(), (GetServerInfoRequest)Mockito.any());
  // Fake ZKW
  ZKWatcher zkw = Mockito.mock(ZKWatcher.class);
  Mockito.doReturn("fakequorum").when(zkw).getQuorum();
  Mockito.doReturn(zkw).when(rs).getZooKeeper();

  // Fake CacheConfig
  LOG.warn("The " + HConstants.HFILE_BLOCK_CACHE_SIZE_KEY + " is set to 0");
  CacheConfig cacheConf = Mockito.mock(CacheConfig.class);
  Mockito.doReturn(null).when(cacheConf).getBlockCache();
  Mockito.doReturn(cacheConf).when(rs).getCacheConfig();

  // Fake MasterAddressTracker
  MasterAddressTracker mat = Mockito.mock(MasterAddressTracker.class);
  Mockito.doReturn(fakeMasterAddress).when(mat).getMasterAddress();
  Mockito.doReturn(mat).when(rs).getMasterAddressTracker();

  MetricsRegionServer rms = Mockito.mock(MetricsRegionServer.class);
  Mockito.doReturn(new MetricsRegionServerWrapperStub()).when(rms).getRegionServerWrapper();
  Mockito.doReturn(rms).when(rs).getRegionServerMetrics();

  MetricsHBaseServer ms = Mockito.mock(MetricsHBaseServer.class);
  Mockito.doReturn(new MetricsHBaseServerWrapperStub()).when(ms).getHBaseServerWrapper();
  Mockito.doReturn(ms).when(rpcServer).getMetrics();
}

@Test
public void testDeleteIfEquals() throws Exception {
  final ServerName sn = ServerName.valueOf("localhost", 1234, System.currentTimeMillis());
  final MasterAddressTracker addressTracker = setupMasterTracker(sn, 1772);
  try {
    assertFalse("shouldn't have deleted wrong master server.",
        MasterAddressTracker.deleteIfEquals(addressTracker.getWatcher(), "some other string."));
  } finally {
    assertTrue("Couldn't clean up master",
        MasterAddressTracker.deleteIfEquals(addressTracker.getWatcher(), sn.toString()));
  }
}

/**
 * create an address tracker instance
 * @param sn if not-null set the active master
 * @param infoPort if there is an active master, set its info port.
 */
private MasterAddressTracker setupMasterTracker(final ServerName sn, final int infoPort)
    throws Exception {
  ZKWatcher zk = new ZKWatcher(TEST_UTIL.getConfiguration(),
      name.getMethodName(), null);
  ZKUtil.createAndFailSilent(zk, zk.znodePaths.baseZNode);

  // Should not have a master yet
  MasterAddressTracker addressTracker = new MasterAddressTracker(zk, null);
  addressTracker.start();
  assertFalse(addressTracker.hasMaster());
  zk.registerListener(addressTracker);

  // Use a listener to capture when the node is actually created
  NodeCreationListener listener = new NodeCreationListener(zk, zk.znodePaths.masterAddressZNode);
  zk.registerListener(listener);

  if (sn != null) {
    LOG.info("Creating master node");
    MasterAddressTracker.setMasterAddress(zk, zk.znodePaths.masterAddressZNode, sn, infoPort);

    // Wait for the node to be created
    LOG.info("Waiting for master address manager to be notified");
    listener.waitForCreation();
    LOG.info("Master node created");
  }
  return addressTracker;
}

@Test
public void testNoBackups() throws Exception {
  final ServerName sn = ServerName.valueOf("localhost", 1234, System.currentTimeMillis());
  final MasterAddressTracker addressTracker = setupMasterTracker(sn, 1772);
  try {
    assertEquals("Should receive 0 for backup not found.", 0,
        addressTracker.getBackupMasterInfoPort(
            ServerName.valueOf("doesnotexist.example.com", 1234, System.currentTimeMillis())));
  } finally {
    assertTrue("Couldn't clean up master",
        MasterAddressTracker.deleteIfEquals(addressTracker.getWatcher(), sn.toString()));
  }
}

@Test
public void testNoMaster() throws Exception {
  final MasterAddressTracker addressTracker = setupMasterTracker(null, 1772);
  assertFalse(addressTracker.hasMaster());
  assertNull("should get null master when none active.", addressTracker.getMasterAddress());
  assertEquals("Should receive 0 for backup not found.", 0, addressTracker.getMasterInfoPort());
}

/**
 * Assert there is an active master and that it has the specified address.
 * @param zk
 * @param thisMasterAddress
 * @throws KeeperException
 * @throws IOException 
 */
private void assertMaster(ZooKeeperWatcher zk,
    ServerName expectedAddress)
throws KeeperException, IOException {
  ServerName readAddress = MasterAddressTracker.getMasterAddress(zk);
  assertNotNull(readAddress);
  assertTrue(expectedAddress.equals(readAddress));
}

/**
 * Unit tests that uses ZooKeeper but does not use the master-side methods
 * but rather acts directly on ZK.
 * @throws Exception
 */
@Test
public void testMasterAddressTrackerFromZK() throws Exception {

  ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
      "testMasterAddressTrackerFromZK", null);
  ZKUtil.createAndFailSilent(zk, zk.baseZNode);

  // Should not have a master yet
  MasterAddressTracker addressTracker = new MasterAddressTracker(zk, null);
  addressTracker.start();
  assertFalse(addressTracker.hasMaster());
  zk.registerListener(addressTracker);

  // Use a listener to capture when the node is actually created
  NodeCreationListener listener = new NodeCreationListener(zk, zk.getMasterAddressZNode());
  zk.registerListener(listener);

  // Create the master node with a dummy address
  String host = "localhost";
  int port = 1234;
  ServerName sn = ServerName.valueOf(host, port, System.currentTimeMillis());
  LOG.info("Creating master node");
  MasterAddressTracker.setMasterAddress(zk, zk.getMasterAddressZNode(), sn);

  // Wait for the node to be created
  LOG.info("Waiting for master address manager to be notified");
  listener.waitForCreation();
  LOG.info("Master node created");
  assertTrue(addressTracker.hasMaster());
  ServerName pulledAddress = addressTracker.getMasterAddress();
  assertTrue(pulledAddress.equals(sn));

}

/**
 * Create a stub. Try once only.  It is not typed because there is no common type to
 * protobuf services nor their interfaces.  Let the caller do appropriate casting.
 * @return A stub for master services.
 * @throws IOException
 * @throws KeeperException
 * @throws ServiceException
 */
private Object makeStubNoRetries() throws IOException, KeeperException, ServiceException {
  ZooKeeperKeepAliveConnection zkw;
  try {
    zkw = getKeepAliveZooKeeperWatcher();
  } catch (IOException e) {
    ExceptionUtil.rethrowIfInterrupt(e);
    throw new ZooKeeperConnectionException("Can't connect to ZooKeeper", e);
  }
  try {
    checkIfBaseNodeAvailable(zkw);
    ServerName sn = MasterAddressTracker.getMasterAddress(zkw);
    if (sn == null) {
      String msg = "ZooKeeper available but no active master location found";
      LOG.info(msg);
      throw new MasterNotRunningException(msg);
    }
    if (isDeadServer(sn)) {
      throw new MasterNotRunningException(sn + " is dead.");
    }
    // Use the security info interface name as our stub key
    String key = getStubKey(getServiceName(), sn.getHostAndPort());
    connectionLock.putIfAbsent(key, key);
    Object stub = null;
    synchronized (connectionLock.get(key)) {
      stub = stubs.get(key);
      if (stub == null) {
        BlockingRpcChannel channel = rpcClient.createBlockingRpcChannel(sn, user, rpcTimeout);
        stub = makeStub(channel);
        isMasterRunning();
        stubs.put(key, stub);
      }
    }
    return stub;
  } finally {
    zkw.close();
  }
}

/**
 * Assert there is an active master and that it has the specified address.
 * @param zk
 * @param thisMasterAddress
 * @throws KeeperException
 * @throws IOException 
 */
private void assertMaster(ZooKeeperWatcher zk,
    ServerName expectedAddress)
throws KeeperException, IOException {
  ServerName readAddress = MasterAddressTracker.getMasterAddress(zk);
  assertNotNull(readAddress);
  assertTrue(expectedAddress.equals(readAddress));
}

/**
 * Unit tests that uses ZooKeeper but does not use the master-side methods
 * but rather acts directly on ZK.
 * @throws Exception
 */
@Test
public void testMasterAddressManagerFromZK() throws Exception {

  ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
      "testMasterAddressManagerFromZK", null);
  ZKUtil.createAndFailSilent(zk, zk.baseZNode);

  // Should not have a master yet
  MasterAddressTracker addressManager = new MasterAddressTracker(zk, null);
  addressManager.start();
  assertFalse(addressManager.hasMaster());
  zk.registerListener(addressManager);

  // Use a listener to capture when the node is actually created
  NodeCreationListener listener = new NodeCreationListener(zk, zk.getMasterAddressZNode());
  zk.registerListener(listener);

  // Create the master node with a dummy address
  String host = "localhost";
  int port = 1234;
  ServerName sn = ServerName.valueOf(host, port, System.currentTimeMillis());
  LOG.info("Creating master node");
  MasterAddressTracker.setMasterAddress(zk, zk.getMasterAddressZNode(), sn);

  // Wait for the node to be created
  LOG.info("Waiting for master address manager to be notified");
  listener.waitForCreation();
  LOG.info("Master node created");
  assertTrue(addressManager.hasMaster());
  ServerName pulledAddress = addressManager.getMasterAddress();
  assertTrue(pulledAddress.equals(sn));

}

/**
 * Create a stub. Try once only.  It is not typed because there is no common type to
 * protobuf services nor their interfaces.  Let the caller do appropriate casting.
 * @return A stub for master services.
 * @throws IOException
 * @throws KeeperException
 * @throws ServiceException
 */
private Object makeStubNoRetries() throws IOException, KeeperException, ServiceException {
  ZooKeeperKeepAliveConnection zkw;
  try {
    zkw = getKeepAliveZooKeeperWatcher();
  } catch (IOException e) {
    throw new ZooKeeperConnectionException("Can't connect to ZooKeeper", e);
  }
  try {
    checkIfBaseNodeAvailable(zkw);
    ServerName sn = MasterAddressTracker.getMasterAddress(zkw);
    if (sn == null) {
      String msg = "ZooKeeper available but no active master location found";
      LOG.info(msg);
      throw new MasterNotRunningException(msg);
    }
    if (isDeadServer(sn)) {
      throw new MasterNotRunningException(sn + " is dead.");
    }
    // Use the security info interface name as our stub key
    String key = getStubKey(getServiceName(), sn.getHostAndPort());
    connectionLock.putIfAbsent(key, key);
    Object stub = null;
    synchronized (connectionLock.get(key)) {
      stub = stubs.get(key);
      if (stub == null) {
        BlockingRpcChannel channel = rpcClient.createBlockingRpcChannel(sn,
          user, rpcTimeout);
        stub = makeStub(channel);
        isMasterRunning();
        stubs.put(key, stub);
      }
    }
    return stub;
  } finally {
    zkw.close();
  }
}

/**
 * Bring up connection to zk ensemble and then wait until a master for this
 * cluster and then after that, wait until cluster 'up' flag has been set.
 * This is the order in which master does things.
 * Finally put up a catalog tracker.
 * @throws IOException
 * @throws InterruptedException
 */
private void initializeZooKeeper() throws IOException, InterruptedException {
  // Open connection to zookeeper and set primary watcher
  this.zooKeeper = new ZooKeeperWatcher(conf, REGIONSERVER + ":" +
    this.isa.getPort(), this);

  // Create the master address manager, register with zk, and start it.  Then
  // block until a master is available.  No point in starting up if no master
  // running.
  this.masterAddressManager = new MasterAddressTracker(this.zooKeeper, this);
  this.masterAddressManager.start();
  blockAndCheckIfStopped(this.masterAddressManager);

  // Wait on cluster being up.  Master will set this flag up in zookeeper
  // when ready.
  this.clusterStatusTracker = new ClusterStatusTracker(this.zooKeeper, this);
  this.clusterStatusTracker.start();
  blockAndCheckIfStopped(this.clusterStatusTracker);

  // Create the catalog tracker and start it;
  this.catalogTracker = new CatalogTracker(this.zooKeeper, this.conf,
    this, this.conf.getInt("hbase.regionserver.catalog.timeout", 600000));
  catalogTracker.start();

  // Retrieve clusterId
  // Since cluster status is now up
  // ID should have already been set by HMaster
  try {
    String clusterId = ZKClusterId.readClusterIdZNode(this.zooKeeper);
    if (clusterId == null) {
      this.abort("Cluster ID has not been set");
    }
    this.conf.set(HConstants.CLUSTER_ID, clusterId);
    LOG.info("ClusterId : "+clusterId);
  } catch (KeeperException e) {
    this.abort("Failed to retrieve Cluster ID",e);
  }
}

/**
 * Assert there is an active master and that it has the specified address.
 * @param zk
 * @param thisMasterAddress
 * @throws KeeperException
 * @throws IOException 
 */
private void assertMaster(ZooKeeperWatcher zk,
    ServerName expectedAddress)
throws KeeperException, IOException {
  ServerName readAddress = MasterAddressTracker.getMasterAddress(zk);
  assertNotNull(readAddress);
  assertTrue(expectedAddress.equals(readAddress));
}

/**
 * Unit tests that uses ZooKeeper but does not use the master-side methods
 * but rather acts directly on ZK.
 * @throws Exception
 */
@Test
public void testMasterAddressManagerFromZK() throws Exception {

  ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
      "testMasterAddressManagerFromZK", null);
  ZKUtil.createAndFailSilent(zk, zk.baseZNode);

  // Should not have a master yet
  MasterAddressTracker addressManager = new MasterAddressTracker(zk, null);
  addressManager.start();
  assertFalse(addressManager.hasMaster());
  zk.registerListener(addressManager);

  // Use a listener to capture when the node is actually created
  NodeCreationListener listener = new NodeCreationListener(zk, zk.getMasterAddressZNode());
  zk.registerListener(listener);

  // Create the master node with a dummy address
  String host = "localhost";
  int port = 1234;
  ServerName sn = new ServerName(host, port, System.currentTimeMillis());
  LOG.info("Creating master node");
  MasterAddressTracker.setMasterAddress(zk, zk.getMasterAddressZNode(), sn);

  // Wait for the node to be created
  LOG.info("Waiting for master address manager to be notified");
  listener.waitForCreation();
  LOG.info("Master node created");
  assertTrue(addressManager.hasMaster());
  ServerName pulledAddress = addressManager.getMasterAddress();
  assertTrue(pulledAddress.equals(sn));

}

private void startActiveMasterManager(int infoPort) throws KeeperException {
  String backupZNode = ZKUtil.joinZNode(
    zooKeeper.backupMasterAddressesZNode, serverName.toString());
  /*
  * Add a ZNode for ourselves in the backup master directory since we
  * may not become the active master. If so, we want the actual active
  * master to know we are backup masters, so that it won't assign
  * regions to us if so configured.
  *
  * If we become the active master later, ActiveMasterManager will delete
  * this node explicitly.  If we crash before then, ZooKeeper will delete
  * this node for us since it is ephemeral.
  */
  LOG.info("Adding backup master ZNode " + backupZNode);
  if (!MasterAddressTracker.setMasterAddress(zooKeeper, backupZNode,
      serverName, infoPort)) {
    LOG.warn("Failed create of " + backupZNode + " by " + serverName);
  }

  activeMasterManager.setInfoPort(infoPort);
  // Start a thread to try to become the active master, so we won't block here
  Threads.setDaemonThreadRunning(new Thread(new Runnable() {
    @Override
    public void run() {
      int timeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT,
        HConstants.DEFAULT_ZK_SESSION_TIMEOUT);
      // If we're a backup master, stall until a primary to writes his address
      if (conf.getBoolean(HConstants.MASTER_TYPE_BACKUP,
        HConstants.DEFAULT_MASTER_TYPE_BACKUP)) {
        LOG.debug("HMaster started in backup mode. "
          + "Stalling until master znode is written.");
        // This will only be a minute or so while the cluster starts up,
        // so don't worry about setting watches on the parent znode
        while (!activeMasterManager.hasActiveMaster()) {
          LOG.debug("Waiting for master address ZNode to be written "
            + "(Also watching cluster state node)");
          Threads.sleep(timeout);
        }
      }
      MonitoredTask status = TaskMonitor.get().createStatus("Master startup");
      status.setDescription("Master startup");
      try {
        if (activeMasterManager.blockUntilBecomingActiveMaster(timeout, status)) {
          finishActiveMasterInitialization(status);
        }
      } catch (Throwable t) {
        status.setStatus("Failed to become active: " + t.getMessage());
        LOG.fatal("Failed to become active master", t);
        // HBASE-5680: Likely hadoop23 vs hadoop 20.x/1.x incompatibility
        if (t instanceof NoClassDefFoundError &&
          t.getMessage()
            .contains("org/apache/hadoop/hdfs/protocol/HdfsConstants$SafeModeAction")) {
          // improved error message for this special case
          abort("HBase is having a problem with its Hadoop jars.  You may need to "
            + "recompile HBase against Hadoop version "
            + org.apache.hadoop.util.VersionInfo.getVersion()
            + " or change your hadoop jars to start properly", t);
        } else {
          abort("Unhandled exception. Starting shutdown.", t);
        }
      } finally {
        status.cleanup();
      }
    }
  }, getServerName().toShortString() + ".activeMasterManager"));
}

/**
 * @return Master address tracker instance.
 */
public MasterAddressTracker getMasterAddressTracker() {
  return this.masterAddressTracker;
}