Java 类org.apache.hadoop.hbase.HBaseTestingUtility 实例源码

@Override
public void perform() throws Exception {
  HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
  Admin admin = util.getHBaseAdmin();
  boolean major = RandomUtils.nextInt(100) < majorRatio;

  LOG.info("Performing action: Compact table " + tableName + ", major=" + major);
  try {
    if (major) {
      admin.majorCompact(tableName);
    } else {
      admin.compact(tableName);
    }
  } catch (Exception ex) {
    LOG.warn("Compaction failed, might be caused by other chaos: " + ex.getMessage());
  }
  if (sleepTime > 0) {
    Thread.sleep(sleepTime);
  }
}

private static Thread killARegionServer(final HBaseTestingUtility utility,
                                        final long timeout, final int rs) {
  Thread killer = new Thread() {
    public void run() {
      try {
        Thread.sleep(timeout);
        utility.getHBaseCluster().getRegionServer(rs).stop("Stopping as part of the test");
      } catch (Exception e) {
        LOG.error("Couldn't kill a region server", e);
      }
    }
  };
  killer.setDaemon(true);
  killer.start();
  return killer;
}

@Override
public void perform() throws Exception {
  HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
  Admin admin = util.getHBaseAdmin();
  // Don't try the split if we're stopping
  if (context.isStopping()) {
    return;
  }


  // Don't always split. This should allow splitting of a full table later in the run
  if (ThreadLocalRandom.current().nextDouble()
      < (((double) splits) / ((double) maxFullTableSplits)) / ((double) 2)) {
    splits++;
    LOG.info("Performing action: Split all regions of  " + tableName);
    admin.split(tableName);
  } else {
    LOG.info("Skipping split of all regions.");
  }
}

@Before
public void setUpBefore() throws Exception {
  TEST_UTIL = new HBaseTestingUtility();
  TEST_UTIL.getConfiguration().setInt("dfs.datanode.max.xceivers", 9192);
  TEST_UTIL.startMiniCluster(3);
  conf = TEST_UTIL.getConfiguration();
  this.connection = ConnectionFactory.createConnection(conf);
  assertEquals(0, TEST_UTIL.getHBaseAdmin().listTables().length);

  // setup the table
  table = TableName.valueOf(TABLE_BASE + "-" + tableIdx);
  tableIdx++;
  htbl = setupTable(table);
  populateTable(htbl);
  assertEquals(5, scanMeta());
  LOG.info("Table " + table + " has " + tableRowCount(conf, table)
      + " entries.");
  assertEquals(16, tableRowCount(conf, table));
  TEST_UTIL.getHBaseAdmin().disableTable(table);
  assertEquals(1, TEST_UTIL.getHBaseAdmin().listTables().length);
}

@Override
public void perform() throws Exception {
  HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
  Admin admin = util.getHBaseAdmin();

  // Don't try the flush if we're stopping
  if (context.isStopping()) {
    return;
  }

  LOG.info("Performing action: Flush table " + tableName);
  try {
    admin.flush(tableName);
  } catch (Exception ex) {
    LOG.warn("Flush failed, might be caused by other chaos: " + ex.getMessage());
  }
  if (sleepTime > 0) {
    Thread.sleep(sleepTime);
  }
}

/**
 * Getting data from memstore only
 * @throws IOException
 */
@Test
public void testGet_FromMemStoreOnly() throws IOException {
  init(this.name.getMethodName());

  //Put data in memstore
  this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
  this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
  this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
  this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
  this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
  this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));

  //Get
  result = HBaseTestingUtility.getFromStoreFile(store,
      get.getRow(), qualifiers);

  //Compare
  assertCheck();
}

/**
 * Grant permissions globally to the given user. Will wait until all active
 * AccessController instances have updated their permissions caches or will
 * throw an exception upon timeout (10 seconds).
 */
public static void grantGlobal(final HBaseTestingUtility util, final String user,
    final Permission.Action... actions) throws Exception {
  SecureTestUtil.updateACLs(util, new Callable<Void>() {
    @Override
    public Void call() throws Exception {
      try (Connection connection = ConnectionFactory.createConnection(util.getConfiguration())) {
        try (Table acl = connection.getTable(AccessControlLists.ACL_TABLE_NAME)) {
          BlockingRpcChannel service = acl.coprocessorService(HConstants.EMPTY_START_ROW);
          AccessControlService.BlockingInterface protocol =
              AccessControlService.newBlockingStub(service);
          ProtobufUtil.grant(null, protocol, user, actions);
        }
      }
      return null;
    }
  });
}

/**
 * Revoke permissions globally from the given user. Will wait until all active
 * AccessController instances have updated their permissions caches or will
 * throw an exception upon timeout (10 seconds).
 */
public static void revokeGlobal(final HBaseTestingUtility util, final String user,
    final Permission.Action... actions) throws Exception {
  SecureTestUtil.updateACLs(util, new Callable<Void>() {
    @Override
    public Void call() throws Exception {
      try (Connection connection = ConnectionFactory.createConnection(util.getConfiguration())) {
        try (Table acl = connection.getTable(AccessControlLists.ACL_TABLE_NAME)) {
          BlockingRpcChannel service = acl.coprocessorService(HConstants.EMPTY_START_ROW);
          AccessControlService.BlockingInterface protocol =
              AccessControlService.newBlockingStub(service);
          ProtobufUtil.revoke(null, protocol, user, actions);
        }
      }
      return null;
    }
  });
}

/**
 * Revoke permissions on a table from the given user. Will wait until all active
 * AccessController instances have updated their permissions caches or will
 * throw an exception upon timeout (10 seconds).
 */
public static void revokeFromTable(final HBaseTestingUtility util, final String user,
    final TableName table, final byte[] family, final byte[] qualifier,
    final Permission.Action... actions) throws Exception {
  SecureTestUtil.updateACLs(util, new Callable<Void>() {
    @Override
    public Void call() throws Exception {
      try (Connection connection = ConnectionFactory.createConnection(util.getConfiguration())) {
        try (Table acl = connection.getTable(AccessControlLists.ACL_TABLE_NAME)) {
          BlockingRpcChannel service = acl.coprocessorService(HConstants.EMPTY_START_ROW);
          AccessControlService.BlockingInterface protocol =
              AccessControlService.newBlockingStub(service);
          ProtobufUtil.revoke(null, protocol, user, table, family, qualifier, actions);
        }
      }
      return null;
    }
  });
}

/**
 * Revoke permissions on a table from the given user using AccessControlClient. Will wait until
 * all active AccessController instances have updated their permissions caches or will
 * throw an exception upon timeout (10 seconds).
 */
public static void revokeFromTableUsingAccessControlClient(final HBaseTestingUtility util,
    final Connection connection, final String user, final TableName table, final byte[] family,
    final byte[] qualifier, final Permission.Action... actions) throws Exception {
  SecureTestUtil.updateACLs(util, new Callable<Void>() {
    @Override
    public Void call() throws Exception {
      try {
        AccessControlClient.revoke(connection, table, user, family, qualifier, actions);
      } catch (Throwable t) {
        t.printStackTrace();
      }
      return null;
    }
  });
}

@BeforeClass
public static void startCluster() throws Exception {
  metricsHelper = CompatibilityFactory.getInstance(MetricsAssertHelper.class);
  TEST_UTIL = new HBaseTestingUtility();
  conf = TEST_UTIL.getConfiguration();
  conf.getLong("hbase.splitlog.max.resubmit", 0);
  // Make the failure test faster
  conf.setInt("zookeeper.recovery.retry", 0);
  conf.setInt(HConstants.REGIONSERVER_INFO_PORT, -1);

  TEST_UTIL.startMiniCluster(1, 2);
  cluster = TEST_UTIL.getHBaseCluster();

  cluster.waitForActiveAndReadyMaster();

  while (cluster.getLiveRegionServerThreads().size() < 2) {
    Threads.sleep(100);
  }
}

public static void checkGlobalPerms(HBaseTestingUtility testUtil, Permission.Action... actions)
    throws IOException {
  Permission[] perms = new Permission[actions.length];
  for (int i = 0; i < actions.length; i++) {
    perms[i] = new Permission(actions[i]);
  }
  CheckPermissionsRequest.Builder request = CheckPermissionsRequest.newBuilder();
  for (Action a : actions) {
    request.addPermission(AccessControlProtos.Permission.newBuilder()
        .setType(AccessControlProtos.Permission.Type.Global)
        .setGlobalPermission(
            AccessControlProtos.GlobalPermission.newBuilder()
                .addAction(ProtobufUtil.toPermissionAction(a)).build()));
  }
  try(Connection conn = ConnectionFactory.createConnection(testUtil.getConfiguration());
      Table acl = conn.getTable(AccessControlLists.ACL_TABLE_NAME)) {
    BlockingRpcChannel channel = acl.coprocessorService(new byte[0]);
    AccessControlService.BlockingInterface protocol =
      AccessControlService.newBlockingStub(channel);
    try {
      protocol.checkPermissions(null, request.build());
    } catch (ServiceException se) {
      ProtobufUtil.toIOException(se);
    }
  }
}

@Override
protected void testWithMockedMapReduce(HBaseTestingUtility util, String snapshotName,
    int numRegions, int expectedNumSplits) throws Exception {
  setupCluster();
  TableName tableName = TableName.valueOf("testWithMockedMapReduce");
  try {
    createTableAndSnapshot(
      util, tableName, snapshotName, getStartRow(), getEndRow(), numRegions);

    JobConf job = new JobConf(util.getConfiguration());
    Path tmpTableDir = util.getRandomDir();

    TableMapReduceUtil.initTableSnapshotMapJob(snapshotName,
      COLUMNS, TestTableSnapshotMapper.class, ImmutableBytesWritable.class,
      NullWritable.class, job, false, tmpTableDir);

    // mapred doesn't support start and end keys? o.O
    verifyWithMockedMapReduce(job, numRegions, expectedNumSplits, getStartRow(), getEndRow());

  } finally {
    util.getHBaseAdmin().deleteSnapshot(snapshotName);
    util.deleteTable(tableName);
    tearDownCluster();
  }
}

@Override
public void perform() throws Exception {
  if (sleepTime > 0) {
    Thread.sleep(sleepTime);
  }

  HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
  Admin admin = util.getHBaseAdmin();

  LOG.info("Performing action: Move random region of table " + tableName);
  List<HRegionInfo> regions = admin.getTableRegions(tableName);
  if (regions == null || regions.isEmpty()) {
    LOG.info("Table " + tableName + " doesn't have regions to move");
    return;
  }

  HRegionInfo region = PolicyBasedChaosMonkey.selectRandomItem(
    regions.toArray(new HRegionInfo[regions.size()]));
  LOG.debug("Unassigning region " + region.getRegionNameAsString());
  admin.unassign(region.getRegionName(), false);
  if (sleepTime > 0) {
    Thread.sleep(sleepTime);
  }
}

@BeforeClass
public static void setUpBeforeClass() throws Exception {
  conf1.setBoolean(HConstants.REPLICATION_ENABLE_KEY, HConstants.REPLICATION_ENABLE_DEFAULT);
  utility1 = new HBaseTestingUtility(conf1);
  utility1.startMiniCluster();
  admin = new ReplicationAdmin(conf1);

  conf2 = HBaseConfiguration.create(conf1);
  conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
  conf2.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, 2182);

  utility2 = new HBaseTestingUtility(conf2);
  utility2.startMiniCluster();

  ReplicationPeerConfig config = new ReplicationPeerConfig();
  config.setClusterKey(utility2.getClusterKey());
  admin.addPeer(peerId, config, null);

  HTableDescriptor table = new HTableDescriptor(tableName);
  HColumnDescriptor fam = new HColumnDescriptor(famName);
  fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
  table.addFamily(fam);

  utility1.getHBaseAdmin().createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
  utility1.waitUntilAllRegionsAssigned(tableName);
}

@BeforeClass
public static void setUpBeforeClass() throws Exception {
  conf = HBaseConfiguration.create();
  conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
      TestCoprocessor.class.getName());
  util = new HBaseTestingUtility(conf);
  util.startMiniCluster();

  Admin admin = util.getHBaseAdmin();
  if (admin.tableExists(tableName)) {
    if (admin.isTableEnabled(tableName)) {
      admin.disableTable(tableName);
    }
    admin.deleteTable(tableName);
  }
  util.createTable(tableName, new byte[][]{dummy, test});

  Table ht = new HTable(conf, tableName);
  Put p = new Put(row1);
  p.add(dummy, dummy, dummy);
  ht.put(p);
}

@BeforeClass
public static void setUpBeforeClass() throws Exception {
  cluster = TEST_UTIL.startMiniCluster(1, ServerNum);
  table = TEST_UTIL.createTable(tableName, FAMILY, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
  TEST_UTIL.waitTableAvailable(tableName, 1000);
  TEST_UTIL.loadTable(table, FAMILY);

  for (int i = 0; i < ServerNum; i++) {
    HRegionServer server = cluster.getRegionServer(i);
    for (Region region : server.getOnlineRegions(tableName)) {
      region.flush(true);
    }
  }

  finder.setConf(TEST_UTIL.getConfiguration());
  finder.setServices(cluster.getMaster());
  finder.setClusterStatus(cluster.getMaster().getClusterStatus());
}

public static void createTable(HBaseTestingUtility testUtil, HBaseAdmin admin,
  HTableDescriptor htd, byte [][] splitKeys)
throws Exception {
  // NOTE: We need a latch because admin is not sync,
  // so the postOp coprocessor method may be called after the admin operation returned.
  MasterSyncObserver observer = (MasterSyncObserver)testUtil.getHBaseCluster().getMaster()
    .getMasterCoprocessorHost().findCoprocessor(MasterSyncObserver.class.getName());
  observer.tableCreationLatch = new CountDownLatch(1);
  if (splitKeys != null) {
    admin.createTable(htd, splitKeys);
  } else {
    admin.createTable(htd);
  }
  observer.tableCreationLatch.await();
  observer.tableCreationLatch = null;
  testUtil.waitUntilAllRegionsAssigned(htd.getTableName());
}

public static void deleteTable(HBaseTestingUtility testUtil, HBaseAdmin admin,
  TableName tableName)
throws Exception {
  // NOTE: We need a latch because admin is not sync,
  // so the postOp coprocessor method may be called after the admin operation returned.
  MasterSyncObserver observer = (MasterSyncObserver)testUtil.getHBaseCluster().getMaster()
    .getMasterCoprocessorHost().findCoprocessor(MasterSyncObserver.class.getName());
  observer.tableDeletionLatch = new CountDownLatch(1);
  try {
    admin.disableTable(tableName);
  } catch (Exception e) {
    LOG.debug("Table: " + tableName + " already disabled, so just deleting it.");
  }
  admin.deleteTable(tableName);
  observer.tableDeletionLatch.await();
  observer.tableDeletionLatch = null;
}

/**
 * Constructor. Modifies the passed configuration.
 * @param hbaseHome the top directory of the HBase source tree
 */
public ProcessBasedLocalHBaseCluster(Configuration conf,
    int numDataNodes, int numRegionServers) {
  this.conf = conf;
  this.hbaseHome = HBaseHomePath.getHomePath();
  this.numMasters = 1;
  this.numRegionServers = numRegionServers;
  this.workDir = hbaseHome + "/target/local_cluster";
  this.numDataNodes = numDataNodes;

  hbaseDaemonScript = hbaseHome + "/bin/hbase-daemon.sh";
  zkClientPort = HBaseTestingUtility.randomFreePort();

  this.rsPorts = sortedPorts(numRegionServers);
  this.masterPorts = sortedPorts(numMasters);

  conf.set(HConstants.ZOOKEEPER_QUORUM, HConstants.LOCALHOST);
  conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkClientPort);
}

@Test
public void testVersion() throws DeserializationException, IOException {
  HBaseTestingUtility htu = new HBaseTestingUtility();
  final FileSystem fs = htu.getTestFileSystem();
  final Path rootdir = htu.getDataTestDir();
  assertNull(FSUtils.getVersion(fs, rootdir));
  // Write out old format version file.  See if we can read it in and convert.
  Path versionFile = new Path(rootdir, HConstants.VERSION_FILE_NAME);
  FSDataOutputStream s = fs.create(versionFile);
  final String version = HConstants.FILE_SYSTEM_VERSION;
  s.writeUTF(version);
  s.close();
  assertTrue(fs.exists(versionFile));
  FileStatus [] status = fs.listStatus(versionFile);
  assertNotNull(status);
  assertTrue(status.length > 0);
  String newVersion = FSUtils.getVersion(fs, rootdir);
  assertEquals(version.length(), newVersion.length());
  assertEquals(version, newVersion);
  // File will have been converted. Exercise the pb format
  assertEquals(version, FSUtils.getVersion(fs, rootdir));
  FSUtils.checkVersion(fs, rootdir, true);
}

@Override
public void testWithMockedMapReduce(HBaseTestingUtility util, String snapshotName,
    int numRegions, int expectedNumSplits) throws Exception {
  setupCluster();
  TableName tableName = TableName.valueOf("testWithMockedMapReduce");
  try {
    createTableAndSnapshot(
      util, tableName, snapshotName, getStartRow(), getEndRow(), numRegions);

    Job job = new Job(util.getConfiguration());
    Path tmpTableDir = util.getRandomDir();
    Scan scan = new Scan(getStartRow(), getEndRow()); // limit the scan

    TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName,
        scan, TestTableSnapshotMapper.class, ImmutableBytesWritable.class,
        NullWritable.class, job, false, tmpTableDir);

    verifyWithMockedMapReduce(job, numRegions, expectedNumSplits, getStartRow(), getEndRow());

  } finally {
    util.getHBaseAdmin().deleteSnapshot(snapshotName);
    util.deleteTable(tableName);
    tearDownCluster();
  }
}

@Test
public void testDeleteAndExists() throws Exception {
  HBaseTestingUtility htu = new HBaseTestingUtility();
  Configuration conf = htu.getConfiguration();
  conf.setBoolean(HConstants.ENABLE_DATA_FILE_UMASK, true);
  FileSystem fs = FileSystem.get(conf);
  FsPermission perms = FSUtils.getFilePermissions(fs, conf, HConstants.DATA_FILE_UMASK_KEY);
  // then that the correct file is created
  String file = UUID.randomUUID().toString();
  Path p = new Path(htu.getDataTestDir(), "temptarget" + File.separator + file);
  Path p1 = new Path(htu.getDataTestDir(), "temppath" + File.separator + file);
  try {
    FSDataOutputStream out = FSUtils.create(conf, fs, p, perms, null);
    out.close();
    assertTrue("The created file should be present", FSUtils.isExists(fs, p));
    // delete the file with recursion as false. Only the file will be deleted.
    FSUtils.delete(fs, p, false);
    // Create another file
    FSDataOutputStream out1 = FSUtils.create(conf, fs, p1, perms, null);
    out1.close();
    // delete the file with recursion as false. Still the file only will be deleted
    FSUtils.delete(fs, p1, true);
    assertFalse("The created file should be present", FSUtils.isExists(fs, p1));
    // and then cleanup
  } finally {
    FSUtils.delete(fs, p, true);
    FSUtils.delete(fs, p1, true);
  }
}

@BeforeClass
public static void setupClass() throws Exception {
  utility = new HBaseTestingUtility();
  Path dataTestDir = utility.getDataTestDir().getParent();
  int length = dataTestDir.toString().length();
  if (length > TEST_DIRECTORY_MAX_LENGTH) {
    System.err.println(TEST_DIRECTORY_INVALID_MESSAGE);
    System.err.println("Current HBase test directory: " + dataTestDir);
    throw new RuntimeException(TEST_DIRECTORY_INVALID_MESSAGE);
  }
  utility.startMiniCluster();
}

@BeforeClass
public static void initCluster() throws Exception {
  assumeNonMaprProfile();
  if (initCount.get() == 0) {
    synchronized (HBaseTestsSuite.class) {
      if (initCount.get() == 0) {
        conf = HBaseConfiguration.create();
        conf.set(HConstants.HBASE_CLIENT_INSTANCE_ID, "dremio-hbase-unit-tests-client");
        if (IS_DEBUG) {
          conf.set("hbase.client.scanner.timeout.period","10000000");
        }

        if (manageHBaseCluster) {
          logger.info("Starting HBase mini cluster.");
          UTIL = new HBaseTestingUtility(conf);
          UTIL.startMiniZKCluster();
          String old_home = System.getProperty("user.home");
          System.setProperty("user.home", UTIL.getDataTestDir().toString());
          UTIL.startMiniHBaseCluster(1, 1);
          System.setProperty("user.home", old_home);
          hbaseClusterCreated = true;
          logger.info("HBase mini cluster started. Zookeeper port: '{}'", getZookeeperPort());
        }

        conn = ConnectionFactory.createConnection(conf);
        admin = conn.getAdmin();

        if (createTables || !tablesExist()) {
          createTestTables();
          tablesCreated = true;
        }
        initCount.incrementAndGet();
        return;
      }
    }
  }
  initCount.incrementAndGet();
}

/**
 * Getting data from memstore and files
 * @throws IOException
 */
@Test
public void testGet_FromMemStoreAndFiles() throws IOException {
  init(this.name.getMethodName());

  //Put data in memstore
  this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
  this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
  //flush
  flush(1);

  //Add more data
  this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
  this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
  //flush
  flush(2);

  //Add more data
  this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
  this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));

  //Get
  result = HBaseTestingUtility.getFromStoreFile(store,
      get.getRow(), qualifiers);

  //Need to sort the result since multiple files
  Collections.sort(result, KeyValue.COMPARATOR);

  //Compare
  assertCheck();
}

public static Path getStoreArchivePath(HBaseTestingUtility util, String tableName,
    byte[] storeName) throws IOException {
  byte[] table = Bytes.toBytes(tableName);
  // get the RS and region serving our table
  List<HRegion> servingRegions = util.getHBaseCluster().getRegions(table);
  HRegion region = servingRegions.get(0);

  // check that we actually have some store files that were archived
  Store store = region.getStore(storeName);
  return HFileArchiveTestingUtil.getStoreArchivePath(util.getConfiguration(), region, store);
}

/**
 * Getting data from files only
 * @throws IOException
 */
@Test
public void testGet_FromFilesOnly() throws IOException {
  init(this.name.getMethodName());

  //Put data in memstore
  this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
  this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
  //flush
  flush(1);

  //Add more data
  this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
  this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
  //flush
  flush(2);

  //Add more data
  this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
  this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
  //flush
  flush(3);

  //Get
  result = HBaseTestingUtility.getFromStoreFile(store,
      get.getRow(),
      qualifiers);
  //this.store.get(get, qualifiers, result);

  //Need to sort the result since multiple files
  Collections.sort(result, KeyValue.COMPARATOR);

  //Compare
  assertCheck();
}

@Test (timeout = 240000)
public void testRegionReplicaWithoutMemstoreReplication() throws Exception {
  int regionReplication = 3;
  TableName tableName = TableName.valueOf("testRegionReplicaWithoutMemstoreReplication");
  HTableDescriptor htd = HTU.createTableDescriptor(tableName.toString());
  htd.setRegionReplication(regionReplication);
  htd.setRegionMemstoreReplication(false);
  HTU.getHBaseAdmin().createTable(htd);

  Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration());
  Table table = connection.getTable(tableName);
  try {
    // write data to the primary. The replicas should not receive the data
    final int STEP = 100;
    for (int i = 0; i < 3; ++i) {
      final int startRow = i * STEP;
      final int endRow = (i + 1) * STEP;
      LOG.info("Writing data from " + startRow + " to " + endRow);
      HTU.loadNumericRows(table, HBaseTestingUtility.fam1, startRow, endRow);
      verifyReplication(tableName, regionReplication, startRow, endRow, false);

      // Flush the table, now the data should show up in the replicas
      LOG.info("flushing table");
      HTU.flush(tableName);
      verifyReplication(tableName, regionReplication, 0, endRow, true);
    }
  } finally {
    table.close();
    connection.close();
  }
}

private WAL createWAL(final Configuration c) throws IOException {
  FSHLog wal = new FSHLog(FileSystem.get(c), hbaseRootDir, logName, c);
  // Set down maximum recovery so we dfsclient doesn't linger retrying something
  // long gone.
  HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1);
  return wal;
}

@Override
public void perform() throws Exception {
  HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
  Admin admin = util.getHBaseAdmin();

  LOG.info("Performing action: Split random region of table " + tableName);
  List<HRegionInfo> regions = admin.getTableRegions(tableName);
  if (regions == null || regions.isEmpty()) {
    LOG.info("Table " + tableName + " doesn't have regions to split");
    return;
  }
  // Don't try the split if we're stopping
  if (context.isStopping()) {
    return;
  }

  HRegionInfo region = PolicyBasedChaosMonkey.selectRandomItem(
      regions.toArray(new HRegionInfo[regions.size()]));
  LOG.debug("Splitting region " + region.getRegionNameAsString());
  try {
    admin.splitRegion(region.getRegionName());
  } catch (Exception ex) {
    LOG.warn("Split failed, might be caused by other chaos: " + ex.getMessage());
  }
  if (sleepTime > 0) {
    Thread.sleep(sleepTime);
  }
}

public static void deleteArchiveDirectory(final HBaseTestingUtility util)
    throws IOException {
  // Ensure the archiver to be empty
  MasterFileSystem mfs = util.getMiniHBaseCluster().getMaster().getMasterFileSystem();
  Path archiveDir = new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
  mfs.getFileSystem().delete(archiveDir, true);
}

public void startHBase() throws IOException {
  startDaemonLogTailer();
  cleanupOldState();

  // start ZK
  LOG.info("Starting ZooKeeper on port " + zkClientPort);
  startZK();

  HBaseTestingUtility.waitForHostPort(HConstants.LOCALHOST, zkClientPort);

  for (int masterPort : masterPorts) {
    startMaster(masterPort);
  }

  ZKUtil.waitForBaseZNode(conf);

  for (int rsPort : rsPorts) {
    startRegionServer(rsPort);
  }

  LOG.info("Waiting for HBase startup by scanning META");
  int attemptsLeft = 10;
  while (attemptsLeft-- > 0) {
    try {
      new HTable(conf, TableName.META_TABLE_NAME);
    } catch (Exception e) {
      LOG.info("Waiting for HBase to startup. Retries left: " + attemptsLeft,
          e);
      Threads.sleep(1000);
    }
  }

  LOG.info("Process-based HBase Cluster with " + numRegionServers +
      " region servers up and running... \n\n");
}

/**
 * Test, on a local filesystem, that the FileLink is still readable
 * even when the current file gets renamed.
 */
@Test
public void testLocalLinkReadDuringRename() throws IOException {
  HBaseTestingUtility testUtil = new HBaseTestingUtility();
  FileSystem fs = testUtil.getTestFileSystem();
  assertEquals("file", fs.getUri().getScheme());
  testLinkReadDuringRename(fs, testUtil.getDataTestDir());
}

/**
 * @param htu Testing utility to use
 * @param zkw If true, create a zkw.
 * @throws ZooKeeperConnectionException
 * @throws IOException
 */
public MockServer(final HBaseTestingUtility htu, final boolean zkw)
throws ZooKeeperConnectionException, IOException {
  this.htu = htu;
  this.zk = zkw?
    new ZooKeeperWatcher(htu.getConfiguration(), NAME.toString(), this, true):
    null;
}

@Override
public void perform() throws Exception {
  Random random = new Random();
  HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
  Admin admin = util.getHBaseAdmin();

  LOG.info("Performing action: Change bloom filter on all columns of table "
      + tableName);
  HTableDescriptor tableDescriptor = admin.getTableDescriptor(tableName);
  HColumnDescriptor[] columnDescriptors = tableDescriptor.getColumnFamilies();

  if (columnDescriptors == null || columnDescriptors.length == 0) {
    return;
  }

  final BloomType[] bloomArray = BloomType.values();
  final int bloomArraySize = bloomArray.length;

  for (HColumnDescriptor descriptor : columnDescriptors) {
    int bloomFilterIndex = random.nextInt(bloomArraySize);
    LOG.debug("Performing action: About to set bloom filter type to "
        + bloomArray[bloomFilterIndex] + " on column "
        + descriptor.getNameAsString() + " of table " + tableName);
    descriptor.setBloomFilterType(bloomArray[bloomFilterIndex]);
    LOG.debug("Performing action: Just set bloom filter type to "
        + bloomArray[bloomFilterIndex] + " on column "
        + descriptor.getNameAsString() + " of table " + tableName);
  }

  // Don't try the modify if we're stopping
  if (context.isStopping()) {
    return;
  }
  admin.modifyTable(tableName, tableDescriptor);
}

public static void checkTablePerms(HBaseTestingUtility testUtil, TableName table, byte[] family,
    byte[] column, Permission.Action... actions) throws IOException {
  Permission[] perms = new Permission[actions.length];
  for (int i = 0; i < actions.length; i++) {
    perms[i] = new TablePermission(table, family, column, actions[i]);
  }
  checkTablePerms(testUtil, table, perms);
}

public static void confirmSnapshotValid(HBaseTestingUtility testUtil,
    SnapshotDescription snapshotDescriptor, TableName tableName, byte[] family)
    throws IOException {
  MasterFileSystem mfs = testUtil.getHBaseCluster().getMaster().getMasterFileSystem();
  confirmSnapshotValid(snapshotDescriptor, tableName, family,
      mfs.getRootDir(), testUtil.getHBaseAdmin(), mfs.getFileSystem());
}

/**
 * Test for hbase-1686.
 * @throws IOException
 */
@Test
public void testEmptyStoreFile() throws IOException {
  init(this.name.getMethodName());
  // Write a store file.
  this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
  this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
  flush(1);
  // Now put in place an empty store file.  Its a little tricky.  Have to
  // do manually with hacked in sequence id.
  StoreFile f = this.store.getStorefiles().iterator().next();
  Path storedir = f.getPath().getParent();
  long seqid = f.getMaxSequenceId();
  Configuration c = HBaseConfiguration.create();
  FileSystem fs = FileSystem.get(c);
  HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).build();
  StoreFile.Writer w = new StoreFile.WriterBuilder(c, new CacheConfig(c),
      fs)
          .withOutputDir(storedir)
          .withFileContext(meta)
          .build();
  w.appendMetadata(seqid + 1, false);
  w.close();
  this.store.close();
  // Reopen it... should pick up two files
  this.store = new HStore(this.store.getHRegion(), this.store.getFamily(), c);
  Assert.assertEquals(2, this.store.getStorefilesCount());

  result = HBaseTestingUtility.getFromStoreFile(store,
      get.getRow(),
      qualifiers);
  Assert.assertEquals(1, result.size());
}

@Before
public void setup() throws IOException {
  TEST_UTIL = HBaseTestingUtility.createLocalHTU();
  CONF = TEST_UTIL.getConfiguration();
  // Disable block cache.
  CONF.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f);
  dir = TEST_UTIL.getDataTestDir("TestHRegion").toString();
  tableName = TableName.valueOf(name.getMethodName());
}