@Test (timeout=300000) public void testCreateTableCalledTwiceAndFirstOneInProgress() throws Exception { final TableName tableName = TableName.valueOf("testCreateTableCalledTwiceAndFirstOneInProgress"); final MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); final HMaster m = cluster.getMaster(); final HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(new HColumnDescriptor(FAMILYNAME)); final HRegionInfo[] hRegionInfos = new HRegionInfo[] { new HRegionInfo(desc.getTableName(), null, null) }; CustomCreateTableHandler handler = new CustomCreateTableHandler(m, m.getMasterFileSystem(), desc, cluster.getConfiguration(), hRegionInfos, m); handler.prepare(); throwException = true; handler.process(); throwException = false; CustomCreateTableHandler handler1 = new CustomCreateTableHandler(m, m.getMasterFileSystem(), desc, cluster.getConfiguration(), hRegionInfos, m); handler1.prepare(); handler1.process(); for (int i = 0; i < 100; i++) { if (!TEST_UTIL.getHBaseAdmin().isTableAvailable(tableName)) { Thread.sleep(200); } } assertTrue(TEST_UTIL.getHBaseAdmin().isTableEnabled(tableName)); }
public void disableTable(Connection connection, TableName tableName) throws IOException { Admin admin = null; try { admin = connection.getAdmin(); if(admin.tableExists(tableName)){ admin.disableTable(tableName); } } finally { if(admin!=null) { admin.close(); } } }
@Override public KeyValueScanner preStoreScannerOpen( final ObserverContext<RegionCoprocessorEnvironment> c, Store store, final Scan scan, final NavigableSet<byte[]> targetCols, KeyValueScanner s) throws IOException { TableName tn = store.getTableName(); if (!tn.isSystemTable()) { Long newTtl = ttls.get(store.getTableName()); Integer newVersions = versions.get(store.getTableName()); ScanInfo oldSI = store.getScanInfo(); HColumnDescriptor family = store.getFamily(); ScanInfo scanInfo = new ScanInfo(TEST_UTIL.getConfiguration(), family.getName(), family.getMinVersions(), newVersions == null ? family.getMaxVersions() : newVersions, newTtl == null ? oldSI.getTtl() : newTtl, family.getKeepDeletedCells(), oldSI.getTimeToPurgeDeletes(), oldSI.getComparator()); return new StoreScanner(store, scanInfo, scan, targetCols, ((HStore) store).getHRegion().getReadpoint(IsolationLevel.READ_COMMITTED)); } else { return s; } }
@Override public void postModifyTable(ObserverContext<MasterCoprocessorEnvironment> c, TableName tableName, final HTableDescriptor htd) throws IOException { final Configuration conf = c.getEnvironment().getConfiguration(); // default the table owner to current user, if not specified. final String owner = (htd.getOwnerString() != null) ? htd.getOwnerString() : getActiveUser().getShortName(); User.runAsLoginUser(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { UserPermission userperm = new UserPermission(Bytes.toBytes(owner), htd.getTableName(), null, Action.values()); AccessControlLists.addUserPermission(conf, userperm); return null; } }); }
private void createColumnFamily(HColumnDescriptor family, TableName table) throws IOException { try { admin.addColumn(table, family); } catch (InvalidFamilyOperationException e) { if (!hasFamily(family, table)) { //Schroedinger's cat: InvalidFamilyOperationException (cf exists) but does not exist at the same time throw new IllegalStateException("Column family should exist but does not", e); } //columnFamily was created in the meantime return; } waitForColumnFamilyCreation(family, table); log.info("Created column family '{}' in HBase table '{}'", family.getNameAsString(), table.getNameAsString()); }
/** * create map based on statDesc * 1. for set, "family qualifier DataType set [v1] [v2] [...]" * 2. for array, "family qualifier DataType min max parts" */ public static Map<TableName, LCStatInfo2> parseStatString(IndexTableRelation indexTableRelation, String statDesc) throws IOException { Map<TableName, LCStatInfo2> map = new HashMap<>(); String[] lines = statDesc.split(LC_TABLE_DESC_RANGE_DELIMITER); for (String line : lines) { String[] parts = line.split("\t"); byte[] family = Bytes.toBytes(parts[0]); byte[] qualifier = Bytes.toBytes(parts[1]); TableName tableName = indexTableRelation.getIndexTableName(family, qualifier); LCStatInfo2 statInfo; try { if ("set".equalsIgnoreCase(parts[3])) { statInfo = new LCStatInfo2(family, qualifier, DataType.valueOf(parts[2]), parts, 4); } else { statInfo = new LCStatInfo2(family, qualifier, DataType.valueOf(parts[2]), Integer.valueOf(parts[5]), parts[3], parts[4]); } } catch (IOException e) { throw new IOException("exception for parsing line: " + line, e); } map.put(tableName, statInfo); } return map; }
/** * Private constructor */ private HBaseEventStoreTable(final EventStoreTimeIntervalEnum timeInterval, final StroomPropertyService propertyService, final HBaseConnection hBaseConnection, final UniqueIdCache uniqueIdCache, final StatisticDataPointAdapterFactory statisticDataPointAdapterFactory) { super(hBaseConnection); this.displayName = timeInterval.longName() + DISPLAY_NAME_POSTFIX; this.tableName = TableName.valueOf(Bytes.toBytes(timeInterval.shortName() + TABLE_NAME_POSTFIX)); this.timeInterval = timeInterval; this.propertyService = propertyService; this.rowKeyBuilder = new SimpleRowKeyBuilder(uniqueIdCache, timeInterval); this.statisticDataPointAdapterFactory = statisticDataPointAdapterFactory; for (StatisticType statisticType : StatisticType.values()) { putCounterMap.put(statisticType, new LongAdder()); } init(); }
@Override protected Map<TableName, Put> parsePut(Put put, boolean serverSide) { Map<TableName, Put> map = new HashMap<>(); byte[] row = put.getRow(); for (Map.Entry<byte[], List<Cell>> entry : put.getFamilyCellMap().entrySet()) { byte[] family = entry.getKey(); for (Cell cell : entry.getValue()) { byte[] q = CellUtil.cloneQualifier(cell); if (tableRelation.isIndexColumn(family, q)) { TableName indexTableName = tableRelation.getIndexTableName(family, q); Put newPut = new Put(getIndexRow(row, CellUtil.cloneValue(cell))); if (serverSide) newPut .addColumn(IndexType.SEDONDARY_FAMILY_BYTES, (byte[]) null, cell.getTimestamp(), null); else newPut.addColumn(IndexType.SEDONDARY_FAMILY_BYTES, null, null); map.put(indexTableName, newPut); } } } tableRelation.getIndexFamilyMap(); return map; }
private static Table createTableAndWriteDataWithLabels(TableName tableName, String... labelExps) throws Exception { Table table = null; try { table = TEST_UTIL.createTable(tableName, fam); int i = 1; List<Put> puts = new ArrayList<Put>(); for (String labelExp : labelExps) { Put put = new Put(Bytes.toBytes("row" + i)); put.add(fam, qual, HConstants.LATEST_TIMESTAMP, value); put.setCellVisibility(new CellVisibility(labelExp)); puts.add(put); i++; } table.put(puts); } finally { if (table != null) { table.close(); } } return table; }
@Override public void setup(Context context) throws IOException { conf = context.getConfiguration(); recordsToWrite = conf.getLong(NUM_TO_WRITE_KEY, NUM_TO_WRITE_DEFAULT); String tableName = conf.get(TABLE_NAME_KEY, TABLE_NAME_DEFAULT); numBackReferencesPerRow = conf.getInt(NUM_BACKREFS_KEY, NUM_BACKREFS_DEFAULT); this.connection = ConnectionFactory.createConnection(conf); mutator = connection.getBufferedMutator( new BufferedMutatorParams(TableName.valueOf(tableName)) .writeBufferSize(4 * 1024 * 1024)); String taskId = conf.get("mapreduce.task.attempt.id"); Matcher matcher = Pattern.compile(".+_m_(\\d+_\\d+)").matcher(taskId); if (!matcher.matches()) { throw new RuntimeException("Strange task ID: " + taskId); } shortTaskId = matcher.group(1); rowsWritten = context.getCounter(Counters.ROWS_WRITTEN); refsWritten = context.getCounter(Counters.REFERENCES_WRITTEN); }
@Test public void testMaxKeyValueSize() throws Exception { byte [] TABLE = Bytes.toBytes("testMaxKeyValueSize"); Configuration conf = TEST_UTIL.getConfiguration(); String oldMaxSize = conf.get(ConnectionConfiguration.MAX_KEYVALUE_SIZE_KEY); Table ht = TEST_UTIL.createTable(TABLE, FAMILY); byte[] value = new byte[4 * 1024 * 1024]; Put put = new Put(ROW); put.add(FAMILY, QUALIFIER, value); ht.put(put); try { TEST_UTIL.getConfiguration().setInt( ConnectionConfiguration.MAX_KEYVALUE_SIZE_KEY, 2 * 1024 * 1024); // Create new table so we pick up the change in Configuration. try (Connection connection = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())) { try (Table t = connection.getTable(TableName.valueOf(FAMILY))) { put = new Put(ROW); put.add(FAMILY, QUALIFIER, value); t.put(put); } } fail("Inserting a too large KeyValue worked, should throw exception"); } catch(Exception e) {} conf.set(ConnectionConfiguration.MAX_KEYVALUE_SIZE_KEY, oldMaxSize); }
/** * Grant permissions on a table to the given user using AccessControlClient. Will wait until all * active AccessController instances have updated their permissions caches or will * throw an exception upon timeout (10 seconds). */ public static void grantOnTableUsingAccessControlClient(final HBaseTestingUtility util, final Connection connection, final String user, final TableName table, final byte[] family, final byte[] qualifier, final Permission.Action... actions) throws Exception { SecureTestUtil.updateACLs(util, new Callable<Void>() { @Override public Void call() throws Exception { try { AccessControlClient.grant(connection, table, user, family, qualifier, actions); } catch (Throwable t) { t.printStackTrace(); } return null; } }); }
protected static void assignRegions(final MasterProcedureEnv env, final TableName tableName, final List<HRegionInfo> regions) throws HBaseException, IOException { ProcedureSyncWait.waitRegionServers(env); final AssignmentManager assignmentManager = env.getMasterServices().getAssignmentManager(); // Mark the table as Enabling assignmentManager.getTableStateManager().setTableState(tableName, ZooKeeperProtos.Table.State.ENABLING); // Trigger immediate assignment of the regions in round-robin fashion ModifyRegionUtils.assignRegions(assignmentManager, regions); // Enable table assignmentManager.getTableStateManager() .setTableState(tableName, ZooKeeperProtos.Table.State.ENABLED); }
private void createTable(Admin admin, TableName tableName, boolean setVersion, boolean acl) throws IOException { if (!admin.tableExists(tableName)) { HTableDescriptor htd = new HTableDescriptor(tableName); HColumnDescriptor family = new HColumnDescriptor(FAMILY_NAME); if (setVersion) { family.setMaxVersions(DEFAULT_TABLES_COUNT); } htd.addFamily(family); admin.createTable(htd); if (acl) { LOG.info("Granting permissions for user " + USER.getShortName()); Permission.Action[] actions = { Permission.Action.READ }; try { AccessControlClient.grant(ConnectionFactory.createConnection(getConf()), tableName, USER.getShortName(), null, null, actions); } catch (Throwable e) { LOG.fatal("Error in granting permission for the user " + USER.getShortName(), e); throw new IOException(e); } } } }
@Override public <T> List<T> find(TableName tableName, final List<Scan> scanList, final ResultsExtractor<T> action) { assertAccessAvailable(); return execute(tableName, new TableCallback<List<T>>() { @Override public List<T> doInTable(Table table) throws Throwable { List<T> result = new ArrayList<>(scanList.size()); for (Scan scan : scanList) { final ResultScanner scanner = table.getScanner(scan); try { T t = action.extractData(scanner); result.add(t); } finally { scanner.close(); } } return result; } }); }
/** * Test creating a pre-split table using the UniformSplit algorithm. */ @Test public void testCreatePresplitTableUniform() throws Exception { List<byte[]> expectedBounds = new ArrayList<byte[]>(); expectedBounds.add(ArrayUtils.EMPTY_BYTE_ARRAY); expectedBounds.add(new byte[] { 0x10, 0, 0, 0, 0, 0, 0, 0}); expectedBounds.add(new byte[] { 0x20, 0, 0, 0, 0, 0, 0, 0}); expectedBounds.add(new byte[] { 0x30, 0, 0, 0, 0, 0, 0, 0}); expectedBounds.add(new byte[] { 0x40, 0, 0, 0, 0, 0, 0, 0}); expectedBounds.add(new byte[] { 0x50, 0, 0, 0, 0, 0, 0, 0}); expectedBounds.add(new byte[] { 0x60, 0, 0, 0, 0, 0, 0, 0}); expectedBounds.add(new byte[] { 0x70, 0, 0, 0, 0, 0, 0, 0}); expectedBounds.add(new byte[] {(byte)0x80, 0, 0, 0, 0, 0, 0, 0}); expectedBounds.add(new byte[] {(byte)0x90, 0, 0, 0, 0, 0, 0, 0}); expectedBounds.add(new byte[] {(byte)0xa0, 0, 0, 0, 0, 0, 0, 0}); expectedBounds.add(new byte[] {(byte)0xb0, 0, 0, 0, 0, 0, 0, 0}); expectedBounds.add(new byte[] {(byte)0xc0, 0, 0, 0, 0, 0, 0, 0}); expectedBounds.add(new byte[] {(byte)0xd0, 0, 0, 0, 0, 0, 0, 0}); expectedBounds.add(new byte[] {(byte)0xe0, 0, 0, 0, 0, 0, 0, 0}); expectedBounds.add(new byte[] {(byte)0xf0, 0, 0, 0, 0, 0, 0, 0}); expectedBounds.add(ArrayUtils.EMPTY_BYTE_ARRAY); // Do table creation/pre-splitting and verification of region boundaries preSplitTableAndVerify(expectedBounds, UniformSplit.class.getSimpleName(), TableName.valueOf("NewUniformPresplitTable")); }
/** * Take a snapshot using the specified handler. * On failure the snapshot temporary working directory is removed. * NOTE: prepareToTakeSnapshot() called before this one takes care of the rejecting the * snapshot request if the table is busy with another snapshot/restore operation. * @param snapshot the snapshot description * @param handler the snapshot handler */ private synchronized void snapshotTable(SnapshotDescription snapshot, final TakeSnapshotHandler handler) throws HBaseSnapshotException { try { handler.prepare(); this.executorService.submit(handler); this.snapshotHandlers.put(TableName.valueOf(snapshot.getTable()), handler); } catch (Exception e) { // cleanup the working directory by trying to delete it from the fs. Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir); try { if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) { LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" + ClientSnapshotDescriptionUtils.toString(snapshot)); } } catch (IOException e1) { LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" + ClientSnapshotDescriptionUtils.toString(snapshot)); } // fail the snapshot throw new SnapshotCreationException("Could not build snapshot handler", e, snapshot); } }
@Test public void testMasterRestartAfterNameSpaceEnablingNodeIsCreated() throws Exception { // Step 1: start mini zk cluster. MiniZooKeeperCluster zkCluster; zkCluster = TEST_UTIL.startMiniZKCluster(); // Step 2: add an orphaned system table ZNODE TableName tableName = TableName.valueOf("hbase:namespace"); ZooKeeperWatcher zkw = TEST_UTIL.getZooKeeperWatcher(); String znode = ZKUtil.joinZNode(zkw.tableZNode, tableName.getNameAsString()); ZooKeeperProtos.Table.Builder builder = ZooKeeperProtos.Table.newBuilder(); builder.setState(ZooKeeperProtos.Table.State.ENABLED); byte [] data = ProtobufUtil.prependPBMagic(builder.build().toByteArray()); ZKUtil.createSetData(zkw, znode, data); LOG.info("Create an orphaned Znode " + znode + " with data " + data); // Step 3: link the zk cluster to hbase cluster TEST_UTIL.setZkCluster(zkCluster); // Step 4: start hbase cluster and expect master to start successfully. TEST_UTIL.startMiniCluster(); assertTrue(TEST_UTIL.getHBaseCluster().getLiveMasterThreads().size() == 1); }
Region initHRegion(byte[] tableName, String callingMethod, Configuration conf, byte[]... families) throws IOException { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); for (byte[] family : families) { htd.addFamily(new HColumnDescriptor(family)); } HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); Path path = new Path(DIR + callingMethod); HRegion r = HRegion.createHRegion(info, path, conf, htd); // this following piece is a hack. currently a coprocessorHost // is secretly loaded at OpenRegionHandler. we don't really // start a region server here, so just manually create cphost // and set it to region. RegionCoprocessorHost host = new RegionCoprocessorHost(r, null, conf); r.setCoprocessorHost(host); return r; }
/** * Lists all of the table regions currently in META. * @param connection * @param tableName * @return Map of all user-space regions to servers * @throws IOException */ public static NavigableMap<HRegionInfo, ServerName> allTableRegions( Connection connection, final TableName tableName) throws IOException { final NavigableMap<HRegionInfo, ServerName> regions = new TreeMap<HRegionInfo, ServerName>(); MetaScannerVisitor visitor = new TableMetaScannerVisitor(tableName) { @Override public boolean processRowInternal(Result result) throws IOException { RegionLocations locations = MetaTableAccessor.getRegionLocations(result); if (locations == null) return true; for (HRegionLocation loc : locations.getRegionLocations()) { if (loc != null) { HRegionInfo regionInfo = loc.getRegionInfo(); regions.put(new UnmodifyableHRegionInfo(regionInfo), loc.getServerName()); } } return true; } }; metaScan(connection, visitor, tableName); return regions; }
private static void waitUntilAllRegionsAssigned() throws IOException { HTable meta = new HTable(TEST_UTIL.getConfiguration(), TableName.META_TABLE_NAME); while (true) { int rows = 0; Scan scan = new Scan(); scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER); ResultScanner s = meta.getScanner(scan); for (Result r = null; (r = s.next()) != null;) { byte [] b = r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER); if (b == null || b.length <= 0) { break; } rows++; } s.close(); // If I get to here and all rows have a Server, then all have been assigned. if (rows >= countOfRegions) { break; } LOG.info("Found=" + rows); Threads.sleep(1000); } meta.close(); }
private void runTest(String testName, BloomType bloomType, byte[][] tableSplitKeys, byte[][][] hfileRanges) throws Exception { final byte[] TABLE_NAME = Bytes.toBytes("mytable_"+testName); final boolean preCreateTable = tableSplitKeys != null; // Run the test bulkloading the table to the default namespace final TableName TABLE_WITHOUT_NS = TableName.valueOf(TABLE_NAME); runTest(testName, TABLE_WITHOUT_NS, bloomType, preCreateTable, tableSplitKeys, hfileRanges); // Run the test bulkloading the table to the specified namespace final TableName TABLE_WITH_NS = TableName.valueOf(Bytes.toBytes(NAMESPACE), TABLE_NAME); runTest(testName, TABLE_WITH_NS, bloomType, preCreateTable, tableSplitKeys, hfileRanges); }
@Test public void testNoSuchTable() throws IOException { final String name = "testNoSuchTable"; FileSystem fs = FileSystem.get(UTIL.getConfiguration()); // Cleanup old tests if any detrius laying around. Path rootdir = new Path(UTIL.getDataTestDir(), name); TableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir); assertNull("There shouldn't be any HTD for this table", htds.get(TableName.valueOf("NoSuchTable"))); }
/** * @param tableName * the name of the table, as a string * @return the named mutator * @throws IOException * if there is a problem opening a table */ BufferedMutator getBufferedMutator(ImmutableBytesWritable tableName) throws IOException { if(this.connection == null){ this.connection = ConnectionFactory.createConnection(conf); } if (!mutatorMap.containsKey(tableName)) { LOG.debug("Opening HTable \"" + Bytes.toString(tableName.get())+ "\" for writing"); BufferedMutator mutator = connection.getBufferedMutator(TableName.valueOf(tableName.get())); mutatorMap.put(tableName, mutator); } return mutatorMap.get(tableName); }
/** * Adds (or updates) the table descriptor to the FileSystem * and updates the local cache with it. */ @Override public void add(HTableDescriptor htd) throws IOException { if (fsreadonly) { throw new NotImplementedException("Cannot add a table descriptor - in read only mode"); } if (TableName.META_TABLE_NAME.equals(htd.getTableName())) { throw new NotImplementedException(); } if (HConstants.HBASE_NON_USER_TABLE_DIRS.contains(htd.getTableName().getNameAsString())) { throw new NotImplementedException( "Cannot add a table descriptor for a reserved subdirectory name: " + htd.getNameAsString()); } updateTableDescriptor(htd); }
public void dropTable(TableName tableName) { try { this.admin.disableTable(tableName); this.admin.deleteTable(tableName); } catch (IOException e) { throw new HbaseSystemException(e); } }
@Override public void postDeleteTableHandler( final ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName) throws IOException { // the AccessController test, some times calls only and directly the postDeleteTableHandler() if (tableDeletionLatch != null) { tableDeletionLatch.countDown(); } }
static int insertData(Configuration conf, TableName tableName, String column, double prob) throws IOException { Random rng = new Random(); byte[] k = new byte[3]; byte [][] famAndQf = KeyValue.parseColumn(Bytes.toBytes(column)); List<Put> puts = new ArrayList<>(); for (byte b1 = 'a'; b1 < 'z'; b1++) { for (byte b2 = 'a'; b2 < 'z'; b2++) { for (byte b3 = 'a'; b3 < 'z'; b3++) { if (rng.nextDouble() < prob) { k[0] = b1; k[1] = b2; k[2] = b3; Put put = new Put(k); put.setDurability(Durability.SKIP_WAL); put.add(famAndQf[0], famAndQf[1], k); puts.add(put); } } } } try (Connection conn = ConnectionFactory.createConnection(conf); Table table = conn.getTable(tableName)) { table.put(puts); } return puts.size(); }
public void flushAll(TableName tableName) throws IOException, InterruptedException { HTableDescriptor desc = admin.getTableDescriptor(tableName); if (isIndexTable(desc)) { throw new TableNotFoundException(tableName); } IndexTableDescriptor indexDesc = new IndexTableDescriptor(desc); if (indexDesc.hasIndex()) { for (IndexSpecification indexSpec : indexDesc.getIndexSpecifications()) { admin.flush(indexSpec.getIndexTableName()); } } admin.flush(tableName); }
@Test public void testDelete_CheckTimestampUpdated() throws IOException { TableName tableName = TableName.valueOf(name.getMethodName()); byte[] row1 = Bytes.toBytes("row1"); byte[] col1 = Bytes.toBytes("col1"); byte[] col2 = Bytes.toBytes("col2"); byte[] col3 = Bytes.toBytes("col3"); // Setting up region String method = this.getName(); this.region = initHRegion(tableName, method, CONF, fam1); try { // Building checkerList List<Cell> kvs = new ArrayList<Cell>(); kvs.add(new KeyValue(row1, fam1, col1, null)); kvs.add(new KeyValue(row1, fam1, col2, null)); kvs.add(new KeyValue(row1, fam1, col3, null)); NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>( Bytes.BYTES_COMPARATOR); deleteMap.put(fam1, kvs); region.delete(deleteMap, Durability.SYNC_WAL); // extract the key values out the memstore: // This is kinda hacky, but better than nothing... long now = System.currentTimeMillis(); DefaultMemStore memstore = (DefaultMemStore) ((HStore) region.getStore(fam1)).memstore; Cell firstCell = memstore.cellSet.first(); assertTrue(firstCell.getTimestamp() <= now); now = firstCell.getTimestamp(); for (Cell cell : memstore.cellSet) { assertTrue(cell.getTimestamp() <= now); now = cell.getTimestamp(); } } finally { HRegion.closeHRegion(this.region); this.region = null; } }
/** * Scans the table and merges two adjacent regions if they are small. This * only happens when a lot of rows are deleted. * * When merging the hbase:meta region, the HBase instance must be offline. * When merging a normal table, the HBase instance must be online, but the * table must be disabled. * * @param conf - configuration object for HBase * @param fs - FileSystem where regions reside * @param tableName - Table to be compacted * @param testMasterRunning True if we are to verify master is down before * running merge * @throws IOException */ public static void merge(Configuration conf, FileSystem fs, final TableName tableName, final boolean testMasterRunning) throws IOException { boolean masterIsRunning = false; if (testMasterRunning) { masterIsRunning = HConnectionManager .execute(new HConnectable<Boolean>(conf) { @Override public Boolean connect(HConnection connection) throws IOException { return connection.isMasterRunning(); } }); } if (tableName.equals(TableName.META_TABLE_NAME)) { if (masterIsRunning) { throw new IllegalStateException( "Can not compact hbase:meta table if instance is on-line"); } // TODO reenable new OfflineMerger(conf, fs).process(); } else { if(!masterIsRunning) { throw new IllegalStateException( "HBase instance must be running to merge a normal table"); } Admin admin = new HBaseAdmin(conf); try { if (!admin.isTableDisabled(tableName)) { throw new TableNotDisabledException(tableName); } } finally { admin.close(); } new OnlineMerger(conf, fs, tableName).process(); } }
@Test public void testDynamicCoprocessorTableEndpoint() throws Throwable { final TableName tableName = TableName.valueOf("testDynamicCoprocessorTableEndpoint"); HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(new HColumnDescriptor(TEST_FAMILY)); createTable(desc); desc.addCoprocessor(org.apache.hadoop.hbase.coprocessor.ColumnAggregationEndpoint.class.getName()); updateTable(desc); verifyTable(tableName); }
@Test public void testDeleteTable() throws Exception { String namespace = prefix + "_dummy"; NamespaceDescriptor nspDesc = NamespaceDescriptor.create(namespace) .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "100") .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "3").build(); ADMIN.createNamespace(nspDesc); assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(namespace)); NamespaceTableAndRegionInfo stateInfo = getNamespaceState(nspDesc.getName()); assertNotNull("Namespace state found null for " + namespace, stateInfo); HTableDescriptor tableDescOne = new HTableDescriptor(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table1")); HTableDescriptor tableDescTwo = new HTableDescriptor(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table2")); ADMIN.createTable(tableDescOne); ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 5); stateInfo = getNamespaceState(nspDesc.getName()); assertNotNull("Namespace state found to be null.", stateInfo); assertEquals(2, stateInfo.getTables().size()); assertEquals(5, stateInfo.getRegionCountOfTable(tableDescTwo.getTableName())); assertEquals(6, stateInfo.getRegionCount()); ADMIN.disableTable(tableDescOne.getTableName()); deleteTable(tableDescOne.getTableName()); stateInfo = getNamespaceState(nspDesc.getName()); assertNotNull("Namespace state found to be null.", stateInfo); assertEquals(5, stateInfo.getRegionCount()); assertEquals(1, stateInfo.getTables().size()); ADMIN.disableTable(tableDescTwo.getTableName()); deleteTable(tableDescTwo.getTableName()); ADMIN.deleteNamespace(namespace); stateInfo = getNamespaceState(namespace); assertNull("Namespace state not found to be null.", stateInfo); }
@Test public void testInProcess() throws KeeperException, IOException { TableName tableName = TableName.valueOf("testTable"); SnapshotManager manager = getNewManager(); TakeSnapshotHandler handler = Mockito.mock(TakeSnapshotHandler.class); assertFalse("Manager is in process when there is no current handler", manager.isTakingSnapshot(tableName)); manager.setSnapshotHandlerForTesting(tableName, handler); Mockito.when(handler.isFinished()).thenReturn(false); assertTrue("Manager isn't in process when handler is running", manager.isTakingSnapshot(tableName)); Mockito.when(handler.isFinished()).thenReturn(true); assertFalse("Manager is process when handler isn't running", manager.isTakingSnapshot(tableName)); }
/** * @return the region name */ @XmlAttribute public String getName() { byte [] tableNameAsBytes = Bytes.toBytes(this.table); TableName tableName = TableName.valueOf(tableNameAsBytes); byte [] nameAsBytes = HRegionInfo.createRegionName( tableName, this.startKey, this.id, !tableName.isSystemTable()); return Bytes.toString(nameAsBytes); }
@Override public <T> List<T> find(TableName tableName, final Scan scan, final AbstractRowKeyDistributor rowKeyDistributor, int limit, final RowMapper<T> action, final LimitEventHandler limitEventHandler) { final LimitRowMapperResultsExtractor<T> resultsExtractor = new LimitRowMapperResultsExtractor<>(action, limit, limitEventHandler); return executeDistributedScan(tableName, scan, rowKeyDistributor, resultsExtractor); }
@Test public void testMROnTable() throws Exception { String tableName = "test-" + UUID.randomUUID(); // Prepare the arguments required for the test. String[] args = new String[] { "-D" + ImportTsv.MAPPER_CONF_KEY + "=org.apache.hadoop.hbase.mapreduce.TsvImporterCustomTestMapperForOprAttr", "-D" + ImportTsv.COLUMNS_CONF_KEY + "=HBASE_ROW_KEY,FAM:A,FAM:B,HBASE_ATTRIBUTES_KEY", "-D" + ImportTsv.SEPARATOR_CONF_KEY + "=\u001b", tableName }; String data = "KEY\u001bVALUE1\u001bVALUE2\u001btest=>myvalue\n"; util.createTable(TableName.valueOf(tableName), FAMILY); doMROnTableTest(util, FAMILY, data, args, 1, true); util.deleteTable(tableName); }
@Override public synchronized boolean isProcedureDone(ProcedureDescription desc) throws IOException { // Procedure instance name is the table name. TableName tableName = TableName.valueOf(desc.getInstance()); Procedure proc = procMap.get(tableName); if (proc == null) { // The procedure has not even been started yet. // The client would request the procedure and call isProcedureDone(). // The HBaseAdmin.execProcedure() wraps both request and isProcedureDone(). return false; } // We reply on the existing Distributed Procedure framework to give us the status. return proc.isCompleted(); }
private RegionLocations locateMeta(final TableName tableName, boolean useCache, int replicaId) throws IOException { // HBASE-10785: We cache the location of the META itself, so that we are not overloading // zookeeper with one request for every region lookup. We cache the META with empty row // key in MetaCache. byte[] metaCacheKey = HConstants.EMPTY_START_ROW; // use byte[0] as the row for meta RegionLocations locations = null; if (useCache) { locations = getCachedLocation(tableName, metaCacheKey); if (locations != null && locations.getRegionLocation(replicaId) != null) { return locations; } } // only one thread should do the lookup. synchronized (metaRegionLock) { // Check the cache again for a hit in case some other thread made the // same query while we were waiting on the lock. if (useCache) { locations = getCachedLocation(tableName, metaCacheKey); if (locations != null && locations.getRegionLocation(replicaId) != null) { return locations; } } // Look up from zookeeper locations = this.registry.getMetaRegionLocation(); if (locations != null) { cacheLocation(tableName, locations); } } return locations; }
protected final <T> T executeParallelDistributedScan(TableName tableName, Scan scan, AbstractRowKeyDistributor rowKeyDistributor, ResultsExtractor<T> action, int numParallelThreads) { assertAccessAvailable(); try { StopWatch watch = null; if (debugEnabled) { watch = new StopWatch(); watch.start(); } ParallelResultScanner scanner = new ParallelResultScanner(tableName, this, this .executor, scan, rowKeyDistributor, numParallelThreads); if (debugEnabled) { logger.debug("ParallelDistributedScanner createTime: {}ms", watch.stop()); watch.start(); } try { return action.extractData(scanner); } finally { scanner.close(); if (debugEnabled) { logger.debug("ParallelDistributedScanner scanTime: {}ms", watch.stop()); } } } catch (Throwable th) { Throwable throwable = th; if (th instanceof ScanTaskException) { throwable = th.getCause(); } if (throwable instanceof Error) { throw ((Error) th); } if (throwable instanceof RuntimeException) { throw ((RuntimeException) th); } throw new HbaseSystemException((Exception) throwable); } }