@InterfaceAudience.Private public Context( final Configuration conf, final FileSystem fs, final ReplicationPeerConfig peerConfig, final String peerId, final UUID clusterId, final ReplicationPeer replicationPeer, final MetricsSource metrics, final TableDescriptors tableDescriptors) { this.peerConfig = peerConfig; this.conf = conf; this.fs = fs; this.clusterId = clusterId; this.peerId = peerId; this.replicationPeer = replicationPeer; this.metrics = metrics; this.tableDescriptors = tableDescriptors; }
public RegionReplicaOutputSink(PipelineController controller, TableDescriptors tableDescriptors, EntryBuffers entryBuffers, ClusterConnection connection, ExecutorService pool, int numWriters, int operationTimeout) { super(controller, entryBuffers, numWriters); this.sinkWriter = new RegionReplicaSinkWriter(this, connection, pool, operationTimeout); this.tableDescriptors = tableDescriptors; // A cache for the table "memstore replication enabled" flag. // It has a default expiry of 5 sec. This means that if the table is altered // with a different flag value, we might miss to replicate for that amount of // time. But this cache avoid the slow lookup and parsing of the TableDescriptor. int memstoreReplicationEnabledCacheExpiryMs = connection.getConfiguration() .getInt("hbase.region.replica.replication.cache.memstoreReplicationEnabled.expiryMs", 5000); this.memstoreReplicationEnabled = CacheBuilder.newBuilder() .expireAfterWrite(memstoreReplicationEnabledCacheExpiryMs, TimeUnit.MILLISECONDS) .initialCapacity(10) .maximumSize(1000) .build(); }
/** * If region open fails with IOException in openRegion() while doing tableDescriptors.get() * the region should not add into regionsInTransitionInRS map * @throws Exception */ @Test public void testRegionOpenFailsDueToIOException() throws Exception { HRegionInfo REGIONINFO = new HRegionInfo(TableName.valueOf("t"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW); HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(0); TableDescriptors htd = Mockito.mock(TableDescriptors.class); Object orizinalState = Whitebox.getInternalState(regionServer,"tableDescriptors"); Whitebox.setInternalState(regionServer, "tableDescriptors", htd); Mockito.doThrow(new IOException()).when(htd).get((TableName) Mockito.any()); try { ProtobufUtil.openRegion(null, regionServer.getRSRpcServices(), regionServer.getServerName(), REGIONINFO); fail("It should throw IOException "); } catch (IOException e) { } Whitebox.setInternalState(regionServer, "tableDescriptors", orizinalState); assertFalse("Region should not be in RIT", regionServer.getRegionsInTransitionInRS().containsKey(REGIONINFO.getEncodedNameAsBytes())); }
/** * If region open fails with IOException in openRegion() while doing tableDescriptors.get() * the region should not add into regionsInTransitionInRS map * @throws Exception */ @Test public void testRegionOpenFailsDueToIOException() throws Exception { HRegionInfo REGIONINFO = new HRegionInfo(TableName.valueOf("t"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW); HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(0); TableDescriptors htd = Mockito.mock(TableDescriptors.class); Object orizinalState = Whitebox.getInternalState(regionServer,"tableDescriptors"); Whitebox.setInternalState(regionServer, "tableDescriptors", htd); Mockito.doThrow(new IOException()).when(htd).get((TableName) Mockito.any()); try { ProtobufUtil.openRegion(regionServer.getRSRpcServices(), regionServer.getServerName(), REGIONINFO); fail("It should throw IOException "); } catch (IOException e) { } Whitebox.setInternalState(regionServer, "tableDescriptors", orizinalState); assertFalse("Region should not be in RIT", regionServer.getRegionsInTransitionInRS().containsKey(REGIONINFO.getEncodedNameAsBytes())); }
/** * If region open fails with IOException in openRegion() while doing tableDescriptors.get() * the region should not add into regionsInTransitionInRS map * @throws Exception */ @Test public void testRegionOpenFailsDueToIOException() throws Exception { HRegionInfo REGIONINFO = new HRegionInfo(TableName.valueOf("t"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW); HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(0); TableDescriptors htd = Mockito.mock(TableDescriptors.class); Object orizinalState = Whitebox.getInternalState(regionServer,"tableDescriptors"); Whitebox.setInternalState(regionServer, "tableDescriptors", htd); Mockito.doThrow(new IOException()).when(htd).get((TableName) Mockito.any()); try { ProtobufUtil.openRegion(regionServer, regionServer.getServerName(), REGIONINFO); fail("It should throw IOException "); } catch (IOException e) { } Whitebox.setInternalState(regionServer, "tableDescriptors", orizinalState); assertFalse("Region should not be in RIT", regionServer.getRegionsInTransitionInRS().containsKey(REGIONINFO.getEncodedNameAsBytes())); }
@InterfaceAudience.Private public Context( final Configuration localConf, final Configuration conf, final FileSystem fs, final String peerId, final UUID clusterId, final ReplicationPeer replicationPeer, final MetricsSource metrics, final TableDescriptors tableDescriptors, final Abortable abortable) { this.localConf = localConf; this.conf = conf; this.fs = fs; this.clusterId = clusterId; this.peerId = peerId; this.replicationPeer = replicationPeer; this.metrics = metrics; this.tableDescriptors = tableDescriptors; this.abortable = abortable; }
public static void fixTableStates(TableDescriptors tableDescriptors, Connection connection) throws IOException { final Map<String, TableDescriptor> allDescriptors = tableDescriptors.getAllDescriptors(); final Map<String, TableState> states = new HashMap<>(); MetaTableAccessor.fullScanTables(connection, new MetaTableAccessor.Visitor() { @Override public boolean visit(Result r) throws IOException { TableState state = MetaTableAccessor.getTableState(r); if (state != null) states.put(state.getTableName().getNameAsString(), state); return true; } }); for (Map.Entry<String, TableDescriptor> entry : allDescriptors.entrySet()) { String table = entry.getKey(); if (table.equals(TableName.META_TABLE_NAME.getNameAsString())) { continue; } if (!states.containsKey(table)) { LOG.warn(table + " has no state, assuming ENABLED"); MetaTableAccessor.updateTableState(connection, TableName.valueOf(table), TableState.State.ENABLED); } } }
/** * If region open fails with IOException in openRegion() while doing tableDescriptors.get() * the region should not add into regionsInTransitionInRS map * @throws Exception */ @Test public void testRegionOpenFailsDueToIOException() throws Exception { HRegionInfo REGIONINFO = new HRegionInfo(TableName.valueOf("t"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW); HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(0); TableDescriptors htd = Mockito.mock(TableDescriptors.class); Object orizinalState = Whitebox.getInternalState(regionServer,"tableDescriptors"); Whitebox.setInternalState(regionServer, "tableDescriptors", htd); Mockito.doThrow(new IOException()).when(htd).get((TableName) Mockito.any()); try { ProtobufUtil.openRegion(regionServer, REGIONINFO); fail("It should throw IOException "); } catch (IOException e) { } Whitebox.setInternalState(regionServer, "tableDescriptors", orizinalState); assertFalse("Region should not be in RIT", regionServer.getRegionsInTransitionInRS().containsKey(REGIONINFO.getEncodedNameAsBytes())); }
/** * If region open fails with IOException in openRegion() while doing tableDescriptors.get() * the region should not add into regionsInTransitionInRS map * @throws Exception */ @Test public void testRegionOpenFailsDueToIOException() throws Exception { HRegionInfo REGIONINFO = new HRegionInfo(Bytes.toBytes("t"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW); HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(0); TableDescriptors htd = Mockito.mock(TableDescriptors.class); Object orizinalState = Whitebox.getInternalState(regionServer,"tableDescriptors"); Whitebox.setInternalState(regionServer, "tableDescriptors", htd); Mockito.doThrow(new IOException()).when(htd).get((byte[]) Mockito.any()); try { ProtobufUtil.openRegion(regionServer, REGIONINFO); fail("It should throw IOException "); } catch (IOException e) { } Whitebox.setInternalState(regionServer, "tableDescriptors", orizinalState); assertFalse("Region should not be in RIT", regionServer.getRegionsInTransitionInRS().containsKey(REGIONINFO.getEncodedNameAsBytes())); }
@Test public void testRemoves() throws IOException { final String name = "testRemoves"; FileSystem fs = FileSystem.get(UTIL.getConfiguration()); // Cleanup old tests if any detrius laying around. Path rootdir = new Path(UTIL.getDataTestDir(), name); TableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name)); htds.add(htd); assertNotNull(htds.remove(htd.getTableName())); assertNull(htds.remove(htd.getTableName())); }
@Test public void testNoSuchTable() throws IOException { final String name = "testNoSuchTable"; FileSystem fs = FileSystem.get(UTIL.getConfiguration()); // Cleanup old tests if any detrius laying around. Path rootdir = new Path(UTIL.getDataTestDir(), name); TableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir); assertNull("There shouldn't be any HTD for this table", htds.get(TableName.valueOf("NoSuchTable"))); }
@Test public void testUpdates() throws IOException { final String name = "testUpdates"; FileSystem fs = FileSystem.get(UTIL.getConfiguration()); // Cleanup old tests if any detrius laying around. Path rootdir = new Path(UTIL.getDataTestDir(), name); TableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name)); htds.add(htd); htds.add(htd); htds.add(htd); }
@Test public void testRemoves() throws IOException { final String name = "testRemoves"; FileSystem fs = FileSystem.get(UTIL.getConfiguration()); // Cleanup old tests if any detrius laying around. Path rootdir = new Path(UTIL.getDataTestDir(), name); TableDescriptors htds = new FSTableDescriptors(fs, rootdir); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name)); htds.add(htd); assertNotNull(htds.remove(htd.getTableName())); assertNull(htds.remove(htd.getTableName())); }
@Test public void testNoSuchTable() throws IOException { final String name = "testNoSuchTable"; FileSystem fs = FileSystem.get(UTIL.getConfiguration()); // Cleanup old tests if any detrius laying around. Path rootdir = new Path(UTIL.getDataTestDir(), name); TableDescriptors htds = new FSTableDescriptors(fs, rootdir); assertNull("There shouldn't be any HTD for this table", htds.get(TableName.valueOf("NoSuchTable"))); }
@Test public void testUpdates() throws IOException { final String name = "testUpdates"; FileSystem fs = FileSystem.get(UTIL.getConfiguration()); // Cleanup old tests if any detrius laying around. Path rootdir = new Path(UTIL.getDataTestDir(), name); TableDescriptors htds = new FSTableDescriptors(fs, rootdir); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name)); htds.add(htd); htds.add(htd); htds.add(htd); }
private void initAndStartReplicationEndpoint(ReplicationEndpoint replicationEndpoint) throws IOException, TimeoutException { TableDescriptors tableDescriptors = null; if (server instanceof HRegionServer) { tableDescriptors = ((HRegionServer) server).getTableDescriptors(); } replicationEndpoint .init(new ReplicationEndpoint.Context(conf, replicationPeer.getConfiguration(), fs, peerId, clusterId, replicationPeer, metrics, tableDescriptors, server)); replicationEndpoint.start(); replicationEndpoint.awaitRunning(waitOnEndpointSeconds, TimeUnit.SECONDS); }
@Override @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="REC_CATCH_EXCEPTION", justification="Intentional") protected void chore() { try { TableDescriptors htds = master.getTableDescriptors(); Map<String, TableDescriptor> map = htds.getAll(); for (TableDescriptor htd : map.values()) { for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) { if (hcd.isMobEnabled() && hcd.getMinVersions() == 0) { // clean only for mob-enabled column. // obtain a read table lock before cleaning, synchronize with MobFileCompactionChore. final LockManager.MasterLock lock = master.getLockManager().createMasterLock( MobUtils.getTableLockName(htd.getTableName()), LockType.SHARED, this.getClass().getSimpleName() + ": Cleaning expired mob files"); try { lock.acquire(); cleaner.cleanExpiredMobFiles(htd.getTableName().getNameAsString(), hcd); } finally { lock.release(); } } } } } catch (Exception e) { LOG.error("Fail to clean the expired mob files", e); } }
@Override protected void chore() { try { TableDescriptors htds = master.getTableDescriptors(); Map<String, TableDescriptor> map = htds.getAll(); for (TableDescriptor htd : map.values()) { if (!master.getTableStateManager().isTableState(htd.getTableName(), TableState.State.ENABLED)) { continue; } boolean reported = false; try { final LockManager.MasterLock lock = master.getLockManager().createMasterLock( MobUtils.getTableLockName(htd.getTableName()), LockType.EXCLUSIVE, this.getClass().getName() + ": mob compaction"); for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) { if (!hcd.isMobEnabled()) { continue; } if (!reported) { master.reportMobCompactionStart(htd.getTableName()); reported = true; } MobUtils.doMobCompaction(master.getConfiguration(), master.getFileSystem(), htd.getTableName(), hcd, pool, false, lock); } } finally { if (reported) { master.reportMobCompactionEnd(htd.getTableName()); } } } } catch (Exception e) { LOG.error("Failed to compact mob files", e); } }
@Test public void testRemoves() throws IOException { final String name = this.name.getMethodName(); FileSystem fs = FileSystem.get(UTIL.getConfiguration()); // Cleanup old tests if any detrius laying around. Path rootdir = new Path(UTIL.getDataTestDir(), name); TableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir); TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build(); htds.add(htd); assertNotNull(htds.remove(htd.getTableName())); assertNull(htds.remove(htd.getTableName())); }
@Test public void testUpdates() throws IOException { final String name = "testUpdates"; FileSystem fs = FileSystem.get(UTIL.getConfiguration()); // Cleanup old tests if any detrius laying around. Path rootdir = new Path(UTIL.getDataTestDir(), name); TableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir); TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build(); htds.add(htd); htds.add(htd); htds.add(htd); }
private static MasterServices getMockedMaster() throws IOException { TableDescriptors tds = Mockito.mock(TableDescriptors.class); Mockito.when(tds.get(tables[0])).thenReturn(tableDescs.get(0)); Mockito.when(tds.get(tables[1])).thenReturn(tableDescs.get(1)); Mockito.when(tds.get(tables[2])).thenReturn(tableDescs.get(2)); Mockito.when(tds.get(tables[3])).thenReturn(tableDescs.get(3)); MasterServices services = Mockito.mock(HMaster.class); Mockito.when(services.getTableDescriptors()).thenReturn(tds); AssignmentManager am = Mockito.mock(AssignmentManager.class); Mockito.when(services.getAssignmentManager()).thenReturn(am); return services; }
public TableDescriptors getTableDescriptors() { return tableDescriptors; }
@Override public TableDescriptors getTableDescriptors() { return this.tableDescriptors; }
/** * @return Return table descriptors implementation. */ public TableDescriptors getTableDescriptors() { return this.tableDescriptors; }
@Override public void postModifyTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName, HTableDescriptor htd) throws IOException { String table = tableName.getNameAsString(); MasterServices master = ctx.getEnvironment().getMasterServices(); List<Pair<HRegionInfo, ServerName>> tableRegionsAndLocations = null; LOG.info("Entering postModifyTable for the table " + table); byte[] indexBytes = htd.getValue(Constants.INDEX_SPEC_KEY); if (indexBytes != null) { TableDescriptors tableDescriptors = master.getTableDescriptors(); Map<String, HTableDescriptor> allTableDesc = tableDescriptors.getAll(); String indexTableName = IndexUtils.getIndexTableName(table); if (allTableDesc.containsKey(indexTableName)) { // Do table modification TableIndices tableIndices = new TableIndices(); tableIndices.readFields(indexBytes); List<IndexSpecification> indices = tableIndices.getIndices(); if (indices.isEmpty()) { LOG.error("Empty indices are passed to modify the table " + table); return; } IndexManager idxManager = IndexManager.getInstance(); idxManager.removeIndices(table); idxManager.addIndexForTable(table, indices); LOG.info("Successfully updated the indexes for the table " + table + " to " + indices); } else { try { tableRegionsAndLocations = MetaReader.getTableRegionsAndLocations(master.getCatalogTracker(), tableName, true); } catch (InterruptedException e) { LOG.error("Exception while trying to create index table for the existing table " + table); return; } if (tableRegionsAndLocations != null) { HRegionInfo[] regionInfo = new HRegionInfo[tableRegionsAndLocations.size()]; for (int i = 0; i < tableRegionsAndLocations.size(); i++) { regionInfo[i] = tableRegionsAndLocations.get(i).getFirst(); } byte[][] splitKeys = getSplitKeys(regionInfo); createSecondaryIndexTable(htd, splitKeys, master, true); } } } LOG.info("Exiting postModifyTable for the table " + table); }