/** * Returns the {@link ServerName} from catalog table {@link Result} * where the region is transitioning. It should be the same as * {@link HRegionInfo#getServerName(Result)} if the server is at OPEN state. * @param r Result to pull the transitioning server name from * @return A ServerName instance or {@link HRegionInfo#getServerName(Result)} * if necessary fields not found or empty. */ static ServerName getRegionServer(final Result r, int replicaId) { Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, getServerNameColumn(replicaId)); if (cell == null || cell.getValueLength() == 0) { RegionLocations locations = MetaTableAccessor.getRegionLocations(r); if (locations != null) { HRegionLocation location = locations.getRegionLocation(replicaId); if (location != null) { return location.getServerName(); } } return null; } return ServerName.parseServerName(Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength())); }
private void replicateUsingCallable(ClusterConnection connection, Queue<Entry> entries) throws IOException, RuntimeException { Entry entry; while ((entry = entries.poll()) != null) { byte[] row = entry.getEdit().getCells().get(0).getRow(); RegionLocations locations = connection.locateRegion(tableName, row, true, true); RegionReplicaReplayCallable callable = new RegionReplicaReplayCallable(connection, RpcControllerFactory.instantiate(connection.getConfiguration()), table.getName(), locations.getRegionLocation(1), locations.getRegionLocation(1).getRegionInfo(), row, Lists.newArrayList(entry), new AtomicLong()); RpcRetryingCallerFactory factory = RpcRetryingCallerFactory.instantiate( connection.getConfiguration()); factory.<ReplicateWALEntryResponse> newCaller().callWithRetries(callable, 10000); } }
@Test public void testHBaseFsckWithFewerMetaReplicas() throws Exception { ClusterConnection c = (ClusterConnection)ConnectionFactory.createConnection( TEST_UTIL.getConfiguration()); RegionLocations rl = c.locateRegion(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW, false, false); HBaseFsckRepair.closeRegionSilentlyAndWait(c, rl.getRegionLocation(1).getServerName(), rl.getRegionLocation(1).getRegionInfo()); // check that problem exists HBaseFsck hbck = doFsck(TEST_UTIL.getConfiguration(), false); assertErrors(hbck, new ERROR_CODE[]{ERROR_CODE.UNKNOWN,ERROR_CODE.NO_META_REGION}); // fix the problem hbck = doFsck(TEST_UTIL.getConfiguration(), true); // run hbck again to make sure we don't see any errors hbck = doFsck(TEST_UTIL.getConfiguration(), false); assertErrors(hbck, new ERROR_CODE[]{}); }
@Test public void testHBaseFsckWithFewerMetaReplicaZnodes() throws Exception { ClusterConnection c = (ClusterConnection)ConnectionFactory.createConnection( TEST_UTIL.getConfiguration()); RegionLocations rl = c.locateRegion(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW, false, false); HBaseFsckRepair.closeRegionSilentlyAndWait(c, rl.getRegionLocation(2).getServerName(), rl.getRegionLocation(2).getRegionInfo()); ZooKeeperWatcher zkw = TEST_UTIL.getZooKeeperWatcher(); ZKUtil.deleteNode(zkw, zkw.getZNodeForReplica(2)); // check that problem exists HBaseFsck hbck = doFsck(TEST_UTIL.getConfiguration(), false); assertErrors(hbck, new ERROR_CODE[]{ERROR_CODE.UNKNOWN,ERROR_CODE.NO_META_REGION}); // fix the problem hbck = doFsck(TEST_UTIL.getConfiguration(), true); // run hbck again to make sure we don't see any errors hbck = doFsck(TEST_UTIL.getConfiguration(), false); assertErrors(hbck, new ERROR_CODE[]{}); }
@Test public void testShutdownOfReplicaHolder() throws Exception { // checks that the when the server holding meta replica is shut down, the meta replica // can be recovered RegionLocations rl = ConnectionManager.getConnectionInternal(TEST_UTIL.getConfiguration()). locateRegion(TableName.META_TABLE_NAME, Bytes.toBytes(""), false, true); HRegionLocation hrl = rl.getRegionLocation(1); ServerName oldServer = hrl.getServerName(); TEST_UTIL.getHBaseClusterInterface().killRegionServer(oldServer); int i = 0; do { LOG.debug("Waiting for the replica " + hrl.getRegionInfo() + " to come up"); Thread.sleep(30000); //wait for the detection/recovery rl = ConnectionManager.getConnectionInternal(TEST_UTIL.getConfiguration()). locateRegion(TableName.META_TABLE_NAME, Bytes.toBytes(""), false, true); hrl = rl.getRegionLocation(1); i++; } while ((hrl == null || hrl.getServerName().equals(oldServer)) && i < 3); assertTrue(i != 3); }
@Test public void testLocations() throws Exception { byte[] b1 = "testLocations".getBytes(); openRegion(hriSecondary); ClusterConnection hc = (ClusterConnection) HTU.getHBaseAdmin().getConnection(); try { hc.clearRegionCache(); RegionLocations rl = hc.locateRegion(table.getName(), b1, false, false); Assert.assertEquals(2, rl.size()); rl = hc.locateRegion(table.getName(), b1, true, false); Assert.assertEquals(2, rl.size()); hc.clearRegionCache(); rl = hc.locateRegion(table.getName(), b1, true, false); Assert.assertEquals(2, rl.size()); rl = hc.locateRegion(table.getName(), b1, false, false); Assert.assertEquals(2, rl.size()); } finally { closeRegion(hriSecondary); } }
private void printLocations(Result r) { RegionLocations rl = null; if (r == null) { LOG.info("FAILED FOR null Result"); return; } LOG.info("FAILED FOR " + resultToString(r) + " Stale " + r.isStale()); if (r.getRow() == null) { return; } try { rl = ((ClusterConnection)connection).locateRegion(tableName, r.getRow(), true, true); } catch (IOException e) { LOG.warn("Couldn't get locations for row " + Bytes.toString(r.getRow())); } HRegionLocation locations[] = rl.getRegionLocations(); for (HRegionLocation h : locations) { LOG.info("LOCATION " + h); } }
/** * Lists all of the table regions currently in META. * @param connection * @param tableName * @return Map of all user-space regions to servers * @throws IOException */ public static NavigableMap<HRegionInfo, ServerName> allTableRegions( Connection connection, final TableName tableName) throws IOException { final NavigableMap<HRegionInfo, ServerName> regions = new TreeMap<HRegionInfo, ServerName>(); MetaScannerVisitor visitor = new TableMetaScannerVisitor(tableName) { @Override public boolean processRowInternal(Result result) throws IOException { RegionLocations locations = MetaTableAccessor.getRegionLocations(result); if (locations == null) return true; for (HRegionLocation loc : locations.getRegionLocations()) { if (loc != null) { HRegionInfo regionInfo = loc.getRegionInfo(); regions.put(new UnmodifyableHRegionInfo(regionInfo), loc.getServerName()); } } return true; } }; metaScan(connection, visitor, tableName); return regions; }
/** * Lists table regions and locations grouped by region range from META. */ public static List<RegionLocations> listTableRegionLocations(Configuration conf, Connection connection, final TableName tableName) throws IOException { final List<RegionLocations> regions = new ArrayList<RegionLocations>(); MetaScannerVisitor visitor = new TableMetaScannerVisitor(tableName) { @Override public boolean processRowInternal(Result result) throws IOException { RegionLocations locations = MetaTableAccessor.getRegionLocations(result); if (locations == null) return true; regions.add(locations); return true; } }; metaScan(connection, visitor, tableName); return regions; }
/** * Two responsibilities * - if the call is already completed (by another replica) stops the retries. * - set the location to the right region, depending on the replica. */ @Override public void prepare(final boolean reload) throws IOException { if (controller.isCanceled()) return; if (Thread.interrupted()) { throw new InterruptedIOException(); } if (reload || location == null) { RegionLocations rl = getRegionLocations(false, id, cConnection, tableName, get.getRow()); location = id < rl.size() ? rl.getRegionLocation(id) : null; } if (location == null || location.getServerName() == null) { // With this exception, there will be a retry. The location can be null for a replica // when the table is created or after a split. throw new HBaseIOException("There is no location for replica id #" + id); } ServerName dest = location.getServerName(); setStub(cConnection.getClient(dest)); }
/** * Put a newly discovered HRegionLocation into the cache. * @param tableName The table name. * @param locations the new locations */ public void cacheLocation(final TableName tableName, final RegionLocations locations) { byte [] startKey = locations.getRegionLocation().getRegionInfo().getStartKey(); ConcurrentMap<byte[], RegionLocations> tableLocations = getTableLocations(tableName); RegionLocations oldLocation = tableLocations.putIfAbsent(startKey, locations); boolean isNewCacheEntry = (oldLocation == null); if (isNewCacheEntry) { if (LOG.isTraceEnabled()) { LOG.trace("Cached location: " + locations); } addToCachedServers(locations); return; } // merge old and new locations and add it to the cache // Meta record might be stale - some (probably the same) server has closed the region // with later seqNum and told us about the new location. RegionLocations mergedLocation = oldLocation.mergeLocations(locations); boolean replaced = tableLocations.replace(startKey, oldLocation, mergedLocation); if (replaced && LOG.isTraceEnabled()) { LOG.trace("Merged cached locations: " + mergedLocation); } addToCachedServers(locations); }
/** * @param tableName * @return Map of cached locations for passed <code>tableName</code> */ private ConcurrentNavigableMap<byte[], RegionLocations> getTableLocations(final TableName tableName) { // find the map of cached locations for this table ConcurrentNavigableMap<byte[], RegionLocations> result; result = this.cachedRegionLocations.get(tableName); // if tableLocations for this table isn't built yet, make one if (result == null) { result = new CopyOnWriteArrayMap<>(Bytes.BYTES_COMPARATOR); ConcurrentNavigableMap<byte[], RegionLocations> old = this.cachedRegionLocations.putIfAbsent(tableName, result); if (old != null) { return old; } } return result; }
/** * Delete a cached location, no matter what it is. Called when we were told to not use cache. * @param tableName tableName * @param row */ public void clearCache(final TableName tableName, final byte [] row, int replicaId) { ConcurrentMap<byte[], RegionLocations> tableLocations = getTableLocations(tableName); boolean removed = false; RegionLocations regionLocations = getCachedLocation(tableName, row); if (regionLocations != null) { HRegionLocation toBeRemoved = regionLocations.getRegionLocation(replicaId); RegionLocations updatedLocations = regionLocations.remove(replicaId); if (updatedLocations != regionLocations) { byte[] startKey = regionLocations.getRegionLocation().getRegionInfo().getStartKey(); if (updatedLocations.isEmpty()) { removed = tableLocations.remove(startKey, regionLocations); } else { removed = tableLocations.replace(startKey, regionLocations, updatedLocations); } } if (removed && LOG.isTraceEnabled() && toBeRemoved != null) { LOG.trace("Removed " + toBeRemoved + " from cache"); } } }
/** * Delete a cached location for a table, row and server */ public void clearCache(final TableName tableName, final byte [] row, ServerName serverName) { ConcurrentMap<byte[], RegionLocations> tableLocations = getTableLocations(tableName); RegionLocations regionLocations = getCachedLocation(tableName, row); if (regionLocations != null) { RegionLocations updatedLocations = regionLocations.removeByServer(serverName); if (updatedLocations != regionLocations) { byte[] startKey = regionLocations.getRegionLocation().getRegionInfo().getStartKey(); boolean removed = false; if (updatedLocations.isEmpty()) { removed = tableLocations.remove(startKey, regionLocations); } else { removed = tableLocations.replace(startKey, regionLocations, updatedLocations); } if (removed && LOG.isTraceEnabled()) { LOG.trace("Removed locations of table: " + tableName + " ,row: " + Bytes.toString(row) + " mapping to server: " + serverName + " from cache"); } } } }
/** * Deletes the cached location of the region if necessary, based on some error from source. * @param hri The region in question. */ public void clearCache(HRegionInfo hri) { ConcurrentMap<byte[], RegionLocations> tableLocations = getTableLocations(hri.getTable()); RegionLocations regionLocations = tableLocations.get(hri.getStartKey()); if (regionLocations != null) { HRegionLocation oldLocation = regionLocations.getRegionLocation(hri.getReplicaId()); if (oldLocation == null) return; RegionLocations updatedLocations = regionLocations.remove(oldLocation); boolean removed = false; if (updatedLocations != regionLocations) { if (updatedLocations.isEmpty()) { removed = tableLocations.remove(hri.getStartKey(), regionLocations); } else { removed = tableLocations.replace(hri.getStartKey(), regionLocations, updatedLocations); } if (removed && LOG.isTraceEnabled()) { LOG.trace("Removed " + oldLocation + " from cache"); } } } }
public void clearCache(final HRegionLocation location) { if (location == null) { return; } TableName tableName = location.getRegionInfo().getTable(); ConcurrentMap<byte[], RegionLocations> tableLocations = getTableLocations(tableName); RegionLocations regionLocations = tableLocations.get(location.getRegionInfo().getStartKey()); if (regionLocations != null) { RegionLocations updatedLocations = regionLocations.remove(location); boolean removed = false; if (updatedLocations != regionLocations) { if (updatedLocations.isEmpty()) { removed = tableLocations.remove(location.getRegionInfo().getStartKey(), regionLocations); } else { removed = tableLocations.replace(location.getRegionInfo().getStartKey(), regionLocations, updatedLocations); } if (removed && LOG.isTraceEnabled()) { LOG.trace("Removed " + location + " from cache"); } } } }
private void addCallsForOtherReplicas( ResultBoundedCompletionService<Pair<Result[], ScannerCallable>> cs, RegionLocations rl, int min, int max) { if (scan.getConsistency() == Consistency.STRONG) { return; // not scheduling on other replicas for strong consistency } for (int id = min; id <= max; id++) { if (currentScannerCallable.id == id) { continue; //this was already scheduled earlier } ScannerCallable s = currentScannerCallable.getScannerCallableForReplica(id); setStartRowForReplicaCallable(s); outstandingCallables.add(s); RetryingRPC retryingOnReplica = new RetryingRPC(s); cs.submit(retryingOnReplica, scannerTimeout, id); } }
@Override public List<HRegionLocation> locateRegions(final TableName tableName, final boolean useCache, final boolean offlined) throws IOException { NavigableMap<HRegionInfo, ServerName> regions = MetaScanner.allTableRegions(this, tableName); final List<HRegionLocation> locations = new ArrayList<HRegionLocation>(); for (HRegionInfo regionInfo : regions.keySet()) { RegionLocations list = locateRegion(tableName, regionInfo.getStartKey(), useCache, true); if (list != null) { for (HRegionLocation loc : list.getRegionLocations()) { if (loc != null) { locations.add(loc); } } } } return locations; }
@Override public RegionLocations locateRegion(final TableName tableName, final byte [] row, boolean useCache, boolean retry, int replicaId) throws IOException { if (this.closed) throw new IOException(toString() + " closed"); if (tableName== null || tableName.getName().length == 0) { throw new IllegalArgumentException( "table name cannot be null or zero length"); } if (tableName.equals(TableName.META_TABLE_NAME)) { return locateMeta(tableName, useCache, replicaId); } else { // Region not in the cache - have to go to the meta RS return locateRegionInMeta(tableName, row, useCache, retry, replicaId); } }
private HRegionLocation getReplicaLocationOrFail(Action<Row> action) { // We are going to try get location once again. For each action, we'll do it once // from cache, because the previous calls in the loop might populate it. int replicaId = action.getReplicaId(); RegionLocations locs = findAllLocationsOrFail(action, true); if (locs == null) return null; // manageError already called HRegionLocation loc = locs.getRegionLocation(replicaId); if (loc == null || loc.getServerName() == null) { locs = findAllLocationsOrFail(action, false); if (locs == null) return null; // manageError already called loc = locs.getRegionLocation(replicaId); } if (loc == null || loc.getServerName() == null) { manageLocationError(action, null); return null; } return loc; }
/** * Tests the case where all replicas of a region throw an exception. It should not cause a hang * but the exception should propagate to the client */ @Test (timeout = 30000) public void testExceptionsFromReplicasArePropagated() throws IOException { scan.setConsistency(Consistency.TIMELINE); // Mock a caller which calls the callable for ScannerCallableWithReplicas, // but throws an exception for the actual scanner calls via callWithRetries. rpcFactory = new MockRpcRetryingCallerFactory(conf); conf.set(RpcRetryingCallerFactory.CUSTOM_CALLER_CONF_KEY, MockRpcRetryingCallerFactory.class.getName()); // mock 3 replica locations when(clusterConn.locateRegion((TableName)any(), (byte[])any(), anyBoolean(), anyBoolean(), anyInt())).thenReturn(new RegionLocations(null, null, null)); try (MockClientScanner scanner = new MockClientScanner(conf, scan, TableName.valueOf("table"), clusterConn, rpcFactory, controllerFactory, pool, Integer.MAX_VALUE)) { Iterator<Result> iter = scanner.iterator(); while (iter.hasNext()) { iter.next(); } fail("Should have failed with RetriesExhaustedException"); } catch (RetriesExhaustedException expected) { } }
/** * Search the cache for a location that fits our table and row key. * Return null if no suitable region is located. * * * @param tableName * @param row * @return Null or region location found in cache. */ public RegionLocations getCachedLocation(final TableName tableName, final byte [] row) { ConcurrentSkipListMap<byte[], RegionLocations> tableLocations = getTableLocations(tableName); Entry<byte[], RegionLocations> e = tableLocations.floorEntry(row); if (e == null) { return null; } RegionLocations possibleRegion = e.getValue(); // make sure that the end key is greater than the row we're looking // for, otherwise the row actually belongs in the next region, not // this one. the exception case is when the endkey is // HConstants.EMPTY_END_ROW, signifying that the region we're // checking is actually the last region in the table. byte[] endKey = possibleRegion.getRegionLocation().getRegionInfo().getEndKey(); if (Bytes.equals(endKey, HConstants.EMPTY_END_ROW) || tableName.getRowComparator().compareRows( endKey, 0, endKey.length, row, 0, row.length) > 0) { return possibleRegion; } // Passed all the way through, so we got nothing - complete cache miss return null; }
/** * @param tableName * @return Map of cached locations for passed <code>tableName</code> */ private ConcurrentSkipListMap<byte[], RegionLocations> getTableLocations(final TableName tableName) { // find the map of cached locations for this table ConcurrentSkipListMap<byte[], RegionLocations> result; result = this.cachedRegionLocations.get(tableName); // if tableLocations for this table isn't built yet, make one if (result == null) { result = new ConcurrentSkipListMap<byte[], RegionLocations>(Bytes.BYTES_COMPARATOR); ConcurrentSkipListMap<byte[], RegionLocations> old = this.cachedRegionLocations.putIfAbsent(tableName, result); if (old != null) { return old; } } return result; }
private int addCallsForOtherReplicas( BoundedCompletionService<Pair<Result[], ScannerCallable>> cs, RegionLocations rl, int min, int max) { if (scan.getConsistency() == Consistency.STRONG) { return 0; // not scheduling on other replicas for strong consistency } for (int id = min; id <= max; id++) { if (currentScannerCallable.getHRegionInfo().getReplicaId() == id) { continue; //this was already scheduled earlier } ScannerCallable s = currentScannerCallable.getScannerCallableForReplica(id); if (this.lastResult != null) { s.getScan().setStartRow(this.lastResult.getRow()); } outstandingCallables.add(s); RetryingRPC retryingOnReplica = new RetryingRPC(s); cs.submit(retryingOnReplica); } return max - min + 1; }
@Override public RegionLocations locateRegion(final TableName tableName, final byte[] row, boolean useCache, boolean retry, int replicaId) throws IOException { if (this.closed) throw new IOException(toString() + " closed"); if (tableName == null || tableName.getName().length == 0) { throw new IllegalArgumentException( "table name cannot be null or zero length"); } if (tableName.equals(TableName.META_TABLE_NAME)) { return locateMeta(tableName, useCache, replicaId); } else { // Region not in the cache - have to go to the meta RS return locateRegionInMeta(tableName, row, useCache, retry, replicaId); } }
@Override public RegionLocations getMetaRegionLocation() throws IOException { ZooKeeperKeepAliveConnection zkw = hci.getKeepAliveZooKeeperWatcher(); try { if (LOG.isTraceEnabled()) { LOG.trace("Looking up meta region location in ZK," + " connection=" + this); } ServerName servername = new MetaTableLocator().blockUntilAvailable(zkw, hci.rpcTimeout); if (LOG.isTraceEnabled()) { LOG.trace("Looked up meta region location, connection=" + this + "; serverName=" + ((servername == null) ? "null" : servername)); } if (servername == null) return null; HRegionLocation loc = new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, servername, 0); return new RegionLocations(new HRegionLocation[] {loc}); } catch (InterruptedException e) { Thread.currentThread().interrupt(); return null; } finally { zkw.close(); } }
/** * {@inheritDoc} */ @Override public Pair<byte[][],byte[][]> getStartEndKeys() throws IOException { List<RegionLocations> regions = listRegionLocations(); final List<byte[]> startKeyList = new ArrayList<byte[]>(regions.size()); final List<byte[]> endKeyList = new ArrayList<byte[]>(regions.size()); for (RegionLocations locations : regions) { HRegionInfo region = locations.getRegionLocation().getRegionInfo(); startKeyList.add(region.getStartKey()); endKeyList.add(region.getEndKey()); } return new Pair<byte [][], byte [][]>( startKeyList.toArray(new byte[startKeyList.size()][]), endKeyList.toArray(new byte[endKeyList.size()][])); }
/** * Returns the {@link ServerName} from catalog table {@link Result} * where the region is transitioning. It should be the same as * {@link MetaTableAccessor#getServerName(Result,int)} if the server is at OPEN state. * @param r Result to pull the transitioning server name from * @return A ServerName instance or {@link MetaTableAccessor#getServerName(Result,int)} * if necessary fields not found or empty. */ static ServerName getRegionServer(final Result r, int replicaId) { final Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, getServerNameColumn(replicaId)); if (cell == null || cell.getValueLength() == 0) { RegionLocations locations = MetaTableAccessor.getRegionLocations(r); if (locations != null) { HRegionLocation location = locations.getRegionLocation(replicaId); if (location != null) { return location.getServerName(); } } return null; } return ServerName.parseServerName(Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength())); }
