/** * Test that closing a scanner while a client is using it doesn't throw * NPEs but instead a UnknownScannerException. HBASE-2503 * @throws Exception */ @Test public void testRaceBetweenClientAndTimeout() throws Exception { try { this.r = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null); HBaseTestCase.addContent(this.r, HConstants.CATALOG_FAMILY); Scan scan = new Scan(); InternalScanner s = r.getScanner(scan); List<Cell> results = new ArrayList<Cell>(); try { s.next(results); s.close(); s.next(results); fail("We don't want anything more, we should be failing"); } catch (UnknownScannerException ex) { // ok! return; } } finally { HRegion.closeHRegion(this.r); } }
@Override public synchronized boolean next(List<KeyValue> outResults, int limit, String metric) throws IOException { if (this.filterClosed) { throw new UnknownScannerException("Scanner was closed (timed out?) " + "after we renewed it. Could be caused by a very slow scanner " + "or a lengthy garbage collection"); } startRegionOperation(); readRequestsCount.increment(); opMetrics.setReadRequestCountMetrics(readRequestsCount.get()); try { // This could be a new thread from the last time we called next(). MultiVersionConsistencyControl.setThreadReadPoint(this.readPt); return nextRaw(outResults, limit, metric); } finally { closeRegionOperation(); } }
/** * Test that closing a scanner while a client is using it doesn't throw * NPEs but instead a UnknownScannerException. HBASE-2503 * @throws Exception */ public void testRaceBetweenClientAndTimeout() throws Exception { try { this.r = createNewHRegion(TESTTABLEDESC, null, null); addContent(this.r, HConstants.CATALOG_FAMILY); Scan scan = new Scan(); InternalScanner s = r.getScanner(scan); List<KeyValue> results = new ArrayList<KeyValue>(); try { s.next(results); s.close(); s.next(results); fail("We don't want anything more, we should be failing"); } catch (UnknownScannerException ex) { // ok! return; } } finally { this.r.close(); this.r.getLog().closeAndDelete(); } }
@Override public boolean nextRaw(List<Cell> outResults, int limit) throws IOException { if (storeHeap == null) { // scanner is closed throw new UnknownScannerException("Scanner was closed"); } boolean returnResult; if (outResults.isEmpty()) { // Usually outResults is empty. This is true when next is called // to handle scan or get operation. returnResult = nextInternal(outResults, limit); } else { List<Cell> tmpList = new ArrayList<Cell>(); returnResult = nextInternal(tmpList, limit); outResults.addAll(tmpList); } resetFilters(); if (isFilterDoneInternal()) { returnResult = false; } return returnResult; }
/** * Test that closing a scanner while a client is using it doesn't throw * NPEs but instead a UnknownScannerException. HBASE-2503 * @throws Exception */ @Test public void testRaceBetweenClientAndTimeout() throws Exception { try { this.region = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null); HBaseTestCase.addContent(this.region, HConstants.CATALOG_FAMILY); Scan scan = new Scan(); InternalScanner s = region.getScanner(scan); List<Cell> results = new ArrayList<>(); try { s.next(results); s.close(); s.next(results); fail("We don't want anything more, we should be failing"); } catch (UnknownScannerException ex) { // ok! return; } } finally { HBaseTestingUtility.closeRegionAndWAL(this.region); } }
@Override public synchronized boolean next(List<Cell> outResults, int limit) throws IOException { if (this.filterClosed) { throw new UnknownScannerException("Scanner was closed (timed out?) " + "after we renewed it. Could be caused by a very slow scanner " + "or a lengthy garbage collection"); } startRegionOperation(Operation.SCAN); readRequestsCount.increment(); try { // This could be a new thread from the last time we called next(). MultiVersionConsistencyControl.setThreadReadPoint(this.readPt); return nextRaw(outResults, limit); } finally { closeRegionOperation(); } }
@Override public synchronized boolean next(List<KeyValue> outResults, int limit, String metric) throws IOException { if (this.filterClosed) { throw new UnknownScannerException("Scanner was closed (timed out?) " + "after we renewed it. Could be caused by a very slow scanner " + "or a lengthy garbage collection"); } startRegionOperation(); readRequestsCount.increment(); try { // This could be a new thread from the last time we called next(). MultiVersionConsistencyControl.setThreadReadPoint(this.readPt); return nextRaw(outResults, limit, metric); } finally { closeRegionOperation(); } }
/** * Test that closing a scanner while a client is using it doesn't throw * NPEs but instead a UnknownScannerException. HBASE-2503 * @throws Exception */ public void testRaceBetweenClientAndTimeout() throws Exception { try { this.r = createNewHRegion(TESTTABLEDESC, null, null); addContent(this.r, HConstants.CATALOG_FAMILY); Scan scan = new Scan(); InternalScanner s = r.getScanner(scan); List<KeyValue> results = new ArrayList<KeyValue>(); try { s.next(results); s.close(); s.next(results); fail("We don't want anything more, we should be failing"); } catch (UnknownScannerException ex) { // ok! return; } } finally { HRegion.closeHRegion(this.r); } }
@Override public synchronized boolean next(List<Cell> outResults, ScannerContext scannerContext) throws IOException { if (this.filterClosed) { throw new UnknownScannerException("Scanner was closed (timed out?) " + "after we renewed it. Could be caused by a very slow scanner " + "or a lengthy garbage collection"); } startRegionOperation(Operation.SCAN); readRequestsCount.increment(); try { return nextRaw(outResults, scannerContext); } finally { closeRegionOperation(Operation.SCAN); } }
@Override public boolean nextRaw(List<Cell> outResults, ScannerContext scannerContext) throws IOException { if (!scanWithIndex && storeHeap == null) { // scanner is closed throw new UnknownScannerException("Scanner was closed"); } boolean moreValues; if (outResults.isEmpty()) { // Usually outResults is empty. This is true when next is called // to handle scan or get operation. moreValues = nextInternal(outResults, scannerContext); } else { List<Cell> tmpList = new ArrayList<Cell>(); moreValues = nextInternal(tmpList, scannerContext); outResults.addAll(tmpList); } // If the size limit was reached it means a partial Result is being // returned. Returning a // partial Result means that we should not reset the filters; filters // should only be reset in // between rows if (!scannerContext.partialResultFormed()) resetFilters(); if (isFilterDoneInternal()) { moreValues = false; } return moreValues; }
public void exception(Throwable throwable) { source.exception(); /** * Keep some metrics for commonly seen exceptions * * Try and put the most common types first. * Place child types before the parent type that they extend. * * If this gets much larger we might have to go to a hashmap */ if (throwable != null) { if (throwable instanceof OutOfOrderScannerNextException) { source.outOfOrderException(); } else if (throwable instanceof RegionTooBusyException) { source.tooBusyException(); } else if (throwable instanceof UnknownScannerException) { source.unknownScannerException(); } else if (throwable instanceof RegionMovedException) { source.movedRegionException(); } else if (throwable instanceof NotServingRegionException) { source.notServingRegionException(); } else if (throwable instanceof FailedSanityCheckException) { source.failedSanityException(); } else if (throwable instanceof MultiActionResultTooLarge) { source.multiActionTooLargeException(); } } }
public Result[] next(final long scannerId, int nbRows) throws IOException { // winter this is called from {@link ScannerCallable} // System.out.println("winter this is the first call from remote"); String scannerName = String.valueOf(scannerId); RegionScanner s = this.scanners.get(scannerName); if (s == null) { LOG.info("Client tried to access missing scanner " + scannerName); throw new UnknownScannerException("Name: " + scannerName); } return internalNext(s, nbRows, scannerName); }
@Override public synchronized boolean next(List<Cell> outResults, int limit) throws IOException { if (this.filterClosed) { throw new UnknownScannerException("Scanner was closed (timed out?) " + "after we renewed it. Could be caused by a very slow scanner " + "or a lengthy garbage collection"); } startRegionOperation(Operation.SCAN); readRequestsCount.increment(); try { return nextRaw(outResults, limit); } finally { closeRegionOperation(Operation.SCAN); } }
public Result[] next(final long scannerId, int nbRows) throws IOException { String scannerName = String.valueOf(scannerId); RegionScanner s = this.scanners.get(scannerName); if (s == null) { LOG.info("Client tried to access missing scanner " + scannerName); throw new UnknownScannerException("Name: " + scannerName); } return internalNext(s, nbRows, scannerName); }
/** * Increment the count for a specific exception type. This is called for each exception type * that is returned to the thrift handler. * @param rawThrowable type of exception */ public void exception(Throwable rawThrowable) { source.exception(); Throwable throwable = unwrap(rawThrowable); /** * Keep some metrics for commonly seen exceptions * * Try and put the most common types first. * Place child types before the parent type that they extend. * * If this gets much larger we might have to go to a hashmap */ if (throwable != null) { if (throwable instanceof OutOfOrderScannerNextException) { source.outOfOrderException(); } else if (throwable instanceof RegionTooBusyException) { source.tooBusyException(); } else if (throwable instanceof UnknownScannerException) { source.unknownScannerException(); } else if (throwable instanceof ScannerResetException) { source.scannerResetException(); } else if (throwable instanceof RegionMovedException) { source.movedRegionException(); } else if (throwable instanceof NotServingRegionException) { source.notServingRegionException(); } else if (throwable instanceof FailedSanityCheckException) { source.failedSanityException(); } else if (throwable instanceof MultiActionResultTooLarge) { source.multiActionTooLargeException(); } else if (throwable instanceof CallQueueTooBigException) { source.callQueueTooBigException(); } } }
@Override public void preGetOp(ObserverContext<RegionCoprocessorEnvironment> e, Get get, List<Cell> results) throws IOException { byte[] errorType = get.getAttribute(SHOULD_ERROR_ATTRIBUTE); if (errorType != null) { ErrorType type = ErrorType.valueOf(Bytes.toString(errorType)); switch (type) { case CALL_QUEUE_TOO_BIG: throw new CallQueueTooBigException("Failing for test"); case MULTI_ACTION_RESULT_TOO_LARGE: throw new MultiActionResultTooLarge("Failing for test"); case FAILED_SANITY_CHECK: throw new FailedSanityCheckException("Failing for test"); case NOT_SERVING_REGION: throw new NotServingRegionException("Failing for test"); case REGION_MOVED: throw new RegionMovedException(e.getEnvironment().getServerName(), 1); case SCANNER_RESET: throw new ScannerResetException("Failing for test"); case UNKNOWN_SCANNER: throw new UnknownScannerException("Failing for test"); case REGION_TOO_BUSY: throw new RegionTooBusyException("Failing for test"); case OUT_OF_ORDER_SCANNER_NEXT: throw new OutOfOrderScannerNextException("Failing for test"); default: throw new DoNotRetryIOException("Failing for test"); } } }
@Override public synchronized boolean next(List<Cell> outResults, ScannerContext scannerContext) throws IOException { if (this.filterClosed) { throw new UnknownScannerException("Scanner was closed (timed out?) " + "after we renewed it. Could be caused by a very slow scanner " + "or a lengthy garbage collection"); } startRegionOperation(Operation.SCAN); try { return nextRaw(outResults, scannerContext); } finally { closeRegionOperation(Operation.SCAN); } }
@Override public boolean nextRaw(List<Cell> outResults, ScannerContext scannerContext) throws IOException { if (storeHeap == null) { // scanner is closed throw new UnknownScannerException("Scanner was closed"); } boolean moreValues = false; if (outResults.isEmpty()) { // Usually outResults is empty. This is true when next is called // to handle scan or get operation. moreValues = nextInternal(outResults, scannerContext); } else { List<Cell> tmpList = new ArrayList<>(); moreValues = nextInternal(tmpList, scannerContext); outResults.addAll(tmpList); } if (!outResults.isEmpty()) { readRequestsCount.increment(); } // If the size limit was reached it means a partial Result is being returned. Returning a // partial Result means that we should not reset the filters; filters should only be reset in // between rows if (!scannerContext.mayHaveMoreCellsInRow()) { resetFilters(); } if (isFilterDoneInternal()) { moreValues = false; } return moreValues; }
public void exception(Throwable throwable) { source.exception(); /** * Keep some metrics for commonly seen exceptions * * Try and put the most common types first. * Place child types before the parent type that they extend. * * If this gets much larger we might have to go to a hashmap */ if (throwable != null) { if (throwable instanceof OutOfOrderScannerNextException) { source.outOfOrderException(); } else if (throwable instanceof RegionTooBusyException) { source.tooBusyException(); } else if (throwable instanceof UnknownScannerException) { source.unknownScannerException(); } else if (throwable instanceof ScannerResetException) { source.scannerResetException(); } else if (throwable instanceof RegionMovedException) { source.movedRegionException(); } else if (throwable instanceof NotServingRegionException) { source.notServingRegionException(); } else if (throwable instanceof FailedSanityCheckException) { source.failedSanityException(); } else if (throwable instanceof MultiActionResultTooLarge) { source.multiActionTooLargeException(); } else if (throwable instanceof CallQueueTooBigException) { source.callQueueTooBigException(); } } }
@Override public synchronized boolean next(List<KeyValue> outResults, int limit) throws IOException { if (this.filterClosed) { throw new UnknownScannerException("Scanner was closed (timed out?) " + "after we renewed it. Could be caused by a very slow scanner " + "or a lengthy garbage collection"); } startRegionOperation(); readRequestsCount.increment(); try { // This could be a new thread from the last time we called next(). MultiVersionConsistencyControl.setThreadReadPoint(this.readPt); results.clear(); boolean returnResult = nextInternal(limit); outResults.addAll(results); resetFilters(); if (isFilterDone()) { return false; } return returnResult; } finally { closeRegionOperation(); } }
/** * Load up 1 tables over 2 region servers and kill a source during * the upload. The failover happens internally. * * WARNING this test sometimes fails because of HBASE-3515 * * @throws Exception */ public void loadTableAndKillRS(HBaseTestingUtility util) throws Exception { // killing the RS with hbase:meta can result into failed puts until we solve // IO fencing int rsToKill1 = util.getHBaseCluster().getServerWithMeta() == 0 ? 1 : 0; // Takes about 20 secs to run the full loading, kill around the middle Thread killer = killARegionServer(util, 5000, rsToKill1); LOG.info("Start loading table"); int initialCount = utility1.loadTable((HTable)htable1, famName); LOG.info("Done loading table"); killer.join(5000); LOG.info("Done waiting for threads"); Result[] res; while (true) { try { Scan scan = new Scan(); ResultScanner scanner = htable1.getScanner(scan); res = scanner.next(initialCount); scanner.close(); break; } catch (UnknownScannerException ex) { LOG.info("Cluster wasn't ready yet, restarting scanner"); } } // Test we actually have all the rows, we may miss some because we // don't have IO fencing. if (res.length != initialCount) { LOG.warn("We lost some rows on the master cluster!"); // We don't really expect the other cluster to have more rows initialCount = res.length; } int lastCount = 0; final long start = System.currentTimeMillis(); int i = 0; while (true) { if (i==NB_RETRIES-1) { fail("Waited too much time for queueFailover replication. " + "Waited "+(System.currentTimeMillis() - start)+"ms."); } Scan scan2 = new Scan(); ResultScanner scanner2 = htable2.getScanner(scan2); Result[] res2 = scanner2.next(initialCount * 2); scanner2.close(); if (res2.length < initialCount) { if (lastCount < res2.length) { i--; // Don't increment timeout if we make progress } else { i++; } lastCount = res2.length; LOG.info("Only got " + lastCount + " rows instead of " + initialCount + " current i=" + i); Thread.sleep(SLEEP_TIME*2); } else { break; } } }
/** * @see java.util.concurrent.Callable#call() */ public Result[] call() throws IOException { if (scannerId != -1L && closed) { close(); } else if (scannerId == -1L && !closed) { this.scannerId = openScanner(); } else { Result[] rrs = null; try { incRPCcallsMetrics(); long timestamp = System.currentTimeMillis(); rrs = server.next(scannerId, caching); if (logScannerActivity) { long now = System.currentTimeMillis(); if (now - timestamp > logCutOffLatency) { int rows = rrs == null ? 0 : rrs.length; LOG.info("Took " + (now - timestamp) + "ms to fetch " + rows + " rows from scanner=" + scannerId); } } updateResultsMetrics(rrs); } catch (IOException e) { if (logScannerActivity) { LOG.info("Got exception in fetching from scanner=" + scannerId, e); } IOException ioe = null; if (e instanceof RemoteException) { ioe = RemoteExceptionHandler.decodeRemoteException((RemoteException) e); } if (ioe == null) throw new IOException(e); if (logScannerActivity && (ioe instanceof UnknownScannerException)) { try { HRegionLocation location = connection.relocateRegion(tableName, scan.getStartRow()); LOG.info("Scanner=" + scannerId + " expired, current region location is " + location.toString() + " ip:" + location.getServerAddress().getBindAddress()); } catch (Throwable t) { LOG.info("Failed to relocate region", t); } } if (ioe instanceof NotServingRegionException) { // Throw a DNRE so that we break out of cycle of calling NSRE // when what we need is to open scanner against new location. // Attach NSRE to signal client that it needs to resetup scanner. if (this.scanMetrics != null) { this.scanMetrics.countOfNSRE.inc(); } throw new DoNotRetryIOException("Reset scanner", ioe); } else if (ioe instanceof RegionServerStoppedException) { // Throw a DNRE so that we break out of cycle of calling RSSE // when what we need is to open scanner against new location. // Attach RSSE to signal client that it needs to resetup scanner. throw new DoNotRetryIOException("Reset scanner", ioe); } else { // The outer layers will retry throw ioe; } } return rrs; } return null; }
/** * Load up multiple tables over 2 region servers and kill a source during * the upload. The failover happens internally. * * WARNING this test sometimes fails because of HBASE-3515 * * @throws Exception */ @Test(timeout=300000) public void queueFailover() throws Exception { // killing the RS with .META. can result into failed puts until we solve // IO fencing int rsToKill1 = utility1.getHBaseCluster().getServerWithMeta() == 0 ? 1 : 0; int rsToKill2 = utility2.getHBaseCluster().getServerWithMeta() == 0 ? 1 : 0; // Takes about 20 secs to run the full loading, kill around the middle Thread killer1 = killARegionServer(utility1, 7500, rsToKill1); Thread killer2 = killARegionServer(utility2, 10000, rsToKill2); LOG.info("Start loading table"); int initialCount = utility1.loadTable(htable1, famName); LOG.info("Done loading table"); killer1.join(5000); killer2.join(5000); LOG.info("Done waiting for threads"); Result[] res; while (true) { try { Scan scan = new Scan(); ResultScanner scanner = htable1.getScanner(scan); res = scanner.next(initialCount); scanner.close(); break; } catch (UnknownScannerException ex) { LOG.info("Cluster wasn't ready yet, restarting scanner"); } } // Test we actually have all the rows, we may miss some because we // don't have IO fencing. if (res.length != initialCount) { LOG.warn("We lost some rows on the master cluster!"); // We don't really expect the other cluster to have more rows initialCount = res.length; } int lastCount = 0; final long start = System.currentTimeMillis(); int i = 0; while (true) { if (i==NB_RETRIES-1) { fail("Waited too much time for queueFailover replication. " + "Waited "+(System.currentTimeMillis() - start)+"ms."); } Scan scan2 = new Scan(); ResultScanner scanner2 = htable2.getScanner(scan2); Result[] res2 = scanner2.next(initialCount * 2); scanner2.close(); if (res2.length < initialCount) { if (lastCount < res2.length) { i--; // Don't increment timeout if we make progress } else { i++; } lastCount = res2.length; LOG.info("Only got " + lastCount + " rows instead of " + initialCount + " current i=" + i); Thread.sleep(SLEEP_TIME*2); } else { break; } } }