/** * Based on the request content, returns the deadline of the request. * * @param header * @param param * @return Deadline of this request. 0 now, otherwise msec of 'delay' */ @Override public long getDeadline(RequestHeader header, Message param) { if (param instanceof ScanRequest) { ScanRequest request = (ScanRequest)param; if (!request.hasScannerId()) { return 0; } // get the 'virtual time' of the scanner, and applies sqrt() to get a // nice curve for the delay. More a scanner is used the less priority it gets. // The weight is used to have more control on the delay. long vtime = rpcServices.getScannerVirtualTime(request.getScannerId()); return Math.round(Math.sqrt(vtime * scanVirtualTimeWeight)); } return 0; }
private void close() { if (this.scannerId == -1L) { return; } try { incRPCcallsMetrics(); ScanRequest request = RequestConverter.buildScanRequest(this.scannerId, 0, true, this.scanMetrics != null); try { getStub().scan(null, request); } catch (ServiceException se) { throw ProtobufUtil.getRemoteException(se); } } catch (IOException e) { LOG.warn("Ignore, probably already closed", e); } this.scannerId = -1L; }
protected long openScanner() throws IOException { incRPCcallsMetrics(); ScanRequest request = RequestConverter.buildScanRequest( getLocation().getRegionInfo().getRegionName(), this.scan, 0, false); try { ScanResponse response = getStub().scan(null, request); long id = response.getScannerId(); if (logScannerActivity) { LOG.info("Open scanner=" + id + " for scan=" + scan.toString() + " on region " + getLocation().toString()); } return id; } catch (ServiceException se) { throw ProtobufUtil.getRemoteException(se); } }
ScanOpenNextThenExceptionThenRecoverConnection(Configuration conf, boolean managed, ExecutorService pool) throws IOException { super(conf, managed); // Mock up my stub so open scanner returns a scanner id and then on next, we throw // exceptions for three times and then after that, we return no more to scan. this.stub = Mockito.mock(ClientService.BlockingInterface.class); long sid = 12345L; try { Mockito.when(stub.scan((RpcController)Mockito.any(), (ClientProtos.ScanRequest)Mockito.any())). thenReturn(ClientProtos.ScanResponse.newBuilder().setScannerId(sid).build()). thenThrow(new ServiceException(new RegionServerStoppedException("From Mockito"))). thenReturn(ClientProtos.ScanResponse.newBuilder().setScannerId(sid). setMoreResults(false).build()); } catch (ServiceException e) { throw new IOException(e); } }
RegionServerStoppedOnScannerOpenConnection(Configuration conf, boolean managed, ExecutorService pool, User user) throws IOException { super(conf, managed); // Mock up my stub so open scanner returns a scanner id and then on next, we throw // exceptions for three times and then after that, we return no more to scan. this.stub = Mockito.mock(ClientService.BlockingInterface.class); long sid = 12345L; try { Mockito.when(stub.scan((RpcController)Mockito.any(), (ClientProtos.ScanRequest)Mockito.any())). thenReturn(ClientProtos.ScanResponse.newBuilder().setScannerId(sid).build()). thenThrow(new ServiceException(new RegionServerStoppedException("From Mockito"))). thenReturn(ClientProtos.ScanResponse.newBuilder().setScannerId(sid). setMoreResults(false).build()); } catch (ServiceException e) { throw new IOException(e); } }
/** * Based on the request content, returns the deadline of the request. * * @param header * @param param * @return Deadline of this request. 0 now, otherwise msec of 'delay' */ @Override public long getDeadline(RequestHeader header, Message param) { String methodName = header.getMethodName(); if (methodName.equalsIgnoreCase("scan")) { ScanRequest request = (ScanRequest)param; if (!request.hasScannerId()) { return 0; } // get the 'virtual time' of the scanner, and applies sqrt() to get a // nice curve for the delay. More a scanner is used the less priority it gets. // The weight is used to have more control on the delay. long vtime = rpcServices.getScannerVirtualTime(request.getScannerId()); return Math.round(Math.sqrt(vtime * scanVirtualTimeWeight)); } return 0; }
/** * attempt to locate the region and perform a get and scan * @return True if successful, False otherwise. */ private boolean test(HConnection con, TableName tableName, byte[] row, HRegionServer server) { // not using HTable to avoid timeouts and retries try { byte[] regionName = con.relocateRegion(tableName, row).getRegionInfo() .getRegionName(); // get and scan should now succeed without exception ClientProtos.GetRequest request = RequestConverter.buildGetRequest(regionName, new Get(row)); server.getRSRpcServices().get(null, request); ScanRequest scanRequest = RequestConverter.buildScanRequest( regionName, new Scan(row), 1, true); try { server.getRSRpcServices().scan( new PayloadCarryingRpcController(), scanRequest); } catch (ServiceException se) { throw ProtobufUtil.getRemoteException(se); } } catch (IOException x) { return false; } catch (ServiceException e) { return false; } return true; }
private void close() { if (this.scannerId == -1L) { return; } try { incRPCcallsMetrics(); ScanRequest request = RequestConverter.buildScanRequest(this.scannerId, 0, true); try { getStub().scan(null, request); } catch (ServiceException se) { throw ProtobufUtil.getRemoteException(se); } } catch (IOException e) { LOG.warn("Ignore, probably already closed", e); } this.scannerId = -1L; }
@Override public Result[] call(int timeout) throws IOException { if (this.closed) return null; if (Thread.interrupted()) { throw new InterruptedIOException(); } ScanRequest request = RequestConverter.buildScanRequest(getLocation() .getRegionInfo().getRegionName(), getScan(), getCaching(), true); ScanResponse response = null; PayloadCarryingRpcController controller = controllerFactory.newController(); try { controller.setPriority(getTableName()); controller.setCallTimeout(timeout); response = getStub().scan(controller, request); return ResponseConverter.getResults(controller.cellScanner(), response); } catch (ServiceException se) { throw ProtobufUtil.getRemoteException(se); } }
/** * attempt to locate the region and perform a get and scan * @return True if successful, False otherwise. */ private boolean test(HConnection con, TableName tableName, byte[] row, HRegionServer server) { // not using HTable to avoid timeouts and retries try { byte[] regionName = con.relocateRegion(tableName, row).getRegionInfo() .getRegionName(); // get and scan should now succeed without exception ProtobufUtil.get(server, regionName, new Get(row)); ScanRequest scanRequest = RequestConverter.buildScanRequest( regionName, new Scan(row), 1, true); try { server.scan(new PayloadCarryingRpcController(), scanRequest); } catch (ServiceException se) { throw ProtobufUtil.getRemoteException(se); } } catch (IOException x) { return false; } return true; }
static RegionServerCallable<Result[]> getSmallScanCallable( final Scan sc, HConnection connection, TableName table, byte[] localStartKey, final int cacheNum, final RpcControllerFactory rpcControllerFactory) throws IOException { sc.setStartRow(localStartKey); RegionServerCallable<Result[]> callable = new RegionServerCallable<Result[]>( connection, table, sc.getStartRow()) { public Result[] call() throws IOException { ScanRequest request = RequestConverter.buildScanRequest(getLocation() .getRegionInfo().getRegionName(), sc, cacheNum, true); ScanResponse response = null; PayloadCarryingRpcController controller = rpcControllerFactory.newController(); try { controller.setPriority(getTableName()); response = getStub().scan(controller, request); return ResponseConverter.getResults(controller.cellScanner(), response); } catch (ServiceException se) { throw ProtobufUtil.getRemoteException(se); } } }; return callable; }
static RegionServerCallable<Result[]> getSmallScanCallable( final Scan sc, HConnection connection, TableName table, byte[] localStartKey, final int cacheNum, final RpcControllerFactory rpcControllerFactory) { sc.setStartRow(localStartKey); RegionServerCallable<Result[]> callable = new RegionServerCallable<Result[]>( connection, table, sc.getStartRow()) { public Result[] call(int callTimeout) throws IOException { ScanRequest request = RequestConverter.buildScanRequest(getLocation() .getRegionInfo().getRegionName(), sc, cacheNum, true); PayloadCarryingRpcController controller = rpcControllerFactory.newController(); controller.setPriority(getTableName()); controller.setCallTimeout(callTimeout); try { ScanResponse response = getStub().scan(controller, request); return ResponseConverter.getResults(controller.cellScanner(), response); } catch (ServiceException se) { throw ProtobufUtil.getRemoteException(se); } } }; return callable; }
private RegionServerCallable<Result[]> getSmallScanCallable( byte[] localStartKey, final int cacheNum) { this.scan.setStartRow(localStartKey); RegionServerCallable<Result[]> callable = new RegionServerCallable<Result[]>( getConnection(), getTable(), scan.getStartRow()) { public Result[] call() throws IOException { ScanRequest request = RequestConverter.buildScanRequest(getLocation() .getRegionInfo().getRegionName(), scan, cacheNum, true); ScanResponse response = null; PayloadCarryingRpcController controller = new PayloadCarryingRpcController(); try { controller.setPriority(getTableName()); response = getStub().scan(controller, request); return ResponseConverter.getResults(controller.cellScanner(), response); } catch (ServiceException se) { throw ProtobufUtil.getRemoteException(se); } } }; return callable; }
private void close() { if (this.scannerId == -1L) { return; } try { incRPCcallsMetrics(); ScanRequest request = RequestConverter.buildScanRequest(this.scannerId, 0, true); try { server.scan(null, request); } catch (ServiceException se) { throw ProtobufUtil.getRemoteException(se); } } catch (IOException e) { LOG.warn("Ignore, probably already closed", e); } this.scannerId = -1L; }
protected long openScanner() throws IOException { incRPCcallsMetrics(); ScanRequest request = RequestConverter.buildScanRequest( this.location.getRegionInfo().getRegionName(), this.scan, 0, false); try { ScanResponse response = server.scan(null, request); long id = response.getScannerId(); if (logScannerActivity) { LOG.info("Open scanner=" + id + " for scan=" + scan.toString() + " on region " + this.location.toString() + " ip:" + this.location.getHostnamePort()); } return id; } catch (ServiceException se) { throw ProtobufUtil.getRemoteException(se); } }
/** * attempt to locate the region and perform a get and scan * @return True if successful, False otherwise. */ private boolean test(HConnection con, byte[] tableName, byte[] row, HRegionServer server) { // not using HTable to avoid timeouts and retries try { byte[] regionName = con.relocateRegion(tableName, row).getRegionInfo() .getRegionName(); // get and scan should now succeed without exception ProtobufUtil.get(server, regionName, new Get(row)); ScanRequest scanRequest = RequestConverter.buildScanRequest( regionName, new Scan(row), 1, true); try { server.scan(null, scanRequest); } catch (ServiceException se) { throw ProtobufUtil.getRemoteException(se); } } catch (IOException x) { return false; } return true; }
private boolean isScanRequest(final RequestHeader header, final Message param) { if (param instanceof ScanRequest) { // The first scan request will be executed as a "short read" ScanRequest request = (ScanRequest)param; return request.hasScannerId(); } return false; }
@Override public ScanResponse scan(final RpcController controller, final ScanRequest request) throws ServiceException { if (request.hasScannerId()) { ScanResponse scanResponse = super.scan(controller, request); if (this.tableScannerId == request.getScannerId() && (sleepAlways || (!slept && seqNoToSleepOn == request.getNextCallSeq()))) { try { LOG.info("SLEEPING " + (rpcTimeout + 500)); Thread.sleep(rpcTimeout + 500); } catch (InterruptedException e) { } slept = true; tryNumber++; if (tryNumber > 2 * CLIENT_RETRIES_NUMBER) { sleepAlways = false; } } return scanResponse; } else { ScanResponse scanRes = super.scan(controller, request); String regionName = Bytes.toString(request.getRegion().getValue().toByteArray()); if (!regionName.contains(TableName.META_TABLE_NAME.getNameAsString())) { tableScannerId = scanRes.getScannerId(); } return scanRes; } }
@Override public ScanResponse scan(RpcController controller, ScanRequest request) throws ServiceException { ScanResponse.Builder builder = ScanResponse.newBuilder(); try { if (request.hasScan()) { byte[] regionName = request.getRegion().getValue().toByteArray(); builder.setScannerId(openScanner(regionName, null)); builder.setMoreResults(true); } else { long scannerId = request.getScannerId(); Result result = next(scannerId); if (result != null) { builder.addCellsPerResult(result.size()); List<CellScannable> results = new ArrayList<CellScannable>(1); results.add(result); ((PayloadCarryingRpcController) controller).setCellScanner(CellUtil .createCellScanner(results)); builder.setMoreResults(true); } else { builder.setMoreResults(false); close(scannerId); } } } catch (IOException ie) { throw new ServiceException(ie); } return builder.build(); }
@Override public ScanResponse scan(RpcController controller, ScanRequest request) throws ServiceException { ScanRequest.Builder builder = ScanRequest.newBuilder(request); builder.setClientHandlesHeartbeats(heartbeatsEnabled); return super.scan(controller, builder.build()); }
/** * Create a protocol buffer ScanRequest for a client Scan * * @param regionName * @param scan * @param numberOfRows * @param closeScanner * @return a scan request * @throws IOException */ public static ScanRequest buildScanRequest(final byte[] regionName, final Scan scan, final int numberOfRows, final boolean closeScanner) throws IOException { ScanRequest.Builder builder = ScanRequest.newBuilder(); RegionSpecifier region = buildRegionSpecifier( RegionSpecifierType.REGION_NAME, regionName); builder.setNumberOfRows(numberOfRows); builder.setCloseScanner(closeScanner); builder.setRegion(region); builder.setScan(ProtobufUtil.toScan(scan)); builder.setClientHandlesPartials(true); builder.setClientHandlesHeartbeats(true); builder.setTrackScanMetrics(scan.isScanMetricsEnabled()); return builder.build(); }
/** * Create a protocol buffer ScanRequest for a scanner id * * @param scannerId * @param numberOfRows * @param closeScanner * @return a scan request */ public static ScanRequest buildScanRequest(final long scannerId, final int numberOfRows, final boolean closeScanner, final boolean trackMetrics) { ScanRequest.Builder builder = ScanRequest.newBuilder(); builder.setNumberOfRows(numberOfRows); builder.setCloseScanner(closeScanner); builder.setScannerId(scannerId); builder.setClientHandlesPartials(true); builder.setClientHandlesHeartbeats(true); builder.setTrackScanMetrics(trackMetrics); return builder.build(); }