@SuppressWarnings("deprecation") private HConnection getMockedConnection(final Configuration conf) throws IOException, ServiceException { HConnection c = Mockito.mock(HConnection.class); Mockito.when(c.getConfiguration()).thenReturn(conf); Mockito.doNothing().when(c).close(); // Make it so we return a particular location when asked. final HRegionLocation loc = new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, ServerName.valueOf("example.org", 1234, 0)); Mockito.when(c.getRegionLocation((TableName) Mockito.any(), (byte[]) Mockito.any(), Mockito.anyBoolean())). thenReturn(loc); Mockito.when(c.locateRegion((TableName) Mockito.any(), (byte[]) Mockito.any())). thenReturn(loc); ClientProtos.ClientService.BlockingInterface hri = Mockito.mock(ClientProtos.ClientService.BlockingInterface.class); Mockito.when(hri.bulkLoadHFile((RpcController)Mockito.any(), (BulkLoadHFileRequest)Mockito.any())). thenThrow(new ServiceException(new IOException("injecting bulk load error"))); Mockito.when(c.getClient(Mockito.any(ServerName.class))). thenReturn(hri); return c; }
/** * Create a protocol buffer bulk load request * * @param familyPaths * @param regionName * @param assignSeqNum * @return a bulk load request */ public static BulkLoadHFileRequest buildBulkLoadHFileRequest( final List<Pair<byte[], String>> familyPaths, final byte[] regionName, boolean assignSeqNum) { BulkLoadHFileRequest.Builder builder = BulkLoadHFileRequest.newBuilder(); RegionSpecifier region = buildRegionSpecifier( RegionSpecifierType.REGION_NAME, regionName); builder.setRegion(region); FamilyPath.Builder familyPathBuilder = FamilyPath.newBuilder(); for (Pair<byte[], String> familyPath: familyPaths) { familyPathBuilder.setFamily(ByteStringer.wrap(familyPath.getFirst())); familyPathBuilder.setPath(familyPath.getSecond()); builder.addFamilyPath(familyPathBuilder.build()); } builder.setAssignSeqNum(assignSeqNum); return builder.build(); }
private HConnection getMockedConnection(final Configuration conf) throws IOException, ServiceException { HConnection c = Mockito.mock(HConnection.class); Mockito.when(c.getConfiguration()).thenReturn(conf); Mockito.doNothing().when(c).close(); // Make it so we return a particular location when asked. final HRegionLocation loc = new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, ServerName.valueOf("example.org", 1234, 0)); Mockito.when(c.getRegionLocation((TableName) Mockito.any(), (byte[]) Mockito.any(), Mockito.anyBoolean())). thenReturn(loc); Mockito.when(c.locateRegion((TableName) Mockito.any(), (byte[]) Mockito.any())). thenReturn(loc); ClientProtos.ClientService.BlockingInterface hri = Mockito.mock(ClientProtos.ClientService.BlockingInterface.class); Mockito.when(hri.bulkLoadHFile((RpcController)Mockito.any(), (BulkLoadHFileRequest)Mockito.any())). thenThrow(new ServiceException(new IOException("injecting bulk load error"))); Mockito.when(c.getClient(Mockito.any(ServerName.class))). thenReturn(hri); return c; }
/** * Create a protocol buffer bulk load request * * @param familyPaths * @param regionName * @param assignSeqNum * @return a bulk load request */ public static BulkLoadHFileRequest buildBulkLoadHFileRequest( final List<Pair<byte[], String>> familyPaths, final byte[] regionName, boolean assignSeqNum) { BulkLoadHFileRequest.Builder builder = BulkLoadHFileRequest.newBuilder(); RegionSpecifier region = buildRegionSpecifier( RegionSpecifierType.REGION_NAME, regionName); builder.setRegion(region); FamilyPath.Builder familyPathBuilder = FamilyPath.newBuilder(); for (Pair<byte[], String> familyPath: familyPaths) { familyPathBuilder.setFamily(HBaseZeroCopyByteString.wrap(familyPath.getFirst())); familyPathBuilder.setPath(familyPath.getSecond()); builder.addFamilyPath(familyPathBuilder.build()); } builder.setAssignSeqNum(assignSeqNum); return builder.build(); }
@Override public void secureBulkLoadHFiles(RpcController controller, SecureBulkLoadHFilesRequest request, RpcCallback<SecureBulkLoadHFilesResponse> done) { boolean loaded = false; Map<byte[], List<Path>> map = null; try { SecureBulkLoadManager secureBulkLoadManager = this.rsServices.getSecureBulkLoadManager(); BulkLoadHFileRequest bulkLoadHFileRequest = ConvertSecureBulkLoadHFilesRequest(request); map = secureBulkLoadManager.secureBulkLoadHFiles((HRegion) this.env.getRegion(), convert(bulkLoadHFileRequest)); loaded = map != null && !map.isEmpty(); } catch (IOException e) { CoprocessorRpcUtils.setControllerException(controller, e); } done.run(SecureBulkLoadHFilesResponse.newBuilder().setLoaded(loaded).build()); }
/** * Create a protocol buffer bulk load request * * @param familyPaths * @param regionName * @param assignSeqNum * @return a bulk load request */ public static BulkLoadHFileRequest buildBulkLoadHFileRequest( final List<Pair<byte[], String>> familyPaths, final byte[] regionName, boolean assignSeqNum) { BulkLoadHFileRequest.Builder builder = BulkLoadHFileRequest.newBuilder(); RegionSpecifier region = buildRegionSpecifier( RegionSpecifierType.REGION_NAME, regionName); builder.setRegion(region); FamilyPath.Builder familyPathBuilder = FamilyPath.newBuilder(); for (Pair<byte[], String> familyPath: familyPaths) { familyPathBuilder.setFamily(ZeroCopyLiteralByteString.wrap(familyPath.getFirst())); familyPathBuilder.setPath(familyPath.getSecond()); builder.addFamilyPath(familyPathBuilder.build()); } builder.setAssignSeqNum(assignSeqNum); return builder.build(); }
/** * Create a protocol buffer bulk load request * * @param familyPaths * @param regionName * @param assignSeqNum * @return a bulk load request */ public static BulkLoadHFileRequest buildBulkLoadHFileRequest( final List<Pair<byte[], String>> familyPaths, final byte[] regionName, boolean assignSeqNum) { BulkLoadHFileRequest.Builder builder = BulkLoadHFileRequest.newBuilder(); RegionSpecifier region = buildRegionSpecifier( RegionSpecifierType.REGION_NAME, regionName); builder.setRegion(region); FamilyPath.Builder familyPathBuilder = FamilyPath.newBuilder(); for (Pair<byte[], String> familyPath: familyPaths) { familyPathBuilder.setFamily(ByteString.copyFrom(familyPath.getFirst())); familyPathBuilder.setPath(familyPath.getSecond()); builder.addFamilyPath(familyPathBuilder.build()); } builder.setAssignSeqNum(assignSeqNum); return builder.build(); }
private HConnection getMockedConnection(final Configuration conf) throws IOException, ServiceException { HConnection c = Mockito.mock(HConnection.class); Mockito.when(c.getConfiguration()).thenReturn(conf); Mockito.doNothing().when(c).close(); // Make it so we return a particular location when asked. final HRegionLocation loc = new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, "example.org", 1234); Mockito.when(c.getRegionLocation((byte[]) Mockito.any(), (byte[]) Mockito.any(), Mockito.anyBoolean())). thenReturn(loc); Mockito.when(c.locateRegion((byte[]) Mockito.any(), (byte[]) Mockito.any())). thenReturn(loc); ClientProtocol hri = Mockito.mock(ClientProtocol.class); Mockito.when(hri.bulkLoadHFile((RpcController)Mockito.any(), (BulkLoadHFileRequest)Mockito.any())). thenThrow(new ServiceException(new IOException("injecting bulk load error"))); Mockito.when(c.getClient(Mockito.anyString(), Mockito.anyInt())). thenReturn(hri); return c; }
/** * Atomically bulk load several HFiles into an open region * @return true if successful, false is failed but recoverably (no action) * @throws ServiceException if failed unrecoverably */ @Override public BulkLoadHFileResponse bulkLoadHFile(final RpcController controller, final BulkLoadHFileRequest request) throws ServiceException { try { checkOpen(); requestCount.increment(); Region region = getRegion(request.getRegion()); List<Pair<byte[], String>> familyPaths = new ArrayList<Pair<byte[], String>>(); for (FamilyPath familyPath: request.getFamilyPathList()) { familyPaths.add(new Pair<byte[], String>(familyPath.getFamily().toByteArray(), familyPath.getPath())); } boolean bypass = false; if (region.getCoprocessorHost() != null) { bypass = region.getCoprocessorHost().preBulkLoadHFile(familyPaths); } boolean loaded = false; if (!bypass) { loaded = region.bulkLoadHFiles(familyPaths, request.getAssignSeqNum(), null); } if (region.getCoprocessorHost() != null) { loaded = region.getCoprocessorHost().postBulkLoadHFile(familyPaths, loaded); } BulkLoadHFileResponse.Builder builder = BulkLoadHFileResponse.newBuilder(); builder.setLoaded(loaded); return builder.build(); } catch (IOException ie) { throw new ServiceException(ie); } }
/** * A helper to bulk load a list of HFiles using client protocol. * * @param client * @param familyPaths * @param regionName * @param assignSeqNum * @return true if all are loaded * @throws IOException */ public static boolean bulkLoadHFile(final ClientService.BlockingInterface client, final List<Pair<byte[], String>> familyPaths, final byte[] regionName, boolean assignSeqNum) throws IOException { BulkLoadHFileRequest request = RequestConverter.buildBulkLoadHFileRequest(familyPaths, regionName, assignSeqNum); try { BulkLoadHFileResponse response = client.bulkLoadHFile(null, request); return response.getLoaded(); } catch (ServiceException se) { throw getRemoteException(se); } }
/** * Atomically bulk load several HFiles into an open region * * @return true if successful, false is failed but recoverably (no action) * @throws IOException if failed unrecoverably */ @Override public BulkLoadHFileResponse bulkLoadHFile(final RpcController controller, final BulkLoadHFileRequest request) throws ServiceException { try { checkOpen(); requestCount.increment(); HRegion region = getRegion(request.getRegion()); List<Pair<byte[], String>> familyPaths = new ArrayList<Pair<byte[], String>>(); for (FamilyPath familyPath : request.getFamilyPathList()) { familyPaths.add(new Pair<byte[], String>(familyPath.getFamily().toByteArray(), familyPath.getPath())); } boolean bypass = false; if (region.getCoprocessorHost() != null) { bypass = region.getCoprocessorHost().preBulkLoadHFile(familyPaths); } boolean loaded = false; if (!bypass) { loaded = region.bulkLoadHFiles(familyPaths, request.getAssignSeqNum()); } if (region.getCoprocessorHost() != null) { loaded = region.getCoprocessorHost().postBulkLoadHFile(familyPaths, loaded); } BulkLoadHFileResponse.Builder builder = BulkLoadHFileResponse.newBuilder(); builder.setLoaded(loaded); return builder.build(); } catch (IOException ie) { throw new ServiceException(ie); } }
/** * Atomically bulk load several HFiles into an open region * @return true if successful, false is failed but recoverably (no action) * @throws IOException if failed unrecoverably */ @Override public BulkLoadHFileResponse bulkLoadHFile(final RpcController controller, final BulkLoadHFileRequest request) throws ServiceException { try { checkOpen(); requestCount.increment(); HRegion region = getRegion(request.getRegion()); List<Pair<byte[], String>> familyPaths = new ArrayList<Pair<byte[], String>>(); for (FamilyPath familyPath: request.getFamilyPathList()) { familyPaths.add(new Pair<byte[], String>(familyPath.getFamily().toByteArray(), familyPath.getPath())); } boolean bypass = false; if (region.getCoprocessorHost() != null) { bypass = region.getCoprocessorHost().preBulkLoadHFile(familyPaths); } boolean loaded = false; if (!bypass) { loaded = region.bulkLoadHFiles(familyPaths, request.getAssignSeqNum()); } if (region.getCoprocessorHost() != null) { loaded = region.getCoprocessorHost().postBulkLoadHFile(familyPaths, loaded); } BulkLoadHFileResponse.Builder builder = BulkLoadHFileResponse.newBuilder(); builder.setLoaded(loaded); return builder.build(); } catch (IOException ie) { throw new ServiceException(ie); } }
/** * A helper to bulk load a list of HFiles using client protocol. * * @param client * @param familyPaths * @param regionName * @param assignSeqNum * @param controller * @return true if all are loaded * @throws IOException */ public static boolean bulkLoadHFile(final ClientService.BlockingInterface client, final List<Pair<byte[], String>> familyPaths, final byte[] regionName, boolean assignSeqNum, PayloadCarryingRpcController controller) throws IOException { BulkLoadHFileRequest request = RequestConverter.buildBulkLoadHFileRequest(familyPaths, regionName, assignSeqNum); try { BulkLoadHFileResponse response = client.bulkLoadHFile(controller, request); return response.getLoaded(); } catch (ServiceException se) { throw getRemoteException(se); } }
org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.BulkLoadHFileRequest convert(BulkLoadHFileRequest request) throws org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException { byte [] bytes = request.toByteArray(); org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.BulkLoadHFileRequest.Builder builder = org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.BulkLoadHFileRequest. newBuilder(); builder.mergeFrom(bytes); return builder.build(); }
private BulkLoadHFileRequest ConvertSecureBulkLoadHFilesRequest( SecureBulkLoadHFilesRequest request) { BulkLoadHFileRequest.Builder bulkLoadHFileRequest = BulkLoadHFileRequest.newBuilder(); RegionSpecifier region = ProtobufUtil.buildRegionSpecifier(RegionSpecifierType.REGION_NAME, this.env .getRegionInfo().getRegionName()); bulkLoadHFileRequest.setRegion(region).setFsToken(request.getFsToken()) .setBulkToken(request.getBulkToken()).setAssignSeqNum(request.getAssignSeqNum()) .addAllFamilyPath(request.getFamilyPathList()); return bulkLoadHFileRequest.build(); }
/** * Atomically bulk load several HFiles into an open region * @return true if successful, false is failed but recoverably (no action) * @throws IOException if failed unrecoverably */ @Override public BulkLoadHFileResponse bulkLoadHFile(final RpcController controller, final BulkLoadHFileRequest request) throws ServiceException { try { requestCount.increment(); HRegion region = getRegion(request.getRegion()); List<Pair<byte[], String>> familyPaths = new ArrayList<Pair<byte[], String>>(); for (FamilyPath familyPath: request.getFamilyPathList()) { familyPaths.add(new Pair<byte[], String>(familyPath.getFamily().toByteArray(), familyPath.getPath())); } boolean bypass = false; if (region.getCoprocessorHost() != null) { bypass = region.getCoprocessorHost().preBulkLoadHFile(familyPaths); } boolean loaded = false; if (!bypass) { loaded = region.bulkLoadHFiles(familyPaths, request.getAssignSeqNum()); } if (region.getCoprocessorHost() != null) { loaded = region.getCoprocessorHost().postBulkLoadHFile(familyPaths, loaded); } BulkLoadHFileResponse.Builder builder = BulkLoadHFileResponse.newBuilder(); builder.setLoaded(loaded); return builder.build(); } catch (IOException ie) { throw new ServiceException(ie); } }
/** * A helper to bulk load a list of HFiles using client protocol. * * @param client * @param familyPaths * @param regionName * @param assignSeqNum * @return true if all are loaded * @throws IOException */ public static boolean bulkLoadHFile(final ClientProtocol client, final List<Pair<byte[], String>> familyPaths, final byte[] regionName, boolean assignSeqNum) throws IOException { BulkLoadHFileRequest request = RequestConverter.buildBulkLoadHFileRequest(familyPaths, regionName, assignSeqNum); try { BulkLoadHFileResponse response = client.bulkLoadHFile(null, request); return response.getLoaded(); } catch (ServiceException se) { throw getRemoteException(se); } }
@Override public BulkLoadHFileResponse bulkLoadHFile(RpcController controller, BulkLoadHFileRequest request) throws ServiceException { // TODO Auto-generated method stub return null; }
public void doAnAction() throws Exception { long iteration = numBulkLoads.getAndIncrement(); Path dir = UTIL.getDataTestDirOnTestFS(String.format("bulkLoad_%08d", iteration)); // create HFiles for different column families FileSystem fs = UTIL.getTestFileSystem(); byte[] val = Bytes.toBytes(String.format("%010d", iteration)); final List<Pair<byte[], String>> famPaths = new ArrayList<Pair<byte[], String>>( NUM_CFS); for (int i = 0; i < NUM_CFS; i++) { Path hfile = new Path(dir, family(i)); byte[] fam = Bytes.toBytes(family(i)); createHFile(fs, hfile, fam, QUAL, val, 1000); famPaths.add(new Pair<byte[], String>(fam, hfile.toString())); } // bulk load HFiles final HConnection conn = UTIL.getHBaseAdmin().getConnection(); RegionServerCallable<Void> callable = new RegionServerCallable<Void>(conn, tableName, Bytes.toBytes("aaa")) { @Override public Void call(int callTimeout) throws Exception { LOG.debug("Going to connect to server " + getLocation() + " for row " + Bytes.toStringBinary(getRow())); byte[] regionName = getLocation().getRegionInfo().getRegionName(); BulkLoadHFileRequest request = RequestConverter.buildBulkLoadHFileRequest(famPaths, regionName, true); getStub().bulkLoadHFile(null, request); return null; } }; RpcRetryingCallerFactory factory = new RpcRetryingCallerFactory(conf); RpcRetryingCaller<Void> caller = factory.<Void> newCaller(); caller.callWithRetries(callable, Integer.MAX_VALUE); // Periodically do compaction to reduce the number of open file handles. if (numBulkLoads.get() % 10 == 0) { // 10 * 50 = 500 open file handles! callable = new RegionServerCallable<Void>(conn, tableName, Bytes.toBytes("aaa")) { @Override public Void call(int callTimeout) throws Exception { LOG.debug("compacting " + getLocation() + " for row " + Bytes.toStringBinary(getRow())); AdminProtos.AdminService.BlockingInterface server = conn.getAdmin(getLocation().getServerName()); CompactRegionRequest request = RequestConverter.buildCompactRegionRequest( getLocation().getRegionInfo().getRegionName(), true, null); server.compactRegion(null, request); numCompactions.incrementAndGet(); return null; } }; caller.callWithRetries(callable, Integer.MAX_VALUE); } }
@Override public BulkLoadHFileResponse bulkLoadHFile( RpcController controller, BulkLoadHFileRequest request) throws ServiceException { throw new NotImplementedException(); }
public void doAnAction() throws Exception { long iteration = numBulkLoads.getAndIncrement(); Path dir = UTIL.getDataTestDirOnTestFS(String.format("bulkLoad_%08d", iteration)); // create HFiles for different column families FileSystem fs = UTIL.getTestFileSystem(); byte[] val = Bytes.toBytes(String.format("%010d", iteration)); final List<Pair<byte[], String>> famPaths = new ArrayList<Pair<byte[], String>>( NUM_CFS); for (int i = 0; i < NUM_CFS; i++) { Path hfile = new Path(dir, family(i)); byte[] fam = Bytes.toBytes(family(i)); createHFile(fs, hfile, fam, QUAL, val, 1000); famPaths.add(new Pair<byte[], String>(fam, hfile.toString())); } // bulk load HFiles final HConnection conn = UTIL.getHBaseAdmin().getConnection(); TableName tbl = TableName.valueOf(tableName); RegionServerCallable<Void> callable = new RegionServerCallable<Void>(conn, tbl, Bytes.toBytes("aaa")) { @Override public Void call() throws Exception { LOG.debug("Going to connect to server " + getLocation() + " for row " + Bytes.toStringBinary(getRow())); byte[] regionName = getLocation().getRegionInfo().getRegionName(); BulkLoadHFileRequest request = RequestConverter.buildBulkLoadHFileRequest(famPaths, regionName, true); getStub().bulkLoadHFile(null, request); return null; } }; RpcRetryingCallerFactory factory = new RpcRetryingCallerFactory(conf); RpcRetryingCaller<Void> caller = factory.<Void> newCaller(); caller.callWithRetries(callable); // Periodically do compaction to reduce the number of open file handles. if (numBulkLoads.get() % 10 == 0) { // 10 * 50 = 500 open file handles! callable = new RegionServerCallable<Void>(conn, tbl, Bytes.toBytes("aaa")) { @Override public Void call() throws Exception { LOG.debug("compacting " + getLocation() + " for row " + Bytes.toStringBinary(getRow())); AdminProtos.AdminService.BlockingInterface server = conn.getAdmin(getLocation().getServerName()); CompactRegionRequest request = RequestConverter.buildCompactRegionRequest( getLocation().getRegionInfo().getRegionName(), true, null); server.compactRegion(null, request); numCompactions.incrementAndGet(); return null; } }; caller.callWithRetries(callable); } }
public void doAnAction() throws Exception { long iteration = numBulkLoads.getAndIncrement(); Path dir = UTIL.getDataTestDirOnTestFS(String.format("bulkLoad_%08d", iteration)); // create HFiles for different column families FileSystem fs = UTIL.getTestFileSystem(); byte[] val = Bytes.toBytes(String.format("%010d", iteration)); final List<Pair<byte[], String>> famPaths = new ArrayList<Pair<byte[], String>>( NUM_CFS); for (int i = 0; i < NUM_CFS; i++) { Path hfile = new Path(dir, family(i)); byte[] fam = Bytes.toBytes(family(i)); createHFile(fs, hfile, fam, QUAL, val, 1000); famPaths.add(new Pair<byte[], String>(fam, hfile.toString())); } // bulk load HFiles final HConnection conn = UTIL.getHBaseAdmin().getConnection(); TableName tbl = TableName.valueOf(tableName); RegionServerCallable<Void> callable = new RegionServerCallable<Void>(conn, tbl, Bytes.toBytes("aaa")) { @Override public Void call(int callTimeout) throws Exception { LOG.debug("Going to connect to server " + getLocation() + " for row " + Bytes.toStringBinary(getRow())); byte[] regionName = getLocation().getRegionInfo().getRegionName(); BulkLoadHFileRequest request = RequestConverter.buildBulkLoadHFileRequest(famPaths, regionName, true); getStub().bulkLoadHFile(null, request); return null; } }; RpcRetryingCallerFactory factory = new RpcRetryingCallerFactory(conf); RpcRetryingCaller<Void> caller = factory.<Void> newCaller(); caller.callWithRetries(callable, Integer.MAX_VALUE); // Periodically do compaction to reduce the number of open file handles. if (numBulkLoads.get() % 10 == 0) { // 10 * 50 = 500 open file handles! callable = new RegionServerCallable<Void>(conn, tbl, Bytes.toBytes("aaa")) { @Override public Void call(int callTimeout) throws Exception { LOG.debug("compacting " + getLocation() + " for row " + Bytes.toStringBinary(getRow())); AdminProtos.AdminService.BlockingInterface server = conn.getAdmin(getLocation().getServerName()); CompactRegionRequest request = RequestConverter.buildCompactRegionRequest( getLocation().getRegionInfo().getRegionName(), true, null); server.compactRegion(null, request); numCompactions.incrementAndGet(); return null; } }; caller.callWithRetries(callable, Integer.MAX_VALUE); } }
