/** * Creates a protocol buffer CreateTableRequest * * @param hTableDesc * @param splitKeys * @return a CreateTableRequest */ public static CreateTableRequest buildCreateTableRequest( final HTableDescriptor hTableDesc, final byte [][] splitKeys, final long nonceGroup, final long nonce) { CreateTableRequest.Builder builder = CreateTableRequest.newBuilder(); builder.setTableSchema(hTableDesc.convert()); if (splitKeys != null) { for (byte [] splitKey : splitKeys) { builder.addSplitKeys(ByteStringer.wrap(splitKey)); } } builder.setNonceGroup(nonceGroup); builder.setNonce(nonce); return builder.build(); }
/** * Shen Li: add parameter replicaNum */ public static CreateTableRequest buildCreateTableRequest( final HTableDescriptor hTableDesc, final byte [][] splitKeys, int replicaNum) { CreateTableRequest.Builder builder = CreateTableRequest.newBuilder(); builder.setTableSchema(hTableDesc.convert()); if (splitKeys != null) { for (byte [] splitKey : splitKeys) { builder.addSplitKeys(HBaseZeroCopyByteString.wrap(splitKey)); } } if (replicaNum > 1) { builder.setReplicaNum(replicaNum); } return builder.build(); }
@Override public CreateTableResponse createTable(RpcController controller, CreateTableRequest req) throws ServiceException { HTableDescriptor hTableDescriptor = HTableDescriptor.convert(req.getTableSchema()); byte [][] splitKeys = ProtobufUtil.getSplitKeysArray(req); try { long procId = master.createTable(hTableDescriptor, splitKeys, req.getNonceGroup(), req.getNonce()); return CreateTableResponse.newBuilder().setProcId(procId).build(); } catch (IOException ioe) { throw new ServiceException(ioe); } }
/** * get the split keys in form "byte [][]" from a CreateTableRequest proto * * @param proto the CreateTableRequest * @return the split keys */ public static byte [][] getSplitKeysArray(final CreateTableRequest proto) { byte [][] splitKeys = new byte[proto.getSplitKeysCount()][]; for (int i = 0; i < proto.getSplitKeysCount(); ++i) { splitKeys[i] = proto.getSplitKeys(i).toByteArray(); } return splitKeys; }
/** * Creates a new table but does not block and wait for it to come online. * You can use Future.get(long, TimeUnit) to wait on the operation to complete. * It may throw ExecutionException if there was an error while executing the operation * or TimeoutException in case the wait timeout was not long enough to allow the * operation to complete. * * @param desc table descriptor for table * @param splitKeys keys to check if the table has been created with all split keys * @throws IllegalArgumentException Bad table name, if the split keys * are repeated and if the split key has empty byte array. * @throws IOException if a remote or network exception occurs * @return the result of the async creation. You can use Future.get(long, TimeUnit) * to wait on the operation to complete. */ // TODO: This should be called Async but it will break binary compatibility private Future<Void> createTableAsyncV2(final HTableDescriptor desc, final byte[][] splitKeys) throws IOException { if (desc.getTableName() == null) { throw new IllegalArgumentException("TableName cannot be null"); } if (splitKeys != null && splitKeys.length > 0) { Arrays.sort(splitKeys, Bytes.BYTES_COMPARATOR); // Verify there are no duplicate split keys byte[] lastKey = null; for (byte[] splitKey : splitKeys) { if (Bytes.compareTo(splitKey, HConstants.EMPTY_BYTE_ARRAY) == 0) { throw new IllegalArgumentException( "Empty split key must not be passed in the split keys."); } if (lastKey != null && Bytes.equals(splitKey, lastKey)) { throw new IllegalArgumentException("All split keys must be unique, " + "found duplicate: " + Bytes.toStringBinary(splitKey) + ", " + Bytes.toStringBinary(lastKey)); } lastKey = splitKey; } } CreateTableResponse response = executeCallable( new MasterCallable<CreateTableResponse>(getConnection()) { @Override public CreateTableResponse call(int callTimeout) throws ServiceException { PayloadCarryingRpcController controller = rpcControllerFactory.newController(); controller.setCallTimeout(callTimeout); controller.setPriority(desc.getTableName()); CreateTableRequest request = RequestConverter.buildCreateTableRequest( desc, splitKeys, ng.getNonceGroup(), ng.newNonce()); return master.createTable(controller, request); } }); return new CreateTableFuture(this, desc, splitKeys, response); }
@Override public CreateTableResponse createTable(RpcController controller, CreateTableRequest req) throws ServiceException { HTableDescriptor hTableDescriptor = HTableDescriptor.convert(req.getTableSchema()); byte [][] splitKeys = ProtobufUtil.getSplitKeysArray(req); try { master.createTable(hTableDescriptor, splitKeys); } catch (IOException ioe) { throw new ServiceException(ioe); } return CreateTableResponse.newBuilder().build(); }
/** * Verify that PleaseHoldException gets retried. * HBASE-8764 * @throws IOException * @throws ZooKeeperConnectionException * @throws MasterNotRunningException * @throws ServiceException */ @Test public void testMasterMonitorCallableRetries() throws MasterNotRunningException, ZooKeeperConnectionException, IOException, ServiceException { Configuration configuration = HBaseConfiguration.create(); // Set the pause and retry count way down. configuration.setLong(HConstants.HBASE_CLIENT_PAUSE, 1); final int count = 10; configuration.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, count); // Get mocked connection. Getting the connection will register it so when HBaseAdmin is // constructed with same configuration, it will find this mocked connection. ClusterConnection connection = HConnectionTestingUtility.getMockedConnection(configuration); // Mock so we get back the master interface. Make it so when createTable is called, we throw // the PleaseHoldException. MasterKeepAliveConnection masterAdmin = Mockito.mock(MasterKeepAliveConnection.class); Mockito.when(masterAdmin.createTable((RpcController)Mockito.any(), (CreateTableRequest)Mockito.any())). thenThrow(new ServiceException("Test fail").initCause(new PleaseHoldException("test"))); Mockito.when(connection.getKeepAliveMasterService()).thenReturn(masterAdmin); Admin admin = new HBaseAdmin(connection); try { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testMasterMonitorCollableRetries")); // Pass any old htable descriptor; not important try { admin.createTable(htd, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); fail(); } catch (RetriesExhaustedException e) { Log.info("Expected fail", e); } // Assert we were called 'count' times. Mockito.verify(masterAdmin, Mockito.atLeast(count)).createTable((RpcController)Mockito.any(), (CreateTableRequest)Mockito.any()); } finally { admin.close(); if (connection != null) connection.close(); } }
/** * Creates a protocol buffer CreateTableRequest * * @param hTableDesc * @param splitKeys * @return a CreateTableRequest */ public static CreateTableRequest buildCreateTableRequest( final HTableDescriptor hTableDesc, final byte [][] splitKeys) { CreateTableRequest.Builder builder = CreateTableRequest.newBuilder(); builder.setTableSchema(hTableDesc.convert()); if (splitKeys != null) { for (byte [] splitKey : splitKeys) { builder.addSplitKeys(ByteStringer.wrap(splitKey)); } } return builder.build(); }
/** * Creates a new table but does not block and wait for it to come online. * Asynchronous operation. To check if the table exists, use * {@link #isTableAvailable} -- it is not safe to create an HTable * instance to this table before it is available. * Note : Avoid passing empty split key. * @param desc table descriptor for table * * @throws IllegalArgumentException Bad table name, if the split keys * are repeated and if the split key has empty byte array. * @throws MasterNotRunningException if master is not running * @throws org.apache.hadoop.hbase.TableExistsException if table already exists (If concurrent * threads, the table may have been created between test-for-existence * and attempt-at-creation). * @throws IOException */ @Override public void createTableAsync( final HTableDescriptor desc, final byte [][] splitKeys) throws IOException { if(desc.getTableName() == null) { throw new IllegalArgumentException("TableName cannot be null"); } if(splitKeys != null && splitKeys.length > 0) { Arrays.sort(splitKeys, Bytes.BYTES_COMPARATOR); // Verify there are no duplicate split keys byte [] lastKey = null; for(byte [] splitKey : splitKeys) { if (Bytes.compareTo(splitKey, HConstants.EMPTY_BYTE_ARRAY) == 0) { throw new IllegalArgumentException( "Empty split key must not be passed in the split keys."); } if(lastKey != null && Bytes.equals(splitKey, lastKey)) { throw new IllegalArgumentException("All split keys must be unique, " + "found duplicate: " + Bytes.toStringBinary(splitKey) + ", " + Bytes.toStringBinary(lastKey)); } lastKey = splitKey; } } executeCallable(new MasterCallable<Void>(getConnection()) { @Override public Void call(int callTimeout) throws ServiceException { CreateTableRequest request = RequestConverter.buildCreateTableRequest(desc, splitKeys); master.createTable(null, request); return null; } }); }
@Override public CreateTableResponse createTable(RpcController controller, CreateTableRequest req) throws ServiceException { HTableDescriptor hTableDescriptor = HTableDescriptor.convert(req.getTableSchema()); byte [][] splitKeys = ProtobufUtil.getSplitKeysArray(req); try { createTable(hTableDescriptor,splitKeys); } catch (IOException ioe) { throw new ServiceException(ioe); } return CreateTableResponse.newBuilder().build(); }
/** * Verify that PleaseHoldException gets retried. * HBASE-8764 * @throws IOException * @throws ZooKeeperConnectionException * @throws MasterNotRunningException * @throws ServiceException */ @Test public void testMasterMonitorCollableRetries() throws MasterNotRunningException, ZooKeeperConnectionException, IOException, ServiceException { Configuration configuration = HBaseConfiguration.create(); // Set the pause and retry count way down. configuration.setLong(HConstants.HBASE_CLIENT_PAUSE, 1); final int count = 10; configuration.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, count); // Get mocked connection. Getting the connection will register it so when HBaseAdmin is // constructed with same configuration, it will find this mocked connection. HConnection connection = HConnectionTestingUtility.getMockedConnection(configuration); // Mock so we get back the master interface. Make it so when createTable is called, we throw // the PleaseHoldException. MasterKeepAliveConnection masterAdmin = Mockito.mock(MasterKeepAliveConnection.class); Mockito.when(masterAdmin.createTable((RpcController)Mockito.any(), (CreateTableRequest)Mockito.any())). thenThrow(new ServiceException("Test fail").initCause(new PleaseHoldException("test"))); Mockito.when(connection.getKeepAliveMasterService()).thenReturn(masterAdmin); // Mock up our admin Interfaces HBaseAdmin admin = new HBaseAdmin(configuration); try { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testMasterMonitorCollableRetries")); // Pass any old htable descriptor; not important try { admin.createTable(htd, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); fail(); } catch (RetriesExhaustedException e) { Log.info("Expected fail", e); } // Assert we were called 'count' times. Mockito.verify(masterAdmin, Mockito.atLeast(count)).createTable((RpcController)Mockito.any(), (CreateTableRequest)Mockito.any()); } finally { admin.close(); if (connection != null)HConnectionManager.deleteConnection(configuration); } }
/** * Creates a protocol buffer CreateTableRequest * * @param hTableDesc * @param splitKeys * @return a CreateTableRequest */ public static CreateTableRequest buildCreateTableRequest( final HTableDescriptor hTableDesc, final byte [][] splitKeys) { CreateTableRequest.Builder builder = CreateTableRequest.newBuilder(); builder.setTableSchema(hTableDesc.convert()); if (splitKeys != null) { for (byte [] splitKey : splitKeys) { builder.addSplitKeys(HBaseZeroCopyByteString.wrap(splitKey)); } } return builder.build(); }
/** * Creates a new table but does not block and wait for it to come online. * Asynchronous operation. To check if the table exists, use * {@link #isTableAvailable} -- it is not safe to create an HTable * instance to this table before it is available. * Note : Avoid passing empty split key. * @param desc table descriptor for table * * @throws IllegalArgumentException Bad table name, if the split keys * are repeated and if the split key has empty byte array. * @throws MasterNotRunningException if master is not running * @throws org.apache.hadoop.hbase.TableExistsException if table already exists (If concurrent * threads, the table may have been created between test-for-existence * and attempt-at-creation). * @throws IOException */ public void createTableAsync( final HTableDescriptor desc, final byte [][] splitKeys) throws IOException { if(desc.getTableName() == null) { throw new IllegalArgumentException("TableName cannot be null"); } if(splitKeys != null && splitKeys.length > 0) { Arrays.sort(splitKeys, Bytes.BYTES_COMPARATOR); // Verify there are no duplicate split keys byte [] lastKey = null; for(byte [] splitKey : splitKeys) { if (Bytes.compareTo(splitKey, HConstants.EMPTY_BYTE_ARRAY) == 0) { throw new IllegalArgumentException( "Empty split key must not be passed in the split keys."); } if(lastKey != null && Bytes.equals(splitKey, lastKey)) { throw new IllegalArgumentException("All split keys must be unique, " + "found duplicate: " + Bytes.toStringBinary(splitKey) + ", " + Bytes.toStringBinary(lastKey)); } lastKey = splitKey; } } executeCallable(new MasterCallable<Void>(getConnection()) { @Override public Void call() throws ServiceException { CreateTableRequest request = RequestConverter.buildCreateTableRequest(desc, splitKeys); master.createTable(null, request); return null; } }); }
@Override public CreateTableResponse createTable(RpcController controller, CreateTableRequest req) throws ServiceException { HTableDescriptor hTableDescriptor = HTableDescriptor.convert(req.getTableSchema()); byte [][] splitKeys = ProtobufUtil.getSplitKeysArray(req); // Shen Li: add parameter replicaNum int replicaNum = req.getReplicaNum(); try { master.createTable(hTableDescriptor, splitKeys, replicaNum); } catch (IOException ioe) { throw new ServiceException(ioe); } return CreateTableResponse.newBuilder().build(); }
/** * Shen Li: add parameter replicaNum */ public void createTableAsync( final HTableDescriptor desc, final byte [][] splitKeys, final int replicaNum) throws IOException { if(desc.getTableName() == null) { throw new IllegalArgumentException("TableName cannot be null"); } if(splitKeys != null && splitKeys.length > 0) { Arrays.sort(splitKeys, Bytes.BYTES_COMPARATOR); // Verify there are no duplicate split keys byte [] lastKey = null; for(byte [] splitKey : splitKeys) { if (Bytes.compareTo(splitKey, HConstants.EMPTY_BYTE_ARRAY) == 0) { throw new IllegalArgumentException( "Empty split key must not be passed in the split keys."); } if(lastKey != null && Bytes.equals(splitKey, lastKey)) { throw new IllegalArgumentException("All split keys must be unique, " + "found duplicate: " + Bytes.toStringBinary(splitKey) + ", " + Bytes.toStringBinary(lastKey)); } lastKey = splitKey; } } // Shen Li: executeCallable(new MasterCallable<Void>(getConnection()) { @Override public Void call(int callTimeout) throws ServiceException { CreateTableRequest request = RequestConverter.buildCreateTableRequest(desc, splitKeys, replicaNum); master.createTable(null, request); return null; } }); }
/** * Creates a protocol buffer CreateTableRequest * * @param hTableDesc * @param splitKeys * @return a CreateTableRequest */ public static CreateTableRequest buildCreateTableRequest( final HTableDescriptor hTableDesc, final byte [][] splitKeys) { CreateTableRequest.Builder builder = CreateTableRequest.newBuilder(); builder.setTableSchema(hTableDesc.convert()); if (splitKeys != null) { for (byte [] splitKey : splitKeys) { builder.addSplitKeys(ZeroCopyLiteralByteString.wrap(splitKey)); } } return builder.build(); }
/** * Verify that PleaseHoldException gets retried. * HBASE-8764 * @throws IOException * @throws ZooKeeperConnectionException * @throws MasterNotRunningException * @throws ServiceException */ //TODO: Clean up, with Procedure V2 and nonce to prevent the same procedure to call mulitple // time, this test is invalid anymore. Just keep the test around for some time before // fully removing it. @Ignore @Test public void testMasterMonitorCallableRetries() throws MasterNotRunningException, ZooKeeperConnectionException, IOException, ServiceException { Configuration configuration = HBaseConfiguration.create(); // Set the pause and retry count way down. configuration.setLong(HConstants.HBASE_CLIENT_PAUSE, 1); final int count = 10; configuration.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, count); // Get mocked connection. Getting the connection will register it so when HBaseAdmin is // constructed with same configuration, it will find this mocked connection. ClusterConnection connection = HConnectionTestingUtility.getMockedConnection(configuration); // Mock so we get back the master interface. Make it so when createTable is called, we throw // the PleaseHoldException. MasterKeepAliveConnection masterAdmin = Mockito.mock(MasterKeepAliveConnection.class); Mockito.when(masterAdmin.createTable((RpcController)Mockito.any(), (CreateTableRequest)Mockito.any())). thenThrow(new ServiceException("Test fail").initCause(new PleaseHoldException("test"))); Mockito.when(connection.getKeepAliveMasterService()).thenReturn(masterAdmin); Admin admin = new HBaseAdmin(connection); try { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testMasterMonitorCollableRetries")); // Pass any old htable descriptor; not important try { admin.createTable(htd, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); fail(); } catch (RetriesExhaustedException e) { LOG.info("Expected fail", e); } // Assert we were called 'count' times. Mockito.verify(masterAdmin, Mockito.atLeast(count)).createTable((RpcController)Mockito.any(), (CreateTableRequest)Mockito.any()); } finally { admin.close(); if (connection != null) connection.close(); } }
/** * Shen Li: redirect * * Creates a protocol buffer CreateTableRequest * * @param hTableDesc * @param splitKeys * @return a CreateTableRequest */ public static CreateTableRequest buildCreateTableRequest( final HTableDescriptor hTableDesc, final byte [][] splitKeys) { return buildCreateTableRequest(hTableDesc, splitKeys, -1); }
