@RequestMapping("/describe/{table}") public @ResponseBody TableDescription definition(@PathVariable String table) { TableDescription description = null; try { if (!names.isEmpty()) { description = amazonDynamoDBClient.describeTable(table).getTable(); } } catch (AmazonClientException ex) { LOGGER.error(ex.getMessage()); } return description; }
@Test public void testCreateTableWithWait() throws Exception { // Create fake responses from AWS. First response is still creating the table, second response the table // has become active. TableDescription creatingDescription = constructTableDescription(TableStatus.CREATING); TableDescription createdDescription = constructTableDescription(TableStatus.ACTIVE); CreateTableResult mockCreateResult = new CreateTableResult().withTableDescription(creatingDescription); DescribeTableResult mockDescribeResultCreating = new DescribeTableResult().withTable(creatingDescription); DescribeTableResult mockDescribeResultCreated = new DescribeTableResult().withTable(createdDescription); // Create the table. CreateTableRequest expectedRequest = dynamoDB.constructCreateTableRequest(); when(mockDynamoDBClient.createTable(expectedRequest)).thenReturn(mockCreateResult); when(mockDynamoDBClient.describeTable(tableName)).thenReturn(mockDescribeResultCreating, mockDescribeResultCreated); assertEquals(dynamoDB.create(), TEST_ARN); verify(mockDynamoDBClient, times(1)).createTable(expectedRequest); verify(mockDynamoDBClient, times(2)).describeTable(tableName); }
private static Task<TableDescription> waitForActiveTableStatus(final DynamoDBConnection dynamoDBConnection, final String tableName) { try { for (int i = 0; i < WAITING_FOR_ACTIVE_TABLE_STATUS_MAX_ATTEMPTS; i++) { final DescribeTableResult describe = dynamoDBConnection.getDynamoClient().describeTable(tableName); if (describe.getTable().getTableStatus().equals(TableStatus.ACTIVE.name())) { return Task.fromValue(describe.getTable()); } Thread.sleep(WAITING_FOR_ACTIVE_TABLE_STATUS_RETRY_DELAY_MILLIS); } } catch (InterruptedException e) { throw new UncheckedException(e); } throw new UncheckedException("Hit max retry attempts while waiting for table to become active: " + tableName); }
private void setTableProperties(JobConf jobConf, String tableName, Double writeRatio) { jobConf.set(DynamoDBConstants.OUTPUT_TABLE_NAME, tableName); jobConf.set(DynamoDBConstants.INPUT_TABLE_NAME, tableName); jobConf.set(DynamoDBConstants.TABLE_NAME, tableName); DynamoDBClient client = new DynamoDBClient(jobConf); TableDescription description = client.describeTable(tableName); Long readThroughput = description.getProvisionedThroughput().getReadCapacityUnits(); Long writeThroughput = description.getProvisionedThroughput().getWriteCapacityUnits(); jobConf.set(DynamoDBConstants.READ_THROUGHPUT, readThroughput.toString()); jobConf.set(DynamoDBConstants.WRITE_THROUGHPUT, writeThroughput.toString()); log.info("Read throughput: " + readThroughput); log.info("Write throughput: " + writeThroughput); // Optional properties if (writeRatio != null) { jobConf.set(DynamoDBConstants.THROUGHPUT_WRITE_PERCENT, writeRatio.toString()); log.info("Throughput write ratio: " + writeRatio); } }
@Override public void preCreateTable(Table table) throws MetaException { DynamoDBClient client = createDynamoDBClient(table); try { boolean isExternal = MetaStoreUtils.isExternalTable(table); if (!isExternal) { throw new MetaException("Only EXTERNAL tables are supported for DynamoDB."); } String tableName = HiveDynamoDBUtil.getDynamoDBTableName(table.getParameters() .get(DynamoDBConstants.TABLE_NAME), table.getTableName()); TableDescription tableDescription = client.describeTable(tableName); checkTableStatus(tableDescription); checkTableSchemaMapping(tableDescription, table); checkTableSchemaType(tableDescription, table); } finally { client.close(); } }
void checkTableSchemaMapping(TableDescription tableDescription, Table table) throws MetaException { String mapping = table.getParameters().get(DynamoDBConstants.DYNAMODB_COLUMN_MAPPING); Map<String, String> columnMapping = HiveDynamoDBUtil.getHiveToDynamoDBSchemaMapping(mapping); List<FieldSchema> tableSchema = table.getSd().getCols(); for (FieldSchema fieldSchema : tableSchema) { if (HiveDynamoDBTypeFactory.isHiveDynamoDBItemMapType(fieldSchema.getType())) { // We don't need column mapping as this column contains full // DynamoDB row continue; } String fieldSchemaName = fieldSchema.getName().toLowerCase(); if (columnMapping.containsKey(fieldSchemaName)) { if (columnMapping.get(fieldSchemaName).isEmpty()) { throw new MetaException("Invalid column mapping for column: " + fieldSchemaName); } } else { throw new MetaException("Could not find column mapping for column: " + fieldSchemaName); } } }
void checkTableSchemaType(TableDescription tableDescription, Table table) throws MetaException { List<FieldSchema> tableSchema = table.getSd().getCols(); for (FieldSchema fieldSchema : tableSchema) { for (AttributeDefinition definition : tableDescription.getAttributeDefinitions()) { validateKeySchema(definition.getAttributeName(), definition.getAttributeType(), fieldSchema); } // Check for each field type if (HiveDynamoDBTypeFactory.getTypeObjectFromHiveType(fieldSchema.getType()) == null) { throw new MetaException("The hive type " + fieldSchema.getType() + " is not supported in " + "DynamoDB"); } } }
@Test public void testCheckTableSchemaMappingMissingColumn() throws MetaException { TableDescription description = getHashRangeTable(); Table table = new Table(); Map<String, String> parameters = Maps.newHashMap(); parameters.put(DynamoDBConstants.DYNAMODB_COLUMN_MAPPING, "col1:dynamo_col1$,hashKey:hashKey"); table.setParameters(parameters); StorageDescriptor sd = new StorageDescriptor(); List<FieldSchema> cols = Lists.newArrayList(); cols.add(new FieldSchema("col1", "string", "")); cols.add(new FieldSchema("col2", "tinyint", "")); cols.add(new FieldSchema("col3", "map<string,string>", "")); cols.add(new FieldSchema("hashMap", "string", "")); sd.setCols(cols); table.setSd(sd); exceptionRule.expect(MetaException.class); exceptionRule.expectMessage("Could not find column mapping for column: col2"); storageHandler.checkTableSchemaMapping(description, table); }
@Test public void testCheckTableSchemaMappingValid() throws MetaException { TableDescription description = getHashRangeTable(); Table table = new Table(); Map<String, String> parameters = Maps.newHashMap(); parameters.put(DynamoDBConstants.DYNAMODB_COLUMN_MAPPING, "col1:dynamo_col1$," + "col2:dynamo_col2#,hashKey:hashKey"); table.setParameters(parameters); StorageDescriptor sd = new StorageDescriptor(); List<FieldSchema> cols = Lists.newArrayList(); cols.add(new FieldSchema("col1", "string", "")); cols.add(new FieldSchema("col2", "bigint", "")); cols.add(new FieldSchema("hashKey", "string", "")); sd.setCols(cols); table.setSd(sd); storageHandler.checkTableSchemaMapping(description, table); }
@Test public void testCheckTableSchemaTypeInvalidType() throws MetaException { TableDescription description = getHashRangeTable(); Table table = new Table(); Map<String, String> parameters = Maps.newHashMap(); parameters.put(DynamoDBConstants.DYNAMODB_COLUMN_MAPPING, "col1:dynamo_col1$," + "col2:dynamo_col2#,hashKey:hashKey"); table.setParameters(parameters); StorageDescriptor sd = new StorageDescriptor(); List<FieldSchema> cols = Lists.newArrayList(); cols.add(new FieldSchema("col1", "string", "")); cols.add(new FieldSchema("col2", "tinyint", "")); cols.add(new FieldSchema("hashKey", "string", "")); sd.setCols(cols); table.setSd(sd); exceptionRule.expect(MetaException.class); exceptionRule.expectMessage("The hive type tinyint is not supported in DynamoDB"); storageHandler.checkTableSchemaType(description, table); }
@Test public void testCheckTableSchemaTypeInvalidHashKeyType() throws MetaException { TableDescription description = getHashRangeTable(); Table table = new Table(); Map<String, String> parameters = Maps.newHashMap(); parameters.put(DynamoDBConstants.DYNAMODB_COLUMN_MAPPING, "col1:dynamo_col1$," + "col2:dynamo_col2#,hashKey:hashKey"); table.setParameters(parameters); StorageDescriptor sd = new StorageDescriptor(); List<FieldSchema> cols = Lists.newArrayList(); cols.add(new FieldSchema("col1", "string", "")); cols.add(new FieldSchema("col2", "bigint", "")); cols.add(new FieldSchema("hashKey", "map<string,string>", "")); sd.setCols(cols); table.setSd(sd); exceptionRule.expect(MetaException.class); exceptionRule.expectMessage("The key element hashKey does not match type. DynamoDB Type: S " + "Hive type: " + "map<string,string>"); storageHandler.checkTableSchemaType(description, table); }
@Test public void testCheckTableSchemaTypeValid() throws MetaException { TableDescription description = getHashRangeTable(); Table table = new Table(); Map<String, String> parameters = Maps.newHashMap(); parameters.put(DynamoDBConstants.DYNAMODB_COLUMN_MAPPING, "col1:dynamo_col1$," + "col2:dynamo_col2#,hashKey:hashKey"); table.setParameters(parameters); StorageDescriptor sd = new StorageDescriptor(); List<FieldSchema> cols = Lists.newArrayList(); cols.add(new FieldSchema("col1", "string", "")); cols.add(new FieldSchema("col2", "bigint", "")); cols.add(new FieldSchema("hashKey", "string", "")); sd.setCols(cols); table.setSd(sd); // This check is expected to pass for the given input storageHandler.checkTableSchemaType(description, table); }
public TableDescription describeTable(String tableName) { final DescribeTableRequest describeTablesRequest = new DescribeTableRequest() .withTableName(tableName); try { RetryResult<DescribeTableResult> describeResult = getRetryDriver().runWithRetry( new Callable<DescribeTableResult>() { @Override public DescribeTableResult call() { DescribeTableResult result = dynamoDB.describeTable(describeTablesRequest); log.info("Describe table output: " + result); return result; } }, null, null); return describeResult.result.getTable(); } catch (Exception e) { throw new RuntimeException("Could not lookup table " + tableName + " in DynamoDB.", e); } }
@Before public void setup() { when(dynamoDBClient.describeTable(TABLE_NAME)).thenReturn(new TableDescription() .withProvisionedThroughput(new ProvisionedThroughputDescription().withWriteCapacityUnits (WRITE_CAPACITY_UNITS))); JobConf jobConf = new JobConf(); jobConf.setNumMapTasks(TOTAL_MAP_TASKS); jobConf.set("mapreduce.task.attempt.id", "attempt_m_1"); jobConf.set(DynamoDBConstants.THROUGHPUT_WRITE_PERCENT, String.valueOf (THROUGHPUT_WRITE_PERCENT)); when(jobClient.getConf()).thenReturn(jobConf); writeIopsCalculator = new WriteIopsCalculator(jobClient, dynamoDBClient, TABLE_NAME) { @Override int calculateMaxMapTasks(int totalMapTasks) { return MAX_CONCURRENT_MAP_TASKS; } }; }
private TableDescription getTableDescription(String hashType, String rangeType) { List<KeySchemaElement> keySchema = new ArrayList<>(); List<AttributeDefinition> definitions = new ArrayList<>(); keySchema.add(new KeySchemaElement().withAttributeName("hashKey").withKeyType(KeyType.HASH)); definitions.add(new AttributeDefinition().withAttributeName("hashKey").withAttributeType (hashType)); if (rangeType != null) { keySchema.add(new KeySchemaElement().withAttributeName("rangeKey").withKeyType(KeyType .RANGE)); definitions.add(new AttributeDefinition().withAttributeName("rangeKey").withAttributeType (rangeType)); } TableDescription description = new TableDescription().withKeySchema(keySchema) .withAttributeDefinitions(definitions).withProvisionedThroughput(new ProvisionedThroughputDescription().withReadCapacityUnits(1000L) .withWriteCapacityUnits(1000L)); return description; }
@Override public DescribeTableResult describeTable(DescribeTableRequest describeTableRequest) { this.describeTableRequest = describeTableRequest; String tableName = describeTableRequest.getTableName(); if ("activeTable".equals(tableName)) { return tableWithStatus(TableStatus.ACTIVE); } else if ("creatibleTable".equals(tableName) && createTableRequest != null) { return tableWithStatus(TableStatus.ACTIVE); } else if ("FULL_DESCRIBE_TABLE".equals(tableName)) { return new DescribeTableResult().withTable(new TableDescription() .withTableName(tableName) .withTableStatus(TableStatus.ACTIVE) .withCreationDateTime(new Date(NOW)) .withItemCount(100L) .withKeySchema(new KeySchemaElement().withAttributeName("name")) .withProvisionedThroughput(new ProvisionedThroughputDescription() .withReadCapacityUnits(20L) .withWriteCapacityUnits(10L)) .withTableSizeBytes(1000L)); } throw new ResourceNotFoundException(tableName + " is missing"); }
private String scaleTable(String tableName, Long readCapacity, Long writeCapacity) { Table table = dynamoDB.getTable(tableName); ProvisionedThroughput tp = new ProvisionedThroughput(); tp.setReadCapacityUnits(readCapacity); tp.setWriteCapacityUnits(writeCapacity); TableDescription d = table.describe(); if (!Objects.equals(d.getProvisionedThroughput().getReadCapacityUnits(), readCapacity) || !Objects.equals(d.getProvisionedThroughput().getWriteCapacityUnits(), writeCapacity)) { d = table.updateTable(tp); return tableName + "\nRequested read/write : " + readCapacity + "/" + writeCapacity + "\nCurrent read/write :" + d.getProvisionedThroughput().getReadCapacityUnits() + "/" + d.getProvisionedThroughput().getWriteCapacityUnits() + "\nStatus : " + d.getTableStatus() + "\n"; } else { return tableName + "\n Requested throughput equals current throughput\n"; } }
private void waitForStatus(String tableName, TableStatus status) { logger.info("Waiting for " + tableName + " to become " + status.toString() + "..."); long startTime = System.currentTimeMillis(); long endTime = startTime + (10 * 60 * 1000); while (System.currentTimeMillis() < endTime) { try { Thread.sleep(1000 * 2); } catch (Exception e) { } try { DescribeTableRequest request = new DescribeTableRequest().withTableName(tableName); TableDescription tableDescription = dynamoDb.describeTable(request).getTable(); String tableStatus = tableDescription.getTableStatus(); logger.debug(" - current state: " + tableStatus); if (tableStatus.equals(status.toString())) return; } catch (AmazonServiceException ase) { if (ase.getErrorCode().equalsIgnoreCase("ResourceNotFoundException") == false) throw ase; } } throw new RuntimeException("Table " + tableName + " never went " + status.toString()); }
void createTableAndWaitForActive(final CreateTableRequest request) throws BackendException { final String tableName = request.getTableName(); Preconditions.checkArgument(!Strings.isNullOrEmpty(tableName), "Table name was null or empty"); final TableDescription desc; try { desc = this.describeTable(tableName); if (null != desc && isTableAcceptingWrites(desc.getTableStatus())) { return; //store existed } } catch (BackendNotFoundException e) { log.debug(tableName + " did not exist yet, creating it", e); } createTable(request); waitForTableCreation(tableName, false /*verifyIndexesList*/, null /*expectedLsiList*/, null /*expectedGsiList*/); }
/** * Creates the DynamoDBBootstrapWorker, calculates the number of segments a * table should have, and creates a thread pool to prepare to scan using an * eventually consistent scan. * * @throws Exception */ public DynamoDBBootstrapWorker(AmazonDynamoDBClient client, double rateLimit, String tableName, int numThreads) throws NullReadCapacityException { this.client = client; this.rateLimit = rateLimit; this.tableName = tableName; TableDescription description = client.describeTable(tableName) .getTable(); this.section = 0; this.totalSections = 1; this.consistentScan = false; this.numSegments = getNumberOfSegments(description); int numProcessors = Runtime.getRuntime().availableProcessors() * 4; if (numProcessors > numThreads) { numThreads = numProcessors; } super.threadPool = Executors.newFixedThreadPool(numThreads); }
/** * returns the approximate number of segments a table should be broken up * when parallel scanning. This function is based off of either read and * write capacity, with which you can scan much faster, or the size of your * table, which should need many more segments in order to scan the table * fast enough in parallel so that one worker does not finish long before * other workers. * * @throws NullReadCapacityException * if the table returns a null readCapacity units. */ public static int getNumberOfSegments(TableDescription description) throws NullReadCapacityException { ProvisionedThroughputDescription provisionedThroughput = description .getProvisionedThroughput(); double tableSizeInGigabytes = Math.ceil(description.getTableSizeBytes() / BootstrapConstants.GIGABYTE); Long readCapacity = provisionedThroughput.getReadCapacityUnits(); Long writeCapacity = provisionedThroughput.getWriteCapacityUnits(); if (writeCapacity == null) { writeCapacity = 1L; } if (readCapacity == null) { throw new NullReadCapacityException( "Cannot scan with a null readCapacity provisioned throughput"); } double throughput = (readCapacity + 3 * writeCapacity) / 3000.0; return (int) (10 * Math.max(Math.ceil(throughput), Math.ceil(tableSizeInGigabytes) / 10)); }
private void waitForTableToBecomeAvailable(String tableName) { System.out.println("Waiting for " + tableName + " to become ACTIVE..."); long startTime = System.currentTimeMillis(); long endTime = startTime + (10 * 60 * 1000); while (System.currentTimeMillis() < endTime) { DescribeTableRequest request = new DescribeTableRequest() .withTableName(tableName); TableDescription tableDescription = client.describeTable( request).getTable(); String tableStatus = tableDescription.getTableStatus(); System.out.println(" - current state: " + tableStatus); if (tableStatus.equals(TableStatus.ACTIVE.toString())) return; try { Thread.sleep(1000 * 20); } catch (Exception e) { } } throw new RuntimeException("Table " + tableName + " never went active"); }
private void waitForTableAvailable(String tableName) { LOG.info("Waiting for table " + tableName + " to become ACTIVE..."); long startTime = System.currentTimeMillis(); long endTime = startTime + (10 * 60 * 1000); while (System.currentTimeMillis() < endTime) { DescribeTableRequest describeTableRequest = new DescribeTableRequest().withTableName(tableName); TableDescription tableDescription = dynamoDBClient.describeTable(describeTableRequest).getTable(); // Display current status of table String tableStatus = tableDescription.getTableStatus(); LOG.info("Current state for table " + tableName + ": " + tableStatus); if (tableStatus.equals(TableStatus.ACTIVE.toString())) { return; } try { Thread.sleep(1000 * 20); } catch (Exception ex) { LOG.warn(ex.getMessage()); } } throw new RuntimeException("Table " + tableName + " never went active"); }
/** * Gives basic information about a DynamoDB table (status, creation date, size). * @param appid name of the {@link com.erudika.para.core.App} * @return a map */ public static Map<String, Object> getTableStatus(final String appid) { if (StringUtils.isBlank(appid)) { return Collections.emptyMap(); } try { final TableDescription td = getClient().describeTable(getTableNameForAppid(appid)).getTable(); HashMap<String, Object> dbStatus = new HashMap<>(); dbStatus.put("id", appid); dbStatus.put("status", td.getTableStatus()); dbStatus.put("created", td.getCreationDateTime().getTime()); dbStatus.put("sizeBytes", td.getTableSizeBytes()); dbStatus.put("itemCount", td.getItemCount()); dbStatus.put("readCapacityUnits", td.getProvisionedThroughput().getReadCapacityUnits()); dbStatus.put("writeCapacityUnits", td.getProvisionedThroughput().getWriteCapacityUnits()); return dbStatus; } catch (Exception e) { logger.error(null, e); } return Collections.emptyMap(); }
/*** * Scan classes to determine which tables and indexes need to be created * * TODO - DynamoDB has a limit of how many tables can be created at once, I think 10 as of now. * This method does not batch but really needs to, so it only tries to create up 10 tables at the same time * * @param packagePrefix * @param blockUntilActive - If true this method will not return until the table is active or maxBlockTimeSeconds has expired * @param maxBlockTimeSeconds - The maximum amount of time to block for each table until the table becomes active */ public void scan(final String packagePrefix, boolean blockUntilActive, long maxBlockTimeSeconds){ final List<String> createdTables = Lists.newArrayList(); final Reflections reflections = new Reflections(packagePrefix); final Set<Class<?>> tableClasses = reflections.getTypesAnnotatedWith(DynamoDBTable.class); for(Class<?> clazz : tableClasses){ if(!tableExists(clazz)){ final CreateTableResult result = this.createTable(clazz); if(result!=null && result.getTableDescription()!=null){ final TableDescription description = result.getTableDescription(); /** If the table is not active add it to the list of tables to wait on **/ if(!ACTIVE_TABLE_STATUS.equalsIgnoreCase(description.getTableStatus())){ createdTables.add(description.getTableName()); } } } } /** If specified, wait for all the tables to become if active **/ if(blockUntilActive){ for(final String table : createdTables){ this.waitForTableToBecomeActive(table, maxBlockTimeSeconds, DEFAULT_PAUSE_TIME_SECONDS); } } }
/** * Verifies if the specified schemas exist * * @return */ @Override public boolean schemaExists() { LOG.info("Verifying schemas."); TableDescription success = null; if (getDynamoDbMapping().getTables().isEmpty()) throw new IllegalStateException("There are not tables defined."); if (getPreferredSchema() == null) { LOG.debug("Verifying schemas"); if (getDynamoDbMapping().getTables().isEmpty()) throw new IllegalStateException("There are not tables defined."); // read the mapping object for (String tableName : getDynamoDbMapping().getTables().keySet()) { success = getTableSchema(tableName); if (success == null) return false; } } else { LOG.info("Verifying schema " + preferredSchema); success = getTableSchema(preferredSchema); } LOG.info("Finished verifying schemas."); return (success != null) ? true : false; }
/** * Waits up to 6 minutes to confirm if a table has been deleted or not * * @param pTableName */ private void waitForTableToBeDeleted(String pTableName) { LOG.debug("Waiting for " + pTableName + " to be deleted."); long startTime = System.currentTimeMillis(); long endTime = startTime + WAIT_TIME; while (System.currentTimeMillis() < endTime) { try { Thread.sleep(SLEEP_DELETE_TIME); } catch (Exception e) { } try { DescribeTableRequest request = new DescribeTableRequest() .withTableName(pTableName); TableDescription tableDescription = getDynamoDBClient().describeTable( request).getTable(); String tableStatus = tableDescription.getTableStatus(); LOG.debug(pTableName + " - current state: " + tableStatus); } catch (AmazonServiceException ase) { if (ase.getErrorCode().equalsIgnoreCase("ResourceNotFoundException") == true) return; LOG.error(ase.getMessage()); } } LOG.debug(pTableName + " deleted."); }
/** * Checks if a resource exists or not * * @param tableName * Table name to be checked * @return */ public TableDescription checkResource(String tableName){ TableDescription tableDescription = null; try{ DescribeTableRequest describeTableRequest = new DescribeTableRequest() .withTableName(tableName); tableDescription = dynamoDBClient.describeTable(describeTableRequest) .getTable(); } catch(ResourceNotFoundException e){ tableDescription = null; } return tableDescription; }
private static void waitForTableToBecomeAvailable(String tableName) { System.out.println("Waiting for " + tableName + " to become ACTIVE..."); long startTime = System.currentTimeMillis(); long endTime = startTime + (10 * 60 * 1000); while (System.currentTimeMillis() < endTime) { DescribeTableRequest request = new DescribeTableRequest() .withTableName(tableName); TableDescription tableDescription = client.describeTable( request).getTable(); String tableStatus = tableDescription.getTableStatus(); System.out.println(" - current state: " + tableStatus); if (tableStatus.equals(TableStatus.ACTIVE.toString())) return; try { Thread.sleep(1000 * 20); } catch (Exception e) { } } throw new RuntimeException("Table " + tableName + " never went active"); }
private static void waitForTableToBecomeAvailable(String tableName) { System.out.println("Waiting for " + tableName + " to become ACTIVE..."); long startTime = System.currentTimeMillis(); long endTime = startTime + (10 * 60 * 1000); while (System.currentTimeMillis() < endTime) { DescribeTableRequest request = new DescribeTableRequest().withTableName(tableName); TableDescription tableDescription = client.describeTable(request).getTable(); String tableStatus = tableDescription.getTableStatus(); System.out.println(" - current state: " + tableStatus); if (tableStatus.equals(TableStatus.ACTIVE.toString())) return; try { Thread.sleep(1000 * 20); } catch (Exception e) { e.printStackTrace(); } } throw new RuntimeException("Table " + tableName + " never went active"); }
@Test public void createTableTest() { AmazonDynamoDB ddb = DynamoDBEmbedded.create().amazonDynamoDB(); try { String tableName = "Movies"; String hashKeyName = "film_id"; CreateTableResult res = createTable(ddb, tableName, hashKeyName); TableDescription tableDesc = res.getTableDescription(); assertEquals(tableName, tableDesc.getTableName()); assertEquals("[{AttributeName: " + hashKeyName + ",KeyType: HASH}]", tableDesc.getKeySchema().toString()); assertEquals("[{AttributeName: " + hashKeyName + ",AttributeType: S}]", tableDesc.getAttributeDefinitions().toString()); assertEquals(Long.valueOf(1000L), tableDesc.getProvisionedThroughput().getReadCapacityUnits()); assertEquals(Long.valueOf(1000L), tableDesc.getProvisionedThroughput().getWriteCapacityUnits()); assertEquals("ACTIVE", tableDesc.getTableStatus()); assertEquals("arn:aws:dynamodb:ddblocal:000000000000:table/Movies", tableDesc.getTableArn()); ListTablesResult tables = ddb.listTables(); assertEquals(1, tables.getTableNames().size()); } finally { ddb.shutdown(); } }
private void waitForTable(String name) { log.info(String.format("Waiting for creation of table '%s' to complete.", name)); long startTime = System.currentTimeMillis(); long endTime = startTime + (10 * 60 * 1000); while (System.currentTimeMillis() < endTime) { sleep(1000 * 20); try { DescribeTableRequest request = new DescribeTableRequest().withTableName(name); TableDescription tableDescription = client.describeTable(request).getTable(); String tableStatus = tableDescription.getTableStatus(); log.info(String.format("Table '%s' is in state: '%s'.", name, tableStatus)); if (tableStatus.equals(TableStatus.ACTIVE.toString())) { return; } } catch (ResourceNotFoundException e) { // nop - maybe the table isn't showing up yet. } } throw new RuntimeException(String.format("Table '%s' never went active.", name)); }
/** * Create ticket tables. * * @param deleteTables the delete tables */ public void createTicketTables(final boolean deleteTables) { final Collection<TicketDefinition> metadata = this.ticketCatalog.findAll(); metadata.forEach(Unchecked.consumer(r -> { final CreateTableRequest request = new CreateTableRequest() .withAttributeDefinitions(new AttributeDefinition(ColumnNames.ID.getName(), ScalarAttributeType.S)) .withKeySchema(new KeySchemaElement(ColumnNames.ID.getName(), KeyType.HASH)) .withProvisionedThroughput(new ProvisionedThroughput(dynamoDbProperties.getReadCapacity(), dynamoDbProperties.getWriteCapacity())) .withTableName(r.getProperties().getStorageName()); if (deleteTables) { final DeleteTableRequest delete = new DeleteTableRequest(r.getProperties().getStorageName()); LOGGER.debug("Sending delete request [{}] to remove table if necessary", delete); TableUtils.deleteTableIfExists(amazonDynamoDBClient, delete); } LOGGER.debug("Sending delete request [{}] to create table", request); TableUtils.createTableIfNotExists(amazonDynamoDBClient, request); LOGGER.debug("Waiting until table [{}] becomes active...", request.getTableName()); TableUtils.waitUntilActive(amazonDynamoDBClient, request.getTableName()); final DescribeTableRequest describeTableRequest = new DescribeTableRequest().withTableName(request.getTableName()); LOGGER.debug("Sending request [{}] to obtain table description...", describeTableRequest); final TableDescription tableDescription = amazonDynamoDBClient.describeTable(describeTableRequest).getTable(); LOGGER.debug("Located newly created table with description: [{}]", tableDescription); })); }
/** * Create tables. * * @param deleteTables the delete tables */ public void createServicesTable(final boolean deleteTables) { try { final CreateTableRequest request = new CreateTableRequest() .withAttributeDefinitions(new AttributeDefinition(ColumnNames.ID.getName(), ScalarAttributeType.S)) .withKeySchema(new KeySchemaElement(ColumnNames.ID.getName(), KeyType.HASH)) .withProvisionedThroughput(new ProvisionedThroughput(dynamoDbProperties.getReadCapacity(), dynamoDbProperties.getWriteCapacity())) .withTableName(TABLE_NAME); if (deleteTables) { final DeleteTableRequest delete = new DeleteTableRequest(request.getTableName()); LOGGER.debug("Sending delete request [{}] to remove table if necessary", delete); TableUtils.deleteTableIfExists(amazonDynamoDBClient, delete); } LOGGER.debug("Sending delete request [{}] to create table", request); TableUtils.createTableIfNotExists(amazonDynamoDBClient, request); LOGGER.debug("Waiting until table [{}] becomes active...", request.getTableName()); TableUtils.waitUntilActive(amazonDynamoDBClient, request.getTableName()); final DescribeTableRequest describeTableRequest = new DescribeTableRequest().withTableName(request.getTableName()); LOGGER.debug("Sending request [{}] to obtain table description...", describeTableRequest); final TableDescription tableDescription = amazonDynamoDBClient.describeTable(describeTableRequest).getTable(); LOGGER.debug("Located newly created table with description: [{}]", tableDescription); } catch (final Exception e) { throw Throwables.propagate(e); } }
private static void createSettingsTable(final AmazonDynamoDBClient amazonDynamoDBClient, final boolean deleteTables) { try { final CreateTableRequest request = new CreateTableRequest() .withAttributeDefinitions(new AttributeDefinition(ColumnNames.ID.getName(), ScalarAttributeType.S)) .withKeySchema(new KeySchemaElement(ColumnNames.ID.getName(), KeyType.HASH)) .withProvisionedThroughput(new ProvisionedThroughput(PROVISIONED_THROUGHPUT, PROVISIONED_THROUGHPUT)) .withTableName(TABLE_NAME); if (deleteTables) { final DeleteTableRequest delete = new DeleteTableRequest(request.getTableName()); LOGGER.debug("Sending delete request [{}] to remove table if necessary", delete); TableUtils.deleteTableIfExists(amazonDynamoDBClient, delete); } LOGGER.debug("Sending delete request [{}] to create table", request); TableUtils.createTableIfNotExists(amazonDynamoDBClient, request); LOGGER.debug("Waiting until table [{}] becomes active...", request.getTableName()); TableUtils.waitUntilActive(amazonDynamoDBClient, request.getTableName()); final DescribeTableRequest describeTableRequest = new DescribeTableRequest().withTableName(request.getTableName()); LOGGER.debug("Sending request [{}] to obtain table description...", describeTableRequest); final TableDescription tableDescription = amazonDynamoDBClient.describeTable(describeTableRequest).getTable(); LOGGER.debug("Located newly created table with description: [{}]", tableDescription); } catch (final Exception e) { throw Throwables.propagate(e); } }
private void createAppsTable(String tableName) { final AttributeDefinition appKey = new AttributeDefinition().withAttributeName(HASH_KEY).withAttributeType( ScalarAttributeType.S); final ArrayList<AttributeDefinition> tableAttributeDefinitions = Lists.newArrayList(appKey); final ArrayList<KeySchemaElement> tableKeySchema = Lists.newArrayList(); tableKeySchema.add( new KeySchemaElement().withAttributeName(HASH_KEY).withKeyType(KeyType.HASH)); final ProvisionedThroughput tableProvisionedThroughput = new ProvisionedThroughput() .withReadCapacityUnits(10L) .withWriteCapacityUnits(10L); final CreateTableRequest createTableRequest = new CreateTableRequest() .withTableName(tableName) .withKeySchema(tableKeySchema) .withAttributeDefinitions(tableAttributeDefinitions) .withProvisionedThroughput(tableProvisionedThroughput); final TableDescription tableDescription = amazonDynamoDB.createTable(createTableRequest).getTableDescription(); logger.info("created_table {}", tableDescription); final DescribeTableRequest describeTableRequest = new DescribeTableRequest().withTableName(tableName); final TableDescription description = amazonDynamoDB.describeTable(describeTableRequest).getTable(); logger.info("table_description: " + description); }
private Optional<TableDescription> describeTable() { try { DescribeTableResult result = client.describeTable(tableName); return Optional.of(result.getTable()); } catch (ResourceNotFoundException e) { return Optional.empty(); } }
private static void cleanUpDynamoDBTables(Regions testRegion, String testResourcePrefix, Date createdBeforeThreshold, AWSCredentialsProvider awsCredentials) { LOG.info("Cleaning DynamoDB..."); AmazonDynamoDB dynamoDBClient = AmazonDynamoDBClientBuilder.standard() .withCredentials(awsCredentials) .withRegion(testRegion) .build(); List<String> tableNames = dynamoDBClient.listTables().getTableNames(); for (String tableName: tableNames) { if (!tableName.startsWith(testResourcePrefix)) { continue; } LOG.info(String.format("Checking if table %s needs cleaning...", tableName)); try { TableDescription desc = dynamoDBClient.describeTable(tableName).getTable(); if (!desc.getTableName().equals(TableStatus.DELETING.toString()) && desc.getCreationDateTime() != null && desc.getCreationDateTime().before(createdBeforeThreshold)) { LOG.info("Cleaning up table: " + tableName); dynamoDBClient.deleteTable(tableName); } } catch (ResourceNotFoundException e) { LOG.info("Looks like table was already cleaned up: " + tableName); } } }