@Override public void closeStorage() { try { dynamoDBConnection.getDynamoClient().describeTable(getTableName()); dynamoDBConnection.getDynamoClient().deleteTable(getTableName()); } catch(ResourceNotFoundException e) { } dynamoDBConnection.getDynamoDB().createTable(getTableName(), Collections.singletonList( new KeySchemaElement("_id", KeyType.HASH)), Collections.singletonList( new AttributeDefinition("_id", ScalarAttributeType.S)), new ProvisionedThroughput(1L, 1L)); }
public AbstractDynamoDBRecordWriter(JobConf jobConf, Progressable progressable) { this.progressable = progressable; client = new DynamoDBClient(jobConf); tableName = jobConf.get(DynamoDBConstants.OUTPUT_TABLE_NAME); if (tableName == null) { throw new ResourceNotFoundException("No output table name was specified."); } IopsCalculator iopsCalculator = new WriteIopsCalculator(createJobClient(jobConf), client, tableName); iopsController = new IopsController(iopsCalculator, DEFAULT_AVERAGE_ITEM_SIZE_IN_BYTES, DynamoDBOperationType.WRITE); permissibleWritesPerSecond = iopsController.getTargetItemsPerSecond(); log.info("Number of allocated item writes per second: " + permissibleWritesPerSecond); // Hive may not have a valid Reporter and pass in null progressable // TODO Check whether this would happen when excluding Hive if (progressable instanceof Reporter) { reporter = (Reporter) progressable; } }
@Override public DescribeTableResult describeTable(DescribeTableRequest describeTableRequest) { this.describeTableRequest = describeTableRequest; String tableName = describeTableRequest.getTableName(); if ("activeTable".equals(tableName)) { return tableWithStatus(TableStatus.ACTIVE); } else if ("creatibleTable".equals(tableName) && createTableRequest != null) { return tableWithStatus(TableStatus.ACTIVE); } else if ("FULL_DESCRIBE_TABLE".equals(tableName)) { return new DescribeTableResult().withTable(new TableDescription() .withTableName(tableName) .withTableStatus(TableStatus.ACTIVE) .withCreationDateTime(new Date(NOW)) .withItemCount(100L) .withKeySchema(new KeySchemaElement().withAttributeName("name")) .withProvisionedThroughput(new ProvisionedThroughputDescription() .withReadCapacityUnits(20L) .withWriteCapacityUnits(10L)) .withTableSizeBytes(1000L)); } throw new ResourceNotFoundException(tableName + " is missing"); }
/** * Private interface for describing tables which handles any instances of * Throttling of the API * * @param dynamoClient * @param dynamoTable * @return * @throws Exception */ public static DescribeTableResult safeDescribeTable( final AmazonDynamoDB dynamoClient, final String dynamoTable) throws Exception { DescribeTableResult res = null; final int tryMax = 10; int tries = 0; while (true) { try { res = dynamoClient.describeTable(dynamoTable); return res; } catch (ResourceNotFoundException e) { if (tries < tryMax) { // sleep for a short time as this is potentially an eventual // consistency issue with the table having been created ms // ago Thread.sleep(10); tries++; } else { throw e; } } } }
/** * Checks if a resource exists or not * * @param tableName * Table name to be checked * @return */ public TableDescription checkResource(String tableName){ TableDescription tableDescription = null; try{ DescribeTableRequest describeTableRequest = new DescribeTableRequest() .withTableName(tableName); tableDescription = dynamoDBClient.describeTable(describeTableRequest) .getTable(); } catch(ResourceNotFoundException e){ tableDescription = null; } return tableDescription; }
public void setupTable() { setupGeoDataManager(); GeoDataManagerConfiguration config = geoDataManager.getGeoDataManagerConfiguration(); DescribeTableRequest describeTableRequest = new DescribeTableRequest().withTableName(config.getTableName()); try { config.getDynamoDBClient().describeTable(describeTableRequest); if (status == Status.NOT_STARTED) { status = Status.READY; } } catch (ResourceNotFoundException e) { PhotoLocationsTable photoLocationsTable = new PhotoLocationsTable(); photoLocationsTable.start(); } }
private void waitForTable(String name) { log.info(String.format("Waiting for creation of table '%s' to complete.", name)); long startTime = System.currentTimeMillis(); long endTime = startTime + (10 * 60 * 1000); while (System.currentTimeMillis() < endTime) { sleep(1000 * 20); try { DescribeTableRequest request = new DescribeTableRequest().withTableName(name); TableDescription tableDescription = client.describeTable(request).getTable(); String tableStatus = tableDescription.getTableStatus(); log.info(String.format("Table '%s' is in state: '%s'.", name, tableStatus)); if (tableStatus.equals(TableStatus.ACTIVE.toString())) { return; } } catch (ResourceNotFoundException e) { // nop - maybe the table isn't showing up yet. } } throw new RuntimeException(String.format("Table '%s' never went active.", name)); }
public static String getTestTableStatus() { try { AmazonDynamoDBClient ddb = UserPreferenceDemoActivity.clientManager .ddb(); DescribeTableRequest request = new DescribeTableRequest() .withTableName(Constants.TEST_TABLE_NAME); DescribeTableResult result = ddb.describeTable(request); String status = result.getTable().getTableStatus(); return status == null ? "" : status; } catch (ResourceNotFoundException e) { } catch (AmazonServiceException ex) { UserPreferenceDemoActivity.clientManager .wipeCredentialsOnAuthError(ex); } return ""; }
public void setupTable() { setupGeoDataManager(); GeoDataManagerConfiguration config = geoDataManager.getGeoDataManagerConfiguration(); DescribeTableRequest describeTableRequest = new DescribeTableRequest().withTableName(config.getTableName()); try { config.getDynamoDBClient().describeTable(describeTableRequest); if (status == Status.NOT_STARTED) { status = Status.READY; } } catch (ResourceNotFoundException e) { SchoolDataLoader schoolDataLoader = new SchoolDataLoader(); schoolDataLoader.start(); } }
private Optional<TableDescription> describeTable() { try { DescribeTableResult result = client.describeTable(tableName); return Optional.of(result.getTable()); } catch (ResourceNotFoundException e) { return Optional.empty(); } }
private Store getCurrentStore(SecretsGroupIdentifier group, ReadWriteLock readWriteLock) { if (userConfig.getLocalFilePath(group).isPresent()) { // TODO: load encryptor once final KMSEncryptor kmsEncryptor = getEncryptor(group); return new File(userConfig.getLocalFilePath(group).get(), kmsEncryptor, new FileEncryptionContext(group), readWriteLock); } try { DynamoDB dynamoDB = DynamoDB.fromCredentials(awsCredentials, clientConfiguration, group, readWriteLock); return dynamoDB; } catch (ResourceNotFoundException e) { throw new DoesNotExistException("No storage backend found!", e); } }
public static boolean groupExists(SecretsGroupManager secretsGroupManager, SecretsGroupIdentifier identifier) { try { secretsGroupManager.info(identifier); return true; } catch (NoSuchElementException | ResourceNotFoundException | NoSuchEntityException e) { return false; } }
private static void cleanUpDynamoDBTables(Regions testRegion, String testResourcePrefix, Date createdBeforeThreshold, AWSCredentialsProvider awsCredentials) { LOG.info("Cleaning DynamoDB..."); AmazonDynamoDB dynamoDBClient = AmazonDynamoDBClientBuilder.standard() .withCredentials(awsCredentials) .withRegion(testRegion) .build(); List<String> tableNames = dynamoDBClient.listTables().getTableNames(); for (String tableName: tableNames) { if (!tableName.startsWith(testResourcePrefix)) { continue; } LOG.info(String.format("Checking if table %s needs cleaning...", tableName)); try { TableDescription desc = dynamoDBClient.describeTable(tableName).getTable(); if (!desc.getTableName().equals(TableStatus.DELETING.toString()) && desc.getCreationDateTime() != null && desc.getCreationDateTime().before(createdBeforeThreshold)) { LOG.info("Cleaning up table: " + tableName); dynamoDBClient.deleteTable(tableName); } } catch (ResourceNotFoundException e) { LOG.info("Looks like table was already cleaned up: " + tableName); } } }
@Test public void testDeleteTableWithWait() throws Exception { // Create fake responses from AWS. TableDescription deletingDescription = constructTableDescription(TableStatus.DELETING); DescribeTableResult mockDescribeResult = new DescribeTableResult().withTable(deletingDescription); // Delete the table. First response the table is still deleting, the second response the table has deleted // and the ResourceNotFoundException is thrown. when(mockDynamoDBClient.describeTable(tableName)).thenReturn(mockDescribeResult).thenThrow( new ResourceNotFoundException("Table not found")); dynamoDB.delete(); verify(mockDynamoDBClient, times(1)).deleteTable(tableName); verify(mockDynamoDBClient, times(2)).describeTable(tableName); }
public boolean tableExists(final String tableName) { try { return TableStatus.ACTIVE.toString().equals(db.getTable(tableName).describe().getTableStatus().toUpperCase()); } catch (final ResourceNotFoundException rnfe) { return false; } }
private static void deleteTableIfPresent() { try { logger.info("Deleting table if present..."); DynamoCommons.getInstance().getDb().getTable(tableName).delete(); DynamoCommons.getInstance().getDb().getTable(tableName).waitForDelete(); } catch(final ResourceNotFoundException | InterruptedException ex) { } }
private void verifyTables() { try { ddb.describeTable(TABLE_NAME); } catch (ResourceNotFoundException rnfe) { createRecipientTable(); } }
public void closeStorage() { try { dynamoDBConnection.getDynamoClient().describeTable(getTableName()); dynamoDBConnection.getDynamoClient().deleteTable(getTableName()); } catch (ResourceNotFoundException e) { } }
@Override public TableDescription getTableDescription(String tableName) { try { TableDescription table = toTableDescription(_dynamodb.getTable(getTableName(tableName)).describe()); table.setTableName(tableName); return table; } catch ( ResourceNotFoundException ex ) { return null; } }
private BackendException processDynamoDbApiException(final Throwable e, final String apiName, final String tableName) { Preconditions.checkArgument(apiName != null); Preconditions.checkArgument(!apiName.isEmpty()); final String prefix; if (tableName == null) { prefix = apiName; } else { prefix = String.format("%s_%s", apiName, tableName); } final String message = String.format("%s %s", prefix, e.getMessage()); if (e instanceof ResourceNotFoundException) { return new BackendNotFoundException(String.format("%s; table not found", message), e); } else if (e instanceof ConditionalCheckFailedException) { return new PermanentLockingException(message, e); } else if (e instanceof AmazonServiceException) { if (e.getMessage() != null && (e.getMessage().contains(HASH_RANGE_KEY_SIZE_LIMIT) || e.getMessage().contains(UPDATE_ITEM_SIZE_LIMIT))) { return new PermanentBackendException(message, e); } else { return new TemporaryBackendException(message, e); } } else if (e instanceof AmazonClientException) { //all client exceptions are retriable by default return new TemporaryBackendException(message, e); } else if (e instanceof SocketException) { //sometimes this doesn't get caught by SDK return new TemporaryBackendException(message, e); } // unknown exception type return new PermanentBackendException(message, e); }
private boolean leaseTableExists() { DescribeTableRequest request = new DescribeTableRequest(); request.setTableName(conf.applicationName); DescribeTableResult result; try { result = dynamoDBClient.describeTable(request); } catch (ResourceNotFoundException e) { LOG.debug("Lease table '{}' does not exist", conf.applicationName); return false; } TableStatus tableStatus = TableStatus.fromValue(result.getTable().getTableStatus()); LOG.debug("Lease table exists and is in '{}' state", tableStatus); return tableStatus == TableStatus.ACTIVE; }
private boolean tableExists(String tableName) { try { dynamoDBClient.describeTable(tableName); return true; } catch (ResourceNotFoundException e) { if (LOG.isDebugEnabled()) { LOG.debug("Table '{}' did not exist.", tableName, e); } return false; } }
private void describeTable(String tableName) throws IllegalArgumentException { try { DescribeTableResult describeResult = dynamoDBClient.describeTable(tableName); tableDescription = describeResult.getTable(); } catch (ResourceNotFoundException rnfe) { throw new IllegalArgumentException("Error: given table " + tableName + " does not exist in given region."); } catch (AmazonServiceException ase) { if (ase.getErrorCode().equals("UnrecognizedClientException")) throw new IllegalArgumentException("Error: Security token in credential file invalid."); else throw new IllegalArgumentException(ase.getMessage()); } }
/** * Check if table has already existed Using describe table API. If table has * not been created, exception will be thrown. */ public String isTableExist(String tableName) { if (tableName == null) throw new IllegalArgumentException("tableName should not be null."); DescribeTableResult table = new DescribeTableResult(); try { table = client.describeTable(tableName); return table.getTable().getTableName(); } catch (ResourceNotFoundException rnfe) { return null; } }
@Override public void save(User user) throws UserExistsException, DataSourceTableDoesNotExistException { try { // See if User item exists User existing = this.find(user.getEmail()); // If the user exists, throw an exception if(existing != null) { throw new UserExistsException(); } // Convert the User object to a Map. The DynamoDB PutItemRequest object // requires the Item to be in the Map<String, AttributeValue> structure Map<String, AttributeValue> userItem = getMapFromUser(user); // Create a request to save and return the user PutItemRequest putItemRequest = new PutItemRequest() .withTableName(config.getProperty(ConfigurationSettings.ConfigProps.DDB_USERS_TABLE)) .withItem(userItem); // Save user dynamoClient.putItem(putItemRequest); } catch (ResourceNotFoundException rnfe) { throw new DataSourceTableDoesNotExistException(config.getProperty(ConfigurationSettings.ConfigProps.DDB_USERS_TABLE)); } catch (AmazonServiceException ase) { throw ase; } }
@Override public void update(User user) throws DataSourceTableDoesNotExistException { try { // If the object includes a profile pic file, upload it to S3 if(user.getprofilePicData() != null && user.getprofilePicData().getSize() > 0) { try { String profilePicUrl = this.uploadFileToS3(user.getprofilePicData()); user.setProfilePicKey(profilePicUrl); } catch (IOException e) { LOG.error("Error uploading profile pic to S3", e); } } // Convert the User object to a Map Map<String, AttributeValue> userItem = getMapFromUser(user); // Create a request to save and return the user PutItemRequest putItemRequest = new PutItemRequest() .withItem(userItem) .withTableName(config.getProperty(ConfigurationSettings.ConfigProps.DDB_USERS_TABLE)); // Save user dynamoClient.putItem(putItemRequest); } catch (ResourceNotFoundException rnfe) { throw new DataSourceTableDoesNotExistException(config.getProperty(ConfigurationSettings.ConfigProps.DDB_USERS_TABLE)); } catch (AmazonServiceException ase) { throw ase; } }
@Override public User find(String email) throws DataSourceTableDoesNotExistException { try { User user = null; // Create a request to find a User by email address GetItemRequest getItemRequest = new GetItemRequest() .withTableName( config.getProperty(ConfigurationSettings.ConfigProps.DDB_USERS_TABLE)) .addKeyEntry(HASH_KEY_NAME, new AttributeValue(email)); // Issue the request to find the User in DynamoDB GetItemResult getItemResult = dynamoClient.getItem(getItemRequest); // If an item was found if (getItemResult.getItem() != null) { // Marshal the Map<String, AttributeValue> structure returned in // the // GetItemResult to a User object user = getUserFromMap(getItemResult.getItem()); } return user; } catch (ResourceNotFoundException rnfe) { // The ResourceNotFoundException method is thrown by the getItem() // method // if the DynamoDB table doesn't exist. This exception is re-thrown // as a // custom, more specific DataSourceTableDoesNotExistException that // users // of this DAO understand. throw new DataSourceTableDoesNotExistException(config.getProperty(ConfigurationSettings.ConfigProps.DDB_USERS_TABLE)); } }
@Override public Map<String, AttributeValue> getItem(String tableName, HashMap<String, AttributeValue> primaryKey) { LOG.info("Retrieves a set of Attributes for an item that matches the primary key " + primaryKey + " from the table " + tableName); try { GetItemRequest getItemRequest = new GetItemRequest().withTableName(tableName) .withKey(primaryKey) .withConsistentRead(true); GetItemResult getItemResult = dynamoDBClient.getItem(getItemRequest); Map<String, AttributeValue> item = getItemResult.getItem(); if (item == null || item.isEmpty()) { LOG.warn("Could not find any item for the given UUID: " + primaryKey + " from " + tableName); return Collections.emptyMap(); } LOG.info("Listing collection from " + tableName + ": " + (item.size() / 8) + " items"); return item; } catch (ResourceNotFoundException rnfe) { LOG.error("Requested resource " + tableName + " not found ", rnfe); } catch (Exception ex) { LOG.error("Failed to get item into the " + tableName, ex); } return Collections.emptyMap(); }
/** * Retrieves information about the table, including the current status of * the table, the primary key schema and when the table was created. * * If the table does not exist, Amazon DynamoDB returns a * ResourceNotFoundException. * * @param tableName * - The name of the table * @return The response from the DescribeTable service method, as returned by AmazonDynamoDB */ private TableDescription describeTable(String tableName) { try { DescribeTableRequest describeTableRequest = new DescribeTableRequest().withTableName(tableName); TableDescription tableDescription = dynamoDBClient.describeTable(describeTableRequest).getTable(); if (tableDescription != null) { LOG.info("Table description of " + tableName + ": " + tableDescription); } return tableDescription; } catch (ResourceNotFoundException rnfe) { LOG.warn(rnfe.getMessage()); } return null; }
private void waitForTableDeleted(String tableName) { LOG.info("Waiting for table " + tableName + " while status DELETING..."); long startTime = System.currentTimeMillis(); long endTime = startTime + (10 * 60 * 1000); while (System.currentTimeMillis() < endTime) { try { DescribeTableRequest describeTableRequest = new DescribeTableRequest().withTableName(tableName); TableDescription tableDescription = dynamoDBClient.describeTable(describeTableRequest) .getTable(); String tableStatus = tableDescription.getTableStatus(); LOG.info("Current state for table " + tableName + ": " + tableStatus); if (tableStatus.equals(TableStatus.ACTIVE.toString())) { return; } } catch (ResourceNotFoundException rne) { LOG.warn("Table " + tableName + " is not found. It was deleted."); return; } try { Thread.sleep(1000 * 20); } catch (Exception ex) { LOG.warn(ex.getMessage()); } } throw new RuntimeException("Table " + tableName + " was never deleted"); }
public static void dropTable(final AmazonDynamoDB dynamoClient, final String dynamoTable) throws Exception { if (dynamoTable != null) { LOG.info(String.format("Dropping Dynamo Table %s", dynamoTable)); try { dynamoClient.deleteTable(dynamoTable); waitForTableState(dynamoClient, dynamoTable, TableStatus.DELETING); } catch (ResourceNotFoundException e) { LOG.info("OK - Table Not Found"); } } }
/*** * Return true if the table exists other false * @param clazz * @return tableExists */ public boolean tableExists(Class<?> clazz) { try{ this.getTableStatus(clazz); }catch(ResourceNotFoundException rnf){ return false; } return true; }
/*** * * @param clazz * @return tableStatus * @throws ResourceNotFoundException */ public String getTableStatus(Class<?> clazz) throws ResourceNotFoundException{ final String tableName = this.getClassAnnotationValue(clazz, DynamoDBTable.class, String.class, "tableName"); if(!Strings.isNullOrEmpty(tableName)){ this.getTableStatus(tableName); } return null; }
/*** * Get Table Status * @param tableName * @return status */ public String getTableStatus(final String tableName) throws ResourceNotFoundException{ final DescribeTableResult description = describeTable(tableName); if(description.getTable()!=null){ return description.getTable().getTableStatus(); } return null; }
/*** * Wait the specified time for the table to become active * @param clazz * @param maxWaitTimeSeconds * @param timeBetweenChecksSeconds */ private void waitForTableToBecomeActive(final String tableName, long maxWaitTimeSeconds, long timeBetweenChecksSeconds){ try{ long waitUntil = (DateTime.now().getMillis() + (maxWaitTimeSeconds * 1000L)); String status = null; while(!ACTIVE_TABLE_STATUS.equals(status) && DateTime.now().getMillis() < waitUntil){ status = this.getTableStatus(tableName); /** If it's active then return **/ if(ACTIVE_TABLE_STATUS.equals(status)){ LOGGER.info("Table for model: {} is active!", tableName); return; } LOGGER.info("Table for model: {} has status of {}. Waiting {} seconds for next check", tableName, status, timeBetweenChecksSeconds); Thread.sleep(timeBetweenChecksSeconds * 1000L); } if(!ACTIVE_TABLE_STATUS.equals(status) && DateTime.now().getMillis() > waitUntil){ LOGGER.warn("The timeout period expired while waiting for table: {} to become active. Status is {} maxWaitTimeInSeconds: {}", tableName, status, maxWaitTimeSeconds); } }catch(ResourceNotFoundException rnf){ LOGGER.warn("Table for model: {} does not exist", tableName); }catch(Exception ex){ LOGGER.error(ex.getMessage(), ex); } }
/** * Retrieves the table description for the specific resource name * * @param tableName * @return */ private TableDescription getTableSchema(String tableName) { TableDescription tableDescription = null; try { DescribeTableRequest describeTableRequest = new DescribeTableRequest() .withTableName(tableName); tableDescription = getDynamoDBClient() .describeTable(describeTableRequest).getTable(); } catch (ResourceNotFoundException e) { LOG.error("Error while getting table schema: " + tableName); return tableDescription; } return tableDescription; }
/** * Creates DynamoDB table. If the table already exists, it validates the key schema. If the key schemas match, a * warning is logged, otherwise an exception is raised. * * @param dynamoDB * {@link AmazonDynamoDB} used to create the table specified in the request. * @param request * Request for creating a table. * @return TableDescription of the existing table or newly created table */ public static TableDescription createTable(final AmazonDynamoDB dynamoDB, final CreateTableRequest request) { try { final DescribeTableResult result = dynamoDB.describeTable(request.getTableName()); if (!request.getKeySchema().equals(result.getTable().getKeySchema())) { throw new IllegalStateException("Table " + request.getTableName() + " already exists and has an invalid schema"); } LOGGER.warning("Table " + request.getTableName() + " already exists"); return result.getTable(); } catch (final ResourceNotFoundException e) { return dynamoDB.createTable(request).getTableDescription(); } }
@Test public void testCreateTableTableDoesNotExist() { final Collection<AttributeDefinition> ads = Arrays.asList(new AttributeDefinition("Hash", ScalarAttributeType.S)); final Collection<KeySchemaElement> kses = Arrays.asList(new KeySchemaElement("Hash", KeyType.HASH)); final TableDescription description = new TableDescription().withAttributeDefinitions(ads).withKeySchema(kses) .withTableName(tableName); final CreateTableResult cTR = new CreateTableResult().withTableDescription(description); EasyMock.expect(dynamoDB.describeTable(tableName)).andThrow(new ResourceNotFoundException(null)); final CreateTableRequest request = new CreateTableRequest().withAttributeDefinitions(ads).withKeySchema(kses) .withTableName(tableName); EasyMock.expect(dynamoDB.createTable(request)).andReturn(cTR); PowerMock.replayAll(); assertEquals(description, DynamoDBManager.createTable(dynamoDB, request)); PowerMock.verifyAll(); }
@Test public void testTableDoesNotExist() { EasyMock.expect(dynamoDB.describeTable(tableName)).andThrow(new ResourceNotFoundException("")); PowerMock.replayAll(); assertFalse(DynamoDBManager.doesTableExist(dynamoDB, tableName)); PowerMock.verifyAll(); }
private void createTableIfNotExisting() { DescribeTableRequest describeTableRequest = new DescribeTableRequest().withTableName(config().getTableName()); try { config().getDynamoDBClient().describeTable(describeTableRequest); } catch (ResourceNotFoundException e) { createTable(); } }