/** * The client execution timer uses interrupts to abort the client but if another thread * interrupts the current thread for another reason we don't want to squash the * {@link InterruptedException}. We should set the thread's interrupted status and throw the * exception back out (we can't throw the actual {@link InterruptedException} because it's * checked) */ @Test(timeout = TEST_TIMEOUT) public void interruptCausedBySomethingOtherThanTimer_PropagatesInterruptToCaller() { final int socketTimeoutInMillis = 100; httpClient = new AmazonHttpClient(new ClientConfiguration().withSocketTimeout(socketTimeoutInMillis) .withClientExecutionTimeout(CLIENT_EXECUTION_TIMEOUT) .withRetryPolicy(new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, new FixedTimeBackoffStrategy(CLIENT_EXECUTION_TIMEOUT), 1, false))); // We make sure the first connection has failed due to the socket timeout before // interrupting so we know that we are sleeping per the backoff strategy. Apache HTTP // client doesn't seem to honor interrupts reliably but Thread.sleep does interruptCurrentThreadAfterDelay(socketTimeoutInMillis * 2); try { httpClient.requestExecutionBuilder().request(newGetRequest()).execute(); fail("Exception expected"); } catch (AmazonClientException e) { assertTrue(Thread.currentThread().isInterrupted()); assertThat(e.getCause(), instanceOf(InterruptedException.class)); } }
@Test(timeout = TEST_TIMEOUT) public void clientExecutionTimeoutEnabled_WithShorterRequestTimeoutAndRetry_ThrowsClientExecutionTimeoutException() throws IOException { final int clientExecutionTimeout = 1500; final int requestTimeout = 1000; final int backoffTime = 300; httpClient = new AmazonHttpClient(new ClientConfiguration().withClientExecutionTimeout(clientExecutionTimeout) .withRequestTimeout(requestTimeout) .withRetryPolicy(new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, new FixedTimeBackoffStrategy(backoffTime), Integer.MAX_VALUE, false))); try { httpClient.requestExecutionBuilder().request(newGetRequest()).execute(); fail("Exception expected"); } catch (AmazonClientException e) { assertThat(e, instanceOf(ClientExecutionTimeoutException.class)); // Completed tasks means the client execution was aborted by the timer assertNumberOfTasksTriggered(httpClient.getClientExecutionTimer(), 1); assertNumberOfTasksTriggered(httpClient.getHttpRequestTimer(), 1); } }
private boolean shouldRetryCompleteMultipartUpload(AmazonWebServiceRequest originalRequest, AmazonS3Exception exception, int retriesAttempted) { final RetryPolicy retryPolicy = clientConfiguration.getRetryPolicy(); if (retryPolicy == null || retryPolicy.getRetryCondition() == null) { return false; } if (retryPolicy == PredefinedRetryPolicies.NO_RETRY_POLICY) { return false; } return completeMultipartUploadRetryCondition.shouldRetry (originalRequest, exception, retriesAttempted); }
@Test public void testProvide() { Env env = new Env().register("DYNAMODB_ENDPOINT", "fakeendpoint"); ClientConfiguration config = new ClientConfiguration(); config.setRetryPolicy(PredefinedRetryPolicies.NO_RETRY_POLICY); AWSCredentialsProvider credentialsProvider = new StaticCredentialsProvider( new BasicAWSCredentials("fake_access_key_id", "fake_secret_access_key")); AmazonDynamoDBFactory factory = new AmazonDynamoDBFactory(env, credentialsProvider, config); AmazonDynamoDB dynamoDB = factory.provide(); String message = ""; try { dynamoDB.listTables(); } catch (AmazonClientException e) { message = e.getMessage(); } assertTrue(message.startsWith("Unable to execute HTTP request: fakeendpoint")); factory.dispose(dynamoDB); }
@Bean AmazonSNSAsyncClient amazonSNS(ListenerConfig config, AWSCredentialsProvider awsCredentialsProvider) { AmazonSNSAsyncClient client = new AmazonSNSAsyncClient(awsCredentialsProvider, new ClientConfiguration().withRetryPolicy(PredefinedRetryPolicies.getDefaultRetryPolicy()), Executors.newSingleThreadScheduledExecutor()); client.setRegion(Region.getRegion(Regions.fromName(config.getRegion()))); return client; }
private static ClientConfiguration withRetryPolicy(ClientConfiguration config, String tableName, String indexName) { RetryPolicy.RetryCondition retryCondition = new PredefinedRetryPolicies.SDKDefaultRetryCondition() { @Override public boolean shouldRetry(AmazonWebServiceRequest originalRequest, AmazonClientException exception, int retriesAttempted) { if (exception instanceof AmazonServiceException) { AmazonServiceException ase = (AmazonServiceException)exception; if (RetryUtils.isThrottlingException(ase)) { if ( null != indexName ) { if(LOG.isWarnEnabled()) LOG.warn("throttling on table "+tableName+" index "+indexName); } else { if(LOG.isWarnEnabled()) LOG.warn("throttling on table "+tableName+" (main index)"); } } } return super.shouldRetry(originalRequest, exception, retriesAttempted); } }; config.withRetryPolicy(new RetryPolicy( retryCondition, PredefinedRetryPolicies.DYNAMODB_DEFAULT_BACKOFF_STRATEGY, PredefinedRetryPolicies.DYNAMODB_DEFAULT_MAX_ERROR_RETRY, true)); return config; }
protected AmazonApiGateway getApiGatewayClient() { if (apiGatewayClient != null) { return apiGatewayClient; } RetryPolicy.RetryCondition retryCondition = new RetryPolicy.RetryCondition() { @Override public boolean shouldRetry(AmazonWebServiceRequest amazonWebServiceRequest, AmazonClientException amazonClientException, int i) { if (amazonClientException instanceof TooManyRequestsException) { return true; } return PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION.shouldRetry(amazonWebServiceRequest, amazonClientException, i); } }; RetryPolicy retryPolicy = new RetryPolicy(retryCondition, PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, 10, true); ClientConfiguration clientConfig = new ClientConfiguration() .withRetryPolicy(retryPolicy); apiGatewayClient = new AmazonApiGatewayClient(getAWSCredentialsProvideChain(), clientConfig).withRegion(Region.getRegion(Regions.fromName(deployment.getRegion()))); return apiGatewayClient; }
@Provides protected ApiGateway provideAmazonApiGateway(AWSCredentialsProvider credsProvider, RetryPolicy.BackoffStrategy backoffStrategy, @Named("region") String region) { final RetryPolicy retrypolicy = new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, backoffStrategy, 5, true); final ClientConfiguration clientConfig = new ClientConfiguration().withUserAgent(USER_AGENT).withRetryPolicy(retrypolicy); return new AmazonApiGateway(getEndpoint(region)).with(credsProvider).with(clientConfig).getApiGateway(); }
/** * Asserts that the retry policy generated by the factory is configured with the correct values. */ @Test public void assertResultRetryPolicyConfiguredCorrectly() { int expectedMaxErrorRetry = 12345; when(configurationHelper.getProperty(any(), eq(Integer.class))).thenReturn(expectedMaxErrorRetry); RetryPolicy retryPolicy = retryPolicyFactory.getRetryPolicy(); assertEquals(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, retryPolicy.getRetryCondition()); assertEquals(backoffStrategy, retryPolicy.getBackoffStrategy()); assertEquals(expectedMaxErrorRetry, retryPolicy.getMaxErrorRetry()); }
@Test public void testDeleteDirectoryNoS3VersionsExist() { // Create an S3 file transfer request parameters DTO to access S3 objects. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME); s3FileTransferRequestParamsDto.setS3KeyPrefix(S3_KEY_PREFIX); // Create a retry policy. RetryPolicy retryPolicy = new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, INTEGER_VALUE, true); // Create an empty version listing. VersionListing versionListing = new VersionListing(); // Mock the external calls. when(retryPolicyFactory.getRetryPolicy()).thenReturn(retryPolicy); when(s3Operations.listVersions(any(ListVersionsRequest.class), any(AmazonS3Client.class))).thenReturn(versionListing); // Call the method under test. s3DaoImpl.deleteDirectory(s3FileTransferRequestParamsDto); // Verify the external calls. verify(retryPolicyFactory).getRetryPolicy(); verify(s3Operations).listVersions(any(ListVersionsRequest.class), any(AmazonS3Client.class)); verifyNoMoreInteractionsHelper(); }
/** * Factory method for DynamoDB's default {@link ClientConfiguration} */ public static ClientConfiguration dynamoDefault() { return new ClientConfiguration().withRetryPolicy(PredefinedRetryPolicies.DYNAMODB_DEFAULT); }
/** * Configures appender instance and makes it ready for use by the consumers. * It validates mandatory parameters and confirms if the configured stream is * ready for publishing data yet. * * Error details are made available through the fallback handler for this * appender * * @throws IllegalStateException if we encounter issues configuring this * appender instance */ @Override public void start() { if(layout == null) { initializationFailed = true; addError("Invalid configuration - No layout for appender: " + name); return; } if(streamName == null) { initializationFailed = true; addError("Invalid configuration - streamName cannot be null for appender: " + name); return; } ClientConfiguration clientConfiguration = new ClientConfiguration(); clientConfiguration.setMaxErrorRetry(maxRetries); clientConfiguration .setRetryPolicy(new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, maxRetries, true)); clientConfiguration.setUserAgent(AppenderConstants.USER_AGENT_STRING); BlockingQueue<Runnable> taskBuffer = new LinkedBlockingDeque<Runnable>(bufferSize); threadPoolExecutor = new ThreadPoolExecutor(threadCount, threadCount, AppenderConstants.DEFAULT_THREAD_KEEP_ALIVE_SEC, TimeUnit.SECONDS, taskBuffer, new BlockFastProducerPolicy()); threadPoolExecutor.prestartAllCoreThreads(); this.client = createClient(credentials, clientConfiguration, threadPoolExecutor); client.setRegion(findRegion()); if(!Validator.isBlank(endpoint)) { if(!Validator.isBlank(region)) { addError("Received configuration for both region as well as Amazon Kinesis endpoint. (" + endpoint + ") will be used as endpoint instead of default endpoint for region (" + region + ")"); } client.setEndpoint(endpoint); } validateStreamName(client, streamName); super.start(); }
@Test public void testGetTemporarySecurityCredentials() { // Create an AWS parameters DTO with proxy settings. AwsParamsDto awsParamsDto = new AwsParamsDto(); awsParamsDto.setHttpProxyHost(HTTP_PROXY_HOST); awsParamsDto.setHttpProxyPort(HTTP_PROXY_PORT); // Specify the duration, in seconds, of the role session. int awsRoleDurationSeconds = INTEGER_VALUE; // Create an IAM policy. Policy policy = new Policy(STRING_VALUE); // Create a retry policy. RetryPolicy retryPolicy = new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, INTEGER_VALUE, true); // Create the expected assume role request. AssumeRoleRequest assumeRoleRequest = new AssumeRoleRequest().withRoleArn(AWS_ROLE_ARN).withRoleSessionName(SESSION_NAME).withPolicy(policy.toJson()) .withDurationSeconds(awsRoleDurationSeconds); // Create AWS credentials for API authentication. Credentials credentials = new Credentials(); credentials.setAccessKeyId(AWS_ASSUMED_ROLE_ACCESS_KEY); credentials.setSecretAccessKey(AWS_ASSUMED_ROLE_SECRET_KEY); credentials.setSessionToken(AWS_ASSUMED_ROLE_SESSION_TOKEN); // Create an assume role result. AssumeRoleResult assumeRoleResult = new AssumeRoleResult(); assumeRoleResult.setCredentials(credentials); // Mock the external calls. when(retryPolicyFactory.getRetryPolicy()).thenReturn(retryPolicy); when(stsOperations.assumeRole(any(AWSSecurityTokenServiceClient.class), eq(assumeRoleRequest))).thenReturn(assumeRoleResult); // Call the method under test. Credentials result = stsDaoImpl.getTemporarySecurityCredentials(awsParamsDto, SESSION_NAME, AWS_ROLE_ARN, awsRoleDurationSeconds, policy); // Verify the external calls. verify(retryPolicyFactory).getRetryPolicy(); verify(stsOperations).assumeRole(any(AWSSecurityTokenServiceClient.class), eq(assumeRoleRequest)); verifyNoMoreInteractionsHelper(); // Validate the returned object. assertEquals(credentials, result); }
@Test public void testGetTemporarySecurityCredentialsMissingOptionalParameters() { // Create an AWS parameters DTO without proxy settings. AwsParamsDto awsParamsDto = new AwsParamsDto(); // Specify the duration, in seconds, of the role session. int awsRoleDurationSeconds = INTEGER_VALUE; // Create a retry policy. RetryPolicy retryPolicy = new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, INTEGER_VALUE, true); // Create the expected assume role request. AssumeRoleRequest assumeRoleRequest = new AssumeRoleRequest().withRoleArn(AWS_ROLE_ARN).withRoleSessionName(SESSION_NAME).withDurationSeconds(awsRoleDurationSeconds); // Create AWS credentials for API authentication. Credentials credentials = new Credentials(); credentials.setAccessKeyId(AWS_ASSUMED_ROLE_ACCESS_KEY); credentials.setSecretAccessKey(AWS_ASSUMED_ROLE_SECRET_KEY); credentials.setSessionToken(AWS_ASSUMED_ROLE_SESSION_TOKEN); // Create an assume role result. AssumeRoleResult assumeRoleResult = new AssumeRoleResult(); assumeRoleResult.setCredentials(credentials); // Mock the external calls. when(retryPolicyFactory.getRetryPolicy()).thenReturn(retryPolicy); when(stsOperations.assumeRole(any(AWSSecurityTokenServiceClient.class), eq(assumeRoleRequest))).thenReturn(assumeRoleResult); // Call the method under test. Please note that we do not specify an IAM policy. Credentials result = stsDaoImpl.getTemporarySecurityCredentials(awsParamsDto, SESSION_NAME, AWS_ROLE_ARN, awsRoleDurationSeconds, null); // Verify the external calls. verify(retryPolicyFactory).getRetryPolicy(); verify(stsOperations).assumeRole(any(AWSSecurityTokenServiceClient.class), eq(assumeRoleRequest)); verifyNoMoreInteractionsHelper(); // Validate the returned object. assertEquals(credentials, result); }
@Test public void testDeleteDirectoryMultiObjectDeleteException() { // Create an S3 file transfer request parameters DTO to access S3 objects. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME); s3FileTransferRequestParamsDto.setS3KeyPrefix(S3_KEY_PREFIX); // Create a retry policy. RetryPolicy retryPolicy = new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, INTEGER_VALUE, true); // Create an S3 version summary. S3VersionSummary s3VersionSummary = new S3VersionSummary(); s3VersionSummary.setKey(S3_KEY); s3VersionSummary.setVersionId(S3_VERSION_ID); // Create a version listing. VersionListing versionListing = new VersionListing(); versionListing.setVersionSummaries(Arrays.asList(s3VersionSummary)); // Create a delete error. MultiObjectDeleteException.DeleteError deleteError = new MultiObjectDeleteException.DeleteError(); deleteError.setKey(S3_KEY); deleteError.setVersionId(S3_VERSION_ID); deleteError.setCode(ERROR_CODE); deleteError.setMessage(ERROR_MESSAGE); // Create a multi object delete exception. MultiObjectDeleteException multiObjectDeleteException = new MultiObjectDeleteException(Arrays.asList(deleteError), new ArrayList<>()); // Mock the external calls. when(retryPolicyFactory.getRetryPolicy()).thenReturn(retryPolicy); when(s3Operations.listVersions(any(ListVersionsRequest.class), any(AmazonS3Client.class))).thenReturn(versionListing); when(s3Operations.deleteObjects(any(DeleteObjectsRequest.class), any(AmazonS3Client.class))).thenThrow(multiObjectDeleteException); // Try to call the method under test. try { s3DaoImpl.deleteDirectory(s3FileTransferRequestParamsDto); } catch (IllegalStateException e) { assertEquals(String.format("Failed to delete keys/key versions with prefix \"%s\" from bucket \"%s\". " + "Reason: One or more objects could not be deleted (Service: null; Status Code: 0; Error Code: null; Request ID: null; S3 Extended Request ID: null)", S3_KEY_PREFIX, S3_BUCKET_NAME), e.getMessage()); } // Verify the external calls. verify(retryPolicyFactory, times(2)).getRetryPolicy(); verify(s3Operations).listVersions(any(ListVersionsRequest.class), any(AmazonS3Client.class)); verify(s3Operations).deleteObjects(any(DeleteObjectsRequest.class), any(AmazonS3Client.class)); verifyNoMoreInteractionsHelper(); }
@Test public void testTagObjects() { // Create an S3 file transfer request parameters DTO to access S3 objects. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME); s3FileTransferRequestParamsDto.setFiles(Arrays.asList(new File(S3_KEY_PREFIX + "/" + LOCAL_FILE))); // Create an S3 file transfer request parameters DTO to tag S3 objects. S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto = new S3FileTransferRequestParamsDto(); s3ObjectTaggerParamsDto.setAwsAccessKeyId(AWS_ASSUMED_ROLE_ACCESS_KEY); s3ObjectTaggerParamsDto.setAwsSecretKey(AWS_ASSUMED_ROLE_SECRET_KEY); s3ObjectTaggerParamsDto.setSessionToken(AWS_ASSUMED_ROLE_SESSION_TOKEN); // Create an S3 object tag. Tag tag = new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE); // Create a retry policy. RetryPolicy retryPolicy = new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, INTEGER_VALUE, true); // Create a get object tagging result. GetObjectTaggingResult getObjectTaggingResult = new GetObjectTaggingResult(null); // Create a set object tagging result. SetObjectTaggingResult setObjectTaggingResult = new SetObjectTaggingResult(); // Mock the external calls. when(retryPolicyFactory.getRetryPolicy()).thenReturn(retryPolicy); when(s3Operations.getObjectTagging(any(GetObjectTaggingRequest.class), any(AmazonS3Client.class))).thenReturn(getObjectTaggingResult); when(s3Operations.setObjectTagging(any(SetObjectTaggingRequest.class), any(AmazonS3Client.class))).thenReturn(setObjectTaggingResult); // Call the method under test. s3DaoImpl.tagObjects(s3FileTransferRequestParamsDto, s3ObjectTaggerParamsDto, tag); // Verify the external calls. verify(retryPolicyFactory, times(2)).getRetryPolicy(); verify(s3Operations).getObjectTagging(any(GetObjectTaggingRequest.class), any(AmazonS3Client.class)); verify(s3Operations).setObjectTagging(any(SetObjectTaggingRequest.class), any(AmazonS3Client.class)); verifyNoMoreInteractionsHelper(); }
public InstrumentedRetryCondition(Registry registry) { this(registry, PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION); }
public InstrumentedBackoffStrategy(Registry registry) { this(registry, PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY); }
/** * Configures this appender instance and makes it ready for use by the * consumers. It validates mandatory parameters and confirms if the configured * stream is ready for publishing data yet. * * Error details are made available through the fallback handler for this * appender * * @throws IllegalStateException * if we encounter issues configuring this appender instance */ @Override public void activateOptions() { if (streamName == null) { initializationFailed = true; error("Invalid configuration - streamName cannot be null for appender: " + name); } if (layout == null) { initializationFailed = true; error("Invalid configuration - No layout for appender: " + name); } ClientConfiguration clientConfiguration = new ClientConfiguration(); clientConfiguration = setProxySettingsFromSystemProperties(clientConfiguration); clientConfiguration.setMaxErrorRetry(maxRetries); clientConfiguration.setRetryPolicy(new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, maxRetries, true)); clientConfiguration.setUserAgent(AppenderConstants.USER_AGENT_STRING); BlockingQueue<Runnable> taskBuffer = new LinkedBlockingDeque<Runnable>(bufferSize); ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(threadCount, threadCount, AppenderConstants.DEFAULT_THREAD_KEEP_ALIVE_SEC, TimeUnit.SECONDS, taskBuffer, new BlockFastProducerPolicy()); threadPoolExecutor.prestartAllCoreThreads(); kinesisClient = new AmazonKinesisAsyncClient(new CustomCredentialsProviderChain(), clientConfiguration, threadPoolExecutor); boolean regionProvided = !Validator.isBlank(region); if (!regionProvided) { region = AppenderConstants.DEFAULT_REGION; } if (!Validator.isBlank(endpoint)) { if (regionProvided) { LOGGER .warn("Received configuration for both region as well as Amazon Kinesis endpoint. (" + endpoint + ") will be used as endpoint instead of default endpoint for region (" + region + ")"); } kinesisClient.setEndpoint(endpoint, AppenderConstants.DEFAULT_SERVICE_NAME, region); } else { kinesisClient.setRegion(Region.getRegion(Regions.fromName(region))); } DescribeStreamResult describeResult = null; try { describeResult = kinesisClient.describeStream(streamName); String streamStatus = describeResult.getStreamDescription().getStreamStatus(); if (!StreamStatus.ACTIVE.name().equals(streamStatus) && !StreamStatus.UPDATING.name().equals(streamStatus)) { initializationFailed = true; error("Stream " + streamName + " is not ready (in active/updating status) for appender: " + name); } } catch (ResourceNotFoundException rnfe) { initializationFailed = true; error("Stream " + streamName + " doesn't exist for appender: " + name, rnfe); } asyncCallHander = new AsyncPutCallStatsReporter(name); }
/** * Gets the application's default retry policy. The policy uses PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, a SimpleExponentialBackoffStrategy, and the * maximum number of attempts is dynamically configurable through AWS_MAX_RETRY_ATTEMPT. * * @return RetryPolicy */ public RetryPolicy getRetryPolicy() { return new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, backoffStrategy, configurationHelper.getProperty(ConfigurationValue.AWS_MAX_RETRY_ATTEMPT, Integer.class), true); }