/** * Capture the metrics for the given throwable. */ private <T extends Throwable> T captureExceptionMetrics(T t) { awsRequestMetrics.incrementCounterWith(Field.Exception) .addProperty(Field.Exception, t); if (t instanceof AmazonServiceException) { AmazonServiceException ase = (AmazonServiceException) t; if (RetryUtils.isThrottlingException(ase)) { awsRequestMetrics.incrementCounterWith(Field.ThrottleException) .addProperty(Field.ThrottleException, ase); } } return t; }
private static ClientConfiguration withRetryPolicy(ClientConfiguration config, String tableName, String indexName) { RetryPolicy.RetryCondition retryCondition = new PredefinedRetryPolicies.SDKDefaultRetryCondition() { @Override public boolean shouldRetry(AmazonWebServiceRequest originalRequest, AmazonClientException exception, int retriesAttempted) { if (exception instanceof AmazonServiceException) { AmazonServiceException ase = (AmazonServiceException)exception; if (RetryUtils.isThrottlingException(ase)) { if ( null != indexName ) { if(LOG.isWarnEnabled()) LOG.warn("throttling on table "+tableName+" index "+indexName); } else { if(LOG.isWarnEnabled()) LOG.warn("throttling on table "+tableName+" (main index)"); } } } return super.shouldRetry(originalRequest, exception, retriesAttempted); } }; config.withRetryPolicy(new RetryPolicy( retryCondition, PredefinedRetryPolicies.DYNAMODB_DEFAULT_BACKOFF_STRATEGY, PredefinedRetryPolicies.DYNAMODB_DEFAULT_MAX_ERROR_RETRY, true)); return config; }
@Provides protected RetryPolicy.BackoffStrategy provideBackoffStrategy() { // tune these parameters to handle throttling errors final int maxBackoffInMilliseconds = 50 * 1000; // maximum exponential back-off time before retrying a request final int throttlingScaleFactor = 800; // base sleep time for throttling exceptions final int maxRetriesBeforeBackoff = 10; // log2(maxBackoffInMilliseconds/throttlingScaleFactor) final int baseScaleFactor = 600; // base sleep time for general exceptions final int throttlingScaleFactorRandomRange = throttlingScaleFactor / 4; final Random random = new Random(); return (originalRequest, exception, retriesAttempted) -> { LOG.debug("Caught error from service. Retry attempt: " + retriesAttempted, exception); if (retriesAttempted < 0) return 0; if (retriesAttempted > maxRetriesBeforeBackoff) return maxBackoffInMilliseconds; int scaleFactor; if (exception instanceof AmazonServiceException && RetryUtils.isThrottlingException((AmazonServiceException) exception)) { scaleFactor = throttlingScaleFactor + random.nextInt(throttlingScaleFactorRandomRange); } else { scaleFactor = baseScaleFactor; } long delay = (1L << retriesAttempted) * scaleFactor; delay = Math.min(delay, maxBackoffInMilliseconds); LOG.info("Client backing off for " + delay + "ms"); return delay; }; }
@Override public void afterError(Request<?> request, Response<?> response, Exception e) { if (isSubsegmentDuplicate(recorder.getCurrentSubsegmentOptional(), request)) { Optional<Subsegment> currentSubsegmentOptional = recorder.getCurrentSubsegmentOptional(); if (!currentSubsegmentOptional.isPresent()) { return; } Subsegment currentSubsegment = currentSubsegmentOptional.get(); int statusCode = -1; if (null != response) { statusCode = response.getHttpResponse().getStatusCode(); } else { if (e instanceof AmazonServiceException) { AmazonServiceException ase = (AmazonServiceException) e; statusCode = ase.getStatusCode(); /* * The S3 client will throw and re-swallow AmazonServiceExceptions if they have these status codes. Customers will never see the exceptions in their application code but they still * travel through our TracingHandler#afterError method. We special case these status codes in order to prevent addition of the full exception object to the current subsegment. * Instead, we'll just add any exception error message to the current subsegment's cause's message. */ if ((304 == statusCode || 412 == statusCode) && S3_SERVICE_NAME.equals(ase.getServiceName())) { populateAndEndSubsegment(currentSubsegment, request, response, ase); return; } if (RetryUtils.isThrottlingException(ase)) { currentSubsegment.setError(true); currentSubsegment.setThrottle(true); } } } if (-1 != statusCode) { int statusCodePrefix = statusCode / 100; if (4 == statusCodePrefix) { currentSubsegment.setError(true); if (429 == statusCode) { currentSubsegment.setThrottle(true); } } } currentSubsegment.addException(e); if (e instanceof AmazonServiceException) { populateAndEndSubsegment(currentSubsegment, request, response, (AmazonServiceException) e); } else { populateAndEndSubsegment(currentSubsegment, request, response); } } }
/** * Returns true if a failed request should be retried. * * @param params Params for the individual request being executed. * @param exception The client/service exception from the failed request. * @return True if the failed request should be retried. */ private boolean shouldRetry(ExecOneRequestParams params, SdkBaseException exception) { final int retriesAttempted = params.requestCount - 1; final HttpRequestBase method = params.apacheRequest; // Never retry on requests containing non-repeatable entity if (method instanceof HttpEntityEnclosingRequest) { HttpEntity entity = ((HttpEntityEnclosingRequest) method).getEntity(); if (entity != null && !entity.isRepeatable()) { if (log.isDebugEnabled()) { log.debug("Entity not repeatable"); } return false; } } // Do not use retry capacity for throttling exceptions if (!RetryUtils.isThrottlingException(exception)) { // See if we have enough available retry capacity to be able to execute // this retry attempt. if (!retryCapacity.acquire(THROTTLED_RETRY_COST)) { awsRequestMetrics.incrementCounter(ThrottledRetryCount); return false; } executionContext.markRetryCapacityConsumed(); } RetryPolicyContext context = RetryPolicyContext.builder() .request(request) .originalRequest(requestConfig.getOriginalRequest()) .exception(exception) .retriesAttempted(retriesAttempted) .httpStatusCode(params.getStatusCode()) .build(); // Finally, pass all the context information to the RetryCondition and let it // decide whether it should be retried. if (!retryPolicy.shouldRetry(context)) { // If the retry policy fails we immediately return consumed capacity to the pool. if (executionContext.retryCapacityConsumed()) { retryCapacity.release(THROTTLED_RETRY_COST); } return false; } return true; }
@Override public boolean shouldRetry(AmazonWebServiceRequest amazonWebServiceRequest, AmazonClientException e, int i) { return RetryUtils.isThrottlingException(e); }
@Override public boolean shouldRetry(AmazonWebServiceRequest originalRequest, AmazonClientException exception, int retriesAttempted) { Utils.log(CustomRetryCondition.class, CustomRetryCondition.class.getSimpleName(), Level.FINE, () -> String .format("Encountered exception %s for request %s, retries attempted: %d", Utils.toString(exception), originalRequest, retriesAttempted)); // Always retry on client exceptions caused by IOException if (exception.getCause() instanceof IOException) { return true; } // Only retry on a subset of service exceptions if (exception instanceof AmazonServiceException) { AmazonServiceException ase = (AmazonServiceException) exception; /* * For 500 internal server errors and 503 service unavailable errors, we want to * retry, but we need to use an exponential back-off strategy so that we don't * overload a server with a flood of retries. */ if (RetryUtils.isRetryableServiceException(new SdkBaseException(ase))) { return true; } /* * Throttling is reported as a 400 error from newer services. To try and smooth out * an occasional throttling error, we'll pause and retry, hoping that the pause is * long enough for the request to get through the next time. */ if (RetryUtils.isThrottlingException(new SdkBaseException(ase))) { return true; } /* * Clock skew exception. If it is then we will get the time offset between the * device time and the server time to set the clock skew and then retry the request. */ if (RetryUtils.isClockSkewError(new SdkBaseException(ase))) { return true; } } return false; }
