private File downloadFile(String objectName) throws IOException { File tempFile = new File(System.getProperty("java.io.tmpdir"), objectName.replace('/', '-')); Download download = s3TransferManager.download(bucketName, objectName, tempFile); try { download.waitForCompletion(); } catch(SdkBaseException | InterruptedException e) { throw new RuntimeException(e); } return tempFile; }
private RequestExecutor(Request<?> request, RequestConfig requestConfig, HttpResponseHandler<? extends SdkBaseException> errorResponseHandler, HttpResponseHandler<Output> responseHandler, ExecutionContext executionContext, List<RequestHandler2> requestHandler2s) { this.request = request; this.requestConfig = requestConfig; this.errorResponseHandler = errorResponseHandler; this.responseHandler = responseHandler; this.executionContext = executionContext; this.requestHandler2s = requestHandler2s; this.awsRequestMetrics = executionContext.getAwsRequestMetrics(); }
/** * Returns the difference between the client's clock time and the service clock time in unit * of seconds. */ private int parseClockSkewOffset(org.apache.http.HttpResponse response, SdkBaseException exception) { final long currentTimeMilli = System.currentTimeMillis(); Date serverDate; String serverDateStr = null; Header[] responseDateHeader = response.getHeaders("Date"); try { if (responseDateHeader.length == 0) { // SQS doesn't return Date header final String errmsg = exception.getMessage(); serverDateStr = getServerDateFromException(errmsg); if (serverDateStr == null) { log.warn("Unable to parse clock skew offset from errmsg: " + errmsg); return 0; } serverDate = DateUtils.parseCompressedISO8601Date(serverDateStr); } else { serverDateStr = responseDateHeader[0].getValue(); serverDate = DateUtils.parseRFC822Date(serverDateStr); } } catch (RuntimeException e) { log.warn("Unable to parse clock skew offset from response: " + serverDateStr, e); return 0; } long diff = currentTimeMilli - serverDate.getTime(); return (int) (diff / 1000); }
/** * Normal invoke with authentication. Credentials are required and may be overriden at the * request level. **/ private <Output, Input> Response<Output> invoke(Request<Input> request, RequestConfig requestConfig, ExecutionContext executionContext, HttpResponseHandler<Output> responseHandler, HttpResponseHandler<? extends SdkBaseException> errorResponseHandler) { executionContext.setCredentialsProvider(CredentialUtils.getCredentialsProvider( requestConfig, awsCredentialsProvider)); return doInvoke(request, requestConfig, executionContext, responseHandler, errorResponseHandler); }
/** * Invoke the request using the http client. Assumes credentials (or lack thereof) have been * configured in the ExecutionContext beforehand. **/ private <Output, Input> Response<Output> doInvoke(Request<Input> request, RequestConfig requestConfig, ExecutionContext executionContext, HttpResponseHandler<Output> responseHandler, HttpResponseHandler<? extends SdkBaseException> errorResponseHandler) { request.setEndpoint(endpoint); return client.requestExecutionBuilder() .request(request) .requestConfig(requestConfig) .executionContext(executionContext) .errorResponseHandler(errorResponseHandler) .execute(responseHandler); }
private RetryPolicyContext(Object originalRequest, Request<?> request, SdkBaseException exception, int retriesAttempted, Integer httpStatusCode) { this.originalRequest = originalRequest; this.request = request; this.exception = exception; this.retriesAttempted = retriesAttempted; this.httpStatusCode = httpStatusCode; }
/** * Returns true if the specified exception is a throttling error. * * @param exception The exception to test. * @return True if the exception resulted from a throttling error message from a service, otherwise false. */ public static boolean isThrottlingException(SdkBaseException exception) { if (!isAse(exception)) { return false; } final AmazonServiceException ase = toAse(exception); return THROTTLING_ERROR_CODES.contains(ase.getErrorCode()) || ase.getStatusCode() == 429; }
@Override protected <T> T call(Supplier<T> callable) { try { return new S3Command<>(setter, callable).execute(); } catch (HystrixRuntimeException | HystrixBadRequestException e) { if (e.getCause() instanceof SdkBaseException) { throw (SdkBaseException) e.getCause(); } else { throw e; } } }
private boolean keyExists(String bucket, String key) throws MojoFailureException { try { s3Service.getObjectMetadata(bucket, key); return true; } catch (SdkBaseException e) { if (e instanceof AmazonServiceException && ((AmazonServiceException) e).getStatusCode() == 404) { return false; } else { throw new MojoFailureException("Error getting object metadata for key " + key, e); } } }
/** * Returns true if a failed request should be retried. * * @param params Params for the individual request being executed. * @param exception The client/service exception from the failed request. * @return True if the failed request should be retried. */ private boolean shouldRetry(ExecOneRequestParams params, SdkBaseException exception) { final int retriesAttempted = params.requestCount - 1; final HttpRequestBase method = params.apacheRequest; // Never retry on requests containing non-repeatable entity if (method instanceof HttpEntityEnclosingRequest) { HttpEntity entity = ((HttpEntityEnclosingRequest) method).getEntity(); if (entity != null && !entity.isRepeatable()) { if (log.isDebugEnabled()) { log.debug("Entity not repeatable"); } return false; } } // Do not use retry capacity for throttling exceptions if (!RetryUtils.isThrottlingException(exception)) { // See if we have enough available retry capacity to be able to execute // this retry attempt. if (!retryCapacity.acquire(THROTTLED_RETRY_COST)) { awsRequestMetrics.incrementCounter(ThrottledRetryCount); return false; } executionContext.markRetryCapacityConsumed(); } RetryPolicyContext context = RetryPolicyContext.builder() .request(request) .originalRequest(requestConfig.getOriginalRequest()) .exception(exception) .retriesAttempted(retriesAttempted) .httpStatusCode(params.getStatusCode()) .build(); // Finally, pass all the context information to the RetryCondition and let it // decide whether it should be retried. if (!retryPolicy.shouldRetry(context)) { // If the retry policy fails we immediately return consumed capacity to the pool. if (executionContext.retryCapacityConsumed()) { retryCapacity.release(THROTTLED_RETRY_COST); } return false; } return true; }
public HttpResponseHandler<? extends SdkBaseException> getErrorResponseHandler() { return errorResponseHandler; }
public ClientExecutionParams<Input, Output> withErrorResponseHandler( HttpResponseHandler<? extends SdkBaseException> errorResponseHandler) { this.errorResponseHandler = errorResponseHandler; return this; }
/** * @return The previous exception (may be a client or a service exception). */ public SdkBaseException exception() { return this.exception; }
/** * @deprecated By {@link RetryUtils#isRetryableServiceException(SdkBaseException)} */ @Deprecated public static boolean isRetryableServiceException(AmazonServiceException exception) { return isRetryableServiceException((SdkBaseException) exception); }
/** * @deprecated In favor of {@link RetryUtils#isThrottlingException(SdkBaseException)} */ @Deprecated public static boolean isThrottlingException(AmazonServiceException exception) { return isThrottlingException((SdkBaseException) exception); }
/** * @deprecated By {@link RetryUtils#isRequestEntityTooLargeException(SdkBaseException)} */ @Deprecated public static boolean isRequestEntityTooLargeException(AmazonServiceException exception) { return isRequestEntityTooLargeException((SdkBaseException) exception); }
/** * @deprecated By {@link RetryUtils#isClockSkewError(SdkBaseException)} */ @Deprecated public static boolean isClockSkewError(AmazonServiceException exception) { return isClockSkewError((SdkBaseException) exception); }
private static boolean isAse(SdkBaseException e) { return e instanceof AmazonServiceException; }
private static AmazonServiceException toAse(SdkBaseException e) { return (AmazonServiceException) e; }
@Test public void genericBaseException_ReturnsFalse() { assertFalse(condition.shouldRetry(RetryPolicyContexts.withException(new SdkBaseException("foo")))); }
public static RetryPolicyContext withException(SdkBaseException e) { return RetryPolicyContext.builder() .exception(e) .build(); }
@Override public boolean shouldRetry(AmazonWebServiceRequest originalRequest, AmazonClientException exception, int retriesAttempted) { Utils.log(CustomRetryCondition.class, CustomRetryCondition.class.getSimpleName(), Level.FINE, () -> String .format("Encountered exception %s for request %s, retries attempted: %d", Utils.toString(exception), originalRequest, retriesAttempted)); // Always retry on client exceptions caused by IOException if (exception.getCause() instanceof IOException) { return true; } // Only retry on a subset of service exceptions if (exception instanceof AmazonServiceException) { AmazonServiceException ase = (AmazonServiceException) exception; /* * For 500 internal server errors and 503 service unavailable errors, we want to * retry, but we need to use an exponential back-off strategy so that we don't * overload a server with a flood of retries. */ if (RetryUtils.isRetryableServiceException(new SdkBaseException(ase))) { return true; } /* * Throttling is reported as a 400 error from newer services. To try and smooth out * an occasional throttling error, we'll pause and retry, hoping that the pause is * long enough for the request to get through the next time. */ if (RetryUtils.isThrottlingException(new SdkBaseException(ase))) { return true; } /* * Clock skew exception. If it is then we will get the time offset between the * device time and the server time to set the clock skew and then retry the request. */ if (RetryUtils.isClockSkewError(new SdkBaseException(ase))) { return true; } } return false; }
/** * Returns true if the specified exception is a retryable service side exception. * * @param exception The exception to test. * @return True if the exception resulted from a retryable service error, otherwise false. */ public static boolean isRetryableServiceException(SdkBaseException exception) { return isAse(exception) && RETRYABLE_STATUS_CODES.contains(toAse(exception).getStatusCode()); }
/** * Returns true if the specified exception is a request entity too large error. * * @param exception The exception to test. * @return True if the exception resulted from a request entity too large error message from a service, otherwise false. */ public static boolean isRequestEntityTooLargeException(SdkBaseException exception) { return isAse(exception) && toAse(exception).getStatusCode() == HttpStatus.SC_REQUEST_TOO_LONG; }
/** * Returns true if the specified exception is a clock skew error. * * @param exception The exception to test. * @return True if the exception resulted from a clock skews error message from a service, otherwise false. */ public static boolean isClockSkewError(SdkBaseException exception) { return isAse(exception) && CLOCK_SKEW_ERROR_CODES.contains(toAse(exception).getErrorCode()); }