Java 类com.amazonaws.util.BinaryUtils 实例源码

@Test
public void testWriterWithExistingStream() throws Exception
{
    initialize("TestKinesisAppender/testWriterWithExistingStream.properties");

    MockKinesisClient mockClient = new MockKinesisClient();

    appender.setThreadFactory(new DefaultThreadFactory());
    appender.setWriterFactory(mockClient.newWriterFactory());

    logger.debug("example message");
    mockClient.allowWriterThread();

    assertEquals("describeStream: invocation count",        1,          mockClient.describeStreamInvocationCount);
    assertEquals("describeStream: stream name",             "argle",    mockClient.describeStreamStreamName);
    assertEquals("createStream: invocation count",          0,          mockClient.createStreamInvocationCount);
    assertEquals("putRecords: invocation count",            1,          mockClient.putRecordsInvocationCount);
    assertEquals("putRecords: source record count",         1,          mockClient.putRecordsSourceRecords.size());
    assertEquals("putRecords: source record partition key", "bargle",   mockClient.putRecordsSourceRecords.get(0).getPartitionKey());
    assertEquals("putRecords: source record content",       "example message\n",
                                                            new String(
                                                                BinaryUtils.copyAllBytesFrom(mockClient.putRecordsSourceRecords.get(0).getData()),
                                                                "UTF-8"));
}

/**
 * Step 2 of the AWS Signature version 4 calculation. Refer to
 * http://docs.aws
 * .amazon.com/general/latest/gr/sigv4-create-string-to-sign.html.
 */
protected String createStringToSign(String canonicalRequest,
        AWS4SignerRequestParams signerParams) {

    final StringBuilder stringToSignBuilder = new StringBuilder(
            signerParams.getSigningAlgorithm());
    stringToSignBuilder.append(LINE_SEPARATOR)
            .append(signerParams.getFormattedSigningDateTime())
            .append(LINE_SEPARATOR)
            .append(signerParams.getScope())
            .append(LINE_SEPARATOR)
            .append(BinaryUtils.toHex(hash(canonicalRequest)));

    final String stringToSign = stringToSignBuilder.toString();

    if (log.isDebugEnabled())
        log.debug("AWS4 String to Sign: '\"" + stringToSign + "\"");

    return stringToSign;
}

/**
 * Creates the authorization header to be included in the request.
 */
private String buildAuthorizationHeader(SignableRequest<?> request,
        byte[] signature, AWSCredentials credentials,
        AWS4SignerRequestParams signerParams) {
    final String signingCredentials = credentials.getAWSAccessKeyId() + "/"
            + signerParams.getScope();

    final String credential = "Credential="
            + signingCredentials;
    final String signerHeaders = "SignedHeaders="
            + getSignedHeadersString(request);
    final String signatureHeader = "Signature="
            + BinaryUtils.toHex(signature);

    final StringBuilder authHeaderBuilder = new StringBuilder();

    authHeaderBuilder.append(AWS4_SIGNING_ALGORITHM)
                     .append(" ")
                     .append(credential)
                     .append(", ")
                     .append(signerHeaders)
                     .append(", ")
                     .append(signatureHeader);

    return authHeaderBuilder.toString();
}

private void finishSimpleUpload() {
    ObjectMetadata objectMetadata = new ObjectMetadata();
    objectMetadata.setContentLength(this.currentOutputStream.size());

    byte[] content = this.currentOutputStream.toByteArray();
    try {
        MessageDigest messageDigest = MessageDigest.getInstance("MD5");
        String md5Digest = BinaryUtils.toBase64(messageDigest.digest(content));
        objectMetadata.setContentMD5(md5Digest);
    } catch (NoSuchAlgorithmException e) {
        throw new IllegalStateException("MessageDigest could not be initialized because it uses an unknown algorithm", e);
    }

    SimpleStorageResource.this.amazonS3.putObject(SimpleStorageResource.this.bucketName, SimpleStorageResource.this.objectName,
            new ByteArrayInputStream(content), objectMetadata);

    //Release the memory early
    this.currentOutputStream = null;
}

@Test
public void testWriterWithNewStream() throws Exception
{
    initialize("TestKinesisAppender/testWriterWithNewStream.properties");

    MockKinesisClient mockClient = new MockKinesisClient();

    appender.setThreadFactory(new DefaultThreadFactory());
    appender.setWriterFactory(mockClient.newWriterFactory());

    logger.debug("example message");
    mockClient.allowWriterThread();

    // writer calls describeStream once to see if stream exists, a second time
    // to verify that it's active -- could perhaps combine those calls?

    assertEquals("describeStream: invocation count",        3,          mockClient.describeStreamInvocationCount);
    assertEquals("describeStream: stream name",             "foo",      mockClient.describeStreamStreamName);
    assertEquals("createStream: invocation count",          1,          mockClient.createStreamInvocationCount);
    assertEquals("createStream: stream name",               "foo",      mockClient.createStreamStreamName);
    assertEquals("putRecords: invocation count",            1,          mockClient.putRecordsInvocationCount);
    assertEquals("putRecords: source record count",         1,          mockClient.putRecordsSourceRecords.size());
    assertEquals("putRecords: source record partition key", "bar",      mockClient.putRecordsSourceRecords.get(0).getPartitionKey());
    assertEquals("putRecords: source record content",       "example message\n",
                                                            new String(
                                                                BinaryUtils.copyAllBytesFrom(mockClient.putRecordsSourceRecords.get(0).getData()),
                                                                "UTF-8"));
}

@Test
public void testStaticClientFactory() throws Exception
{
    initialize("TestKinesisAppender/testStaticClientFactory.properties");
    appender.setThreadFactory(new DefaultThreadFactory());
    appender.setWriterFactory(new KinesisWriterFactory());

    // first message triggers writer creation

    logger.debug("example message");
    waitForInitialization();
    AbstractLogWriter writer = (AbstractLogWriter)appender.getWriter();

    assertNotNull("factory was called to create client", staticFactoryMock);
    assertEquals("no initialization errors",             "",
                                                         writer.getInitializationMessage());
    assertEquals("factory method called",                "com.kdgregory.log4j.aws.TestKinesisAppender.createMockClient",
                                                         writer.getClientFactoryUsed());

    // although we should be happy at this point, we'll actually verify that the
    // message got written; assertions copied from testWriterWithExistingStream()

    staticFactoryMock.allowWriterThread();

    assertEquals("describeStream: invocation count",        1,          staticFactoryMock.describeStreamInvocationCount);
    assertEquals("describeStream: stream name",             "argle",    staticFactoryMock.describeStreamStreamName);
    assertEquals("createStream: invocation count",          0,          staticFactoryMock.createStreamInvocationCount);
    assertEquals("putRecords: invocation count",            1,          staticFactoryMock.putRecordsInvocationCount);
    assertEquals("putRecords: source record count",         1,          staticFactoryMock.putRecordsSourceRecords.size());
    assertEquals("putRecords: source record partition key", "bargle",   staticFactoryMock.putRecordsSourceRecords.get(0).getPartitionKey());
    assertEquals("putRecords: source record content",       "example message\n",
                                                            new String(
                                                                BinaryUtils.copyAllBytesFrom(staticFactoryMock.putRecordsSourceRecords.get(0).getData()),
                                                                "UTF-8"));
}

public RetrievedRecord(String shardId, Record record)
throws Exception
{
    this.shardId = shardId;
    this.partitionKey = record.getPartitionKey();
    this.message = new String(BinaryUtils.copyAllBytesFrom(record.getData()), "UTF-8").trim();
}

private String getObjectMD5(byte[] objectData) {
  byte[] expectedMd5 = null;
  try {
    expectedMd5 = Md5Utils.computeMD5Hash(objectData);
  } catch (Exception e) {}
  return BinaryUtils.toHex(expectedMd5);
}

private String getAwsMessageMD5(String message) {
  byte[] expectedMd5 = null;
  try {
    expectedMd5 = Md5Utils.computeMD5Hash(message.getBytes(UTF8));
  } catch (Exception e) {}
  return BinaryUtils.toHex(expectedMd5);
}

/**
 * Binary data should be placed as is, directly into the content.
 */
private void marshallBinaryPayload(Object val) {
    if (val instanceof ByteBuffer) {
        request.setContent(BinaryUtils.toStream((ByteBuffer) val));
    } else if (val instanceof InputStream) {
        request.setContent((InputStream) val);
    }
}

@Override
public StructuredJsonGenerator writeValue(ByteBuffer bytes) {
    try {
        writer.writeBlob(BinaryUtils.copyAllBytesFrom(bytes));
    } catch (IOException e) {
        throw new SdkClientException(e);
    }
    return this;
}

@Override
public StructuredJsonGenerator writeValue(ByteBuffer bytes) {
    try {
        generator.writeBinary(BinaryUtils.copyBytesFrom(bytes));
    } catch (IOException e) {
        throw new JsonGenerationException(e);
    }
    return this;
}

/**
 * Calculate the hash of the request's payload. Subclass could override this
 * method to provide different values for "x-amz-content-sha256" header or
 * do any other necessary set-ups on the request headers. (e.g. aws-chunked
 * uses a pre-defined header value, and needs to change some headers
 * relating to content-encoding and content-length.)
 */
protected String calculateContentHash(SignableRequest<?> request) {
    InputStream payloadStream = getBinaryRequestPayloadStream(request);
    ReadLimitInfo info = request.getReadLimitInfo();
    payloadStream.mark(info == null ? -1 : info.getReadLimit());
    String contentSha256 = BinaryUtils.toHex(hash(payloadStream));
    try {
        payloadStream.reset();
    } catch (IOException e) {
        throw new SdkClientException(
                "Unable to reset stream after calculating AWS4 signature",
                e);
    }
    return contentSha256;
}

private byte[] createSignedChunk(byte[] chunkData) {
    StringBuilder chunkHeader = new StringBuilder();
    // chunk-size
    chunkHeader.append(Integer.toHexString(chunkData.length));
    // sig-extension
    final String chunkStringToSign =
            CHUNK_STRING_TO_SIGN_PREFIX + "\n" +
            dateTime + "\n" +
            keyPath + "\n" +
            priorChunkSignature + "\n" +
            AbstractAWSSigner.EMPTY_STRING_SHA256_HEX + "\n" +
            BinaryUtils.toHex(sha256.digest(chunkData));
    final String chunkSignature =
        BinaryUtils.toHex(aws4Signer.signWithMac(chunkStringToSign, hmacSha256));
    priorChunkSignature = chunkSignature;
    chunkHeader.append(CHUNK_SIGNATURE_HEADER)
               .append(chunkSignature)
               .append(CRLF)
               ;
    try {
        byte[] header = chunkHeader.toString().getBytes(UTF8);
        byte[] trailer = CRLF.getBytes(UTF8);
        byte[] signedChunk = new byte[header.length + chunkData.length + trailer.length];
        System.arraycopy(header, 0, signedChunk, 0, header.length);
        System.arraycopy(chunkData, 0, signedChunk, header.length, chunkData.length);
        System.arraycopy(trailer, 0,
                signedChunk, header.length + chunkData.length,
                trailer.length);
        return signedChunk;
    } catch (Exception e) {
        throw new SdkClientException("Unable to sign the chunked data. " + e.getMessage(), e);
    }
}

@Override
public void setBucketCrossOriginConfiguration(
        SetBucketCrossOriginConfigurationRequest setBucketCrossOriginConfigurationRequest) {
    setBucketCrossOriginConfigurationRequest = beforeClientExecution(setBucketCrossOriginConfigurationRequest);
    rejectNull(setBucketCrossOriginConfigurationRequest,
            "The set bucket cross origin configuration request object must be specified.");

    String bucketName = setBucketCrossOriginConfigurationRequest.getBucketName();
    BucketCrossOriginConfiguration bucketCrossOriginConfiguration = setBucketCrossOriginConfigurationRequest.getCrossOriginConfiguration();

    rejectNull(bucketName,
            "The bucket name parameter must be specified when setting bucket cross origin configuration.");
    rejectNull(bucketCrossOriginConfiguration,
            "The cross origin configuration parameter must be specified when setting bucket cross origin configuration.");

    Request<SetBucketCrossOriginConfigurationRequest> request = createRequest(bucketName, null, setBucketCrossOriginConfigurationRequest, HttpMethodName.PUT);
    request.addParameter("cors", null);

    byte[] content = new BucketConfigurationXmlFactory().convertToXmlByteArray(bucketCrossOriginConfiguration);
    request.addHeader("Content-Length", String.valueOf(content.length));
    request.addHeader("Content-Type", "application/xml");
    request.setContent(new ByteArrayInputStream(content));
    try {
        byte[] md5 = Md5Utils.computeMD5Hash(content);
        String md5Base64 = BinaryUtils.toBase64(md5);
        request.addHeader("Content-MD5", md5Base64);
    } catch ( Exception e ) {
        throw new SdkClientException("Couldn't compute md5 sum", e);
    }

    invoke(request, voidResponseHandler, bucketName, null);
}

@Override
public void setBucketTaggingConfiguration(
        SetBucketTaggingConfigurationRequest setBucketTaggingConfigurationRequest) {
    setBucketTaggingConfigurationRequest = beforeClientExecution(setBucketTaggingConfigurationRequest);
    rejectNull(setBucketTaggingConfigurationRequest,
            "The set bucket tagging configuration request object must be specified.");

    String bucketName = setBucketTaggingConfigurationRequest.getBucketName();
    BucketTaggingConfiguration bucketTaggingConfiguration = setBucketTaggingConfigurationRequest.getTaggingConfiguration();

    rejectNull(bucketName,
            "The bucket name parameter must be specified when setting bucket tagging configuration.");
    rejectNull(bucketTaggingConfiguration,
            "The tagging configuration parameter must be specified when setting bucket tagging configuration.");

    Request<SetBucketTaggingConfigurationRequest> request = createRequest(bucketName, null, setBucketTaggingConfigurationRequest, HttpMethodName.PUT);
    request.addParameter("tagging", null);

    byte[] content = new BucketConfigurationXmlFactory().convertToXmlByteArray(bucketTaggingConfiguration);
    request.addHeader("Content-Length", String.valueOf(content.length));
    request.addHeader("Content-Type", "application/xml");
    request.setContent(new ByteArrayInputStream(content));
    try {
        byte[] md5 = Md5Utils.computeMD5Hash(content);
        String md5Base64 = BinaryUtils.toBase64(md5);
        request.addHeader("Content-MD5", md5Base64);
    } catch ( Exception e ) {
        throw new SdkClientException("Couldn't compute md5 sum", e);
    }

    invoke(request, voidResponseHandler, bucketName, null);
}

private void setContent(Request<?> request, byte[] content, String contentType, boolean setMd5) {
    request.setContent(new ByteArrayInputStream(content));
    request.addHeader("Content-Length", Integer.toString(content.length));
    request.addHeader("Content-Type", contentType);
    if (setMd5) {
        try {
            byte[] md5 = Md5Utils.computeMD5Hash(content);
            String md5Base64 = BinaryUtils.toBase64(md5);
            request.addHeader("Content-MD5", md5Base64);
        } catch ( Exception e ) {
            throw new AmazonClientException("Couldn't compute md5 sum", e);
        }
    }
}

/**
 * If necessary, creates a chunk-encoding wrapper on the request payload.
 */
@Override
protected void processRequestPayload(SignableRequest<?> request, byte[] signature,
        byte[] signingKey, AWS4SignerRequestParams signerRequestParams) {
    if (useChunkEncoding(request)) {
        AwsChunkedEncodingInputStream chunkEncodededStream = new AwsChunkedEncodingInputStream(
                request.getContent(), signingKey,
                signerRequestParams.getFormattedSigningDateTime(),
                signerRequestParams.getScope(),
                BinaryUtils.toHex(signature), this);
        request.setContent(chunkEncodededStream);
    }
}

private void uploadContents(String bucket, String key, byte[] contents)
        throws IOException {
    int failures = 0;
    boolean uploaded = false;
    while (!uploaded) {
        ObjectMetadata objectMetadata = new ObjectMetadata();
        objectMetadata.setContentType(MediaType.TEXT_PLAIN);
        objectMetadata.setContentLength(contents.length);
        objectMetadata.setContentMD5(BinaryUtils.toBase64(Hashing.md5().hashBytes(contents).asBytes()));

        try {
            _amazonS3.putObject(
                    new PutObjectRequest(bucket, key, new ByteArrayInputStream(contents), objectMetadata));
            uploaded = true;
        } catch (AmazonClientException e) {
            if (++failures == MAX_RETRIES) {
                throw new IOException(e);
            }
            try {
                Thread.sleep(_retryDelay.getMillis());
            } catch (InterruptedException e2) {
                // Stop retrying and propagate the original exception
                throw new IOException(e);
            }
        }
    }
}

private String etag(File file) {
    try {
        FileInputStream fileInputStream = new FileInputStream(file);
        byte[] md5Hash = Md5Utils.computeMD5Hash(fileInputStream);
        return BinaryUtils.toHex(md5Hash);
    } catch (IOException e) {
        throw new IllegalStateException(e);
    }
}

@Override
public Integer call() throws Exception {

    // this is the most up to date digest, it's initialized here but later holds the most up to date valid digest
    currentDigest = MessageDigest.getInstance("MD5");
    currentDigest = retryingGet();

    if(progressListener != null) {
        progressListener.progressChanged(new ProgressEvent(ProgressEventType.TRANSFER_STARTED_EVENT));
    }

    if (!fullETag.contains("-")) {
        byte[] expected = BinaryUtils.fromHex(fullETag);
        byte[] current = currentDigest.digest();
        if (!Arrays.equals(expected, current)) {
            throw new AmazonClientException("Unable to verify integrity of data download.  "
                    + "Client calculated content hash didn't match hash calculated by Amazon S3.  "
                    + "The data may be corrupt.");
        }
    } else {
        // TODO log warning that we can't validate the MD5
        // TODO implement this algorithm for several common chunk sizes http://stackoverflow.com/questions/6591047/etag-definition-changed-in-amazon-s3
        if(verbose) {
            System.err.println("\nMD5 does not exist on AWS for file, calculated value: " + BinaryUtils.toHex(currentDigest.digest()));
        }
    }
    // TODO add ability to resume from previously downloaded chunks
    // TODO add rate limiter

    return 0;

}

@Override
public byte[] encrypt(final byte[] plaintext, final byte[] associatedData)
    throws GeneralSecurityException {
  try {
    EncryptRequest req =
        new EncryptRequest().withKeyId(keyArn).withPlaintext(ByteBuffer.wrap(plaintext));
    if (associatedData != null && associatedData.length != 0) {
      req = req.addEncryptionContextEntry("associatedData", BinaryUtils.toHex(associatedData));
    }
    return kmsClient.encrypt(req).getCiphertextBlob().array();
  } catch (AmazonServiceException e) {
    throw new GeneralSecurityException("encryption failed", e);
  }
}

@Override
public byte[] decrypt(final byte[] ciphertext, final byte[] associatedData)
    throws GeneralSecurityException {
  try {
    DecryptRequest req = new DecryptRequest().withCiphertextBlob(ByteBuffer.wrap(ciphertext));
    if (associatedData != null && associatedData.length != 0) {
      req = req.addEncryptionContextEntry("associatedData", BinaryUtils.toHex(associatedData));
    }
    return kmsClient.decrypt(req).getPlaintext().array();
  } catch (AmazonServiceException e) {
    throw new GeneralSecurityException("decryption failed", e);
  }
}

@Test
public void testMessageErrorHandling() throws Exception
{
    initialize("TestKinesisAppender/testMessageErrorHandling.properties");

    // the mock client will report an error on every third record
    MockKinesisClient mockClient = new MockKinesisClient()
    {
        @Override
        public PutRecordsResult putRecords(PutRecordsRequest request)
        {
            int failedRecordCount = 0;
            List<PutRecordsResultEntry> resultRecords = new ArrayList<PutRecordsResultEntry>();
            for (int ii = 0 ; ii < request.getRecords().size() ; ii++)
            {
                PutRecordsResultEntry resultRecord = new PutRecordsResultEntry();
                resultRecords.add(resultRecord);
                if ((ii % 3) == 1)
                {
                    failedRecordCount++;
                    resultRecord.setErrorCode("anything, really");
                }
            }
            return new PutRecordsResult()
                   .withFailedRecordCount(Integer.valueOf(failedRecordCount))
                   .withRecords(resultRecords);
        }
    };

    appender.setThreadFactory(new DefaultThreadFactory());
    appender.setWriterFactory(mockClient.newWriterFactory());

    for (int ii = 0 ; ii < 10 ; ii++)
    {
        logger.debug("message " + ii);
    }

    mockClient.allowWriterThread();

    assertEquals("first batch, number of successful messages", 7, mockClient.putRecordsSuccesses.size());
    assertEquals("first batch, number of failed messages",     3, mockClient.putRecordsFailures.size());

    PutRecordsRequestEntry savedFailure1 = mockClient.putRecordsFailures.get(0);
    PutRecordsRequestEntry savedFailure2 = mockClient.putRecordsFailures.get(1);
    PutRecordsRequestEntry savedFailure3 = mockClient.putRecordsFailures.get(2);

    mockClient.allowWriterThread();

    assertEquals("second batch, number of successful messages", 2, mockClient.putRecordsSuccesses.size());
    assertEquals("second batch, number of failed messages",     1, mockClient.putRecordsFailures.size());

    assertTrue("first failure is now first success",
               Arrays.equals(
                   BinaryUtils.copyAllBytesFrom(savedFailure1.getData()),
                   BinaryUtils.copyAllBytesFrom(mockClient.putRecordsSuccesses.get(0).getData())));
    assertTrue("third failure is now second success (second failure failed again)",
               Arrays.equals(
                   BinaryUtils.copyAllBytesFrom(savedFailure3.getData()),
                   BinaryUtils.copyAllBytesFrom(mockClient.putRecordsSuccesses.get(1).getData())));


    mockClient.allowWriterThread();

    assertEquals("third batch, number of successful messages", 1, mockClient.putRecordsSuccesses.size());
    assertEquals("third batch, number of failed messages",     0, mockClient.putRecordsFailures.size());

    assertTrue("second original failure is now a success",
               Arrays.equals(
                   BinaryUtils.copyAllBytesFrom(savedFailure2.getData()),
                   BinaryUtils.copyAllBytesFrom(mockClient.putRecordsSuccesses.get(0).getData())));
}

@Override
public void presignRequest(SignableRequest<?> request, AWSCredentials credentials,
        Date userSpecifiedExpirationDate) {

    // anonymous credentials, don't sign
    if (isAnonymous(credentials)) {
        return;
    }

    long expirationInSeconds = generateExpirationDate(userSpecifiedExpirationDate);

    addHostHeader(request);

    AWSCredentials sanitizedCredentials = sanitizeCredentials(credentials);
    if (sanitizedCredentials instanceof AWSSessionCredentials) {
        // For SigV4 pre-signing URL, we need to add "X-Amz-Security-Token"
        // as a query string parameter, before constructing the canonical
        // request.
        request.addParameter(X_AMZ_SECURITY_TOKEN,
                ((AWSSessionCredentials) sanitizedCredentials)
                        .getSessionToken());
    }

    final AWS4SignerRequestParams signerRequestParams = new AWS4SignerRequestParams(
            request, overriddenDate, regionName, serviceName,
            AWS4_SIGNING_ALGORITHM);

    // Add the important parameters for v4 signing
    final String timeStamp = signerRequestParams.getFormattedSigningDateTime();

    addPreSignInformationToRequest(request, sanitizedCredentials,
            signerRequestParams, timeStamp, expirationInSeconds);

    final String contentSha256 = calculateContentHashPresign(request);

    final String canonicalRequest = createCanonicalRequest(request,
            contentSha256);

    final String stringToSign = createStringToSign(canonicalRequest,
            signerRequestParams);

    final byte[] signingKey = deriveSigningKey(sanitizedCredentials,
            signerRequestParams);

    final byte[] signature = computeSignature(stringToSign, signingKey,
            signerRequestParams);

    request.addParameter(X_AMZ_SIGNATURE, BinaryUtils.toHex(signature));
}

@Override
public DeleteObjectsResult deleteObjects(DeleteObjectsRequest deleteObjectsRequest) {
    deleteObjectsRequest = beforeClientExecution(deleteObjectsRequest);
    Request<DeleteObjectsRequest> request = createRequest(deleteObjectsRequest.getBucketName(), null, deleteObjectsRequest, HttpMethodName.POST);
    request.addParameter("delete", null);

    if ( deleteObjectsRequest.getMfa() != null ) {
        populateRequestWithMfaDetails(request, deleteObjectsRequest.getMfa());
    }

    populateRequesterPaysHeader(request, deleteObjectsRequest.isRequesterPays());

    byte[] content = new MultiObjectDeleteXmlFactory().convertToXmlByteArray(deleteObjectsRequest);
    request.addHeader("Content-Length", String.valueOf(content.length));
    request.addHeader("Content-Type", "application/xml");
    request.setContent(new ByteArrayInputStream(content));
    try {
        byte[] md5 = Md5Utils.computeMD5Hash(content);
        String md5Base64 = BinaryUtils.toBase64(md5);
        request.addHeader("Content-MD5", md5Base64);
    } catch ( Exception e ) {
        throw new SdkClientException("Couldn't compute md5 sum", e);
    }

    @SuppressWarnings("unchecked")
    ResponseHeaderHandlerChain<DeleteObjectsResponse> responseHandler = new ResponseHeaderHandlerChain<DeleteObjectsResponse>(
            new Unmarshallers.DeleteObjectsResultUnmarshaller(),
            new S3RequesterChargedHeaderHandler<DeleteObjectsResponse>());

    DeleteObjectsResponse response = invoke(request, responseHandler, deleteObjectsRequest.getBucketName(), null);

    /*
     * If the result was only partially successful, throw an exception
     */
    if ( !response.getErrors().isEmpty() ) {
        Map<String, String> headers = responseHandler.getResponseHeaders();

        MultiObjectDeleteException ex = new MultiObjectDeleteException(
                response.getErrors(),
                response.getDeletedObjects());

        ex.setStatusCode(200);
        ex.setRequestId(headers.get(Headers.REQUEST_ID));
        ex.setExtendedRequestId(headers.get(Headers.EXTENDED_REQUEST_ID));
        ex.setCloudFrontId(headers.get(Headers.CLOUD_FRONT_ID));

        throw ex;
    }
    DeleteObjectsResult result = new DeleteObjectsResult(response.getDeletedObjects(), response.isRequesterCharged());

    return result;
}

private UploadPartResult doUploadPart(final String bucketName,
        final String key, final String uploadId, final int partNumber,
        final long partSize, Request<UploadPartRequest> request,
        InputStream inputStream,
        MD5DigestCalculatingInputStream md5DigestStream,
        final ProgressListener listener) {
    try {
        request.setContent(inputStream);
        ObjectMetadata metadata = invoke(request, new S3MetadataResponseHandler(), bucketName, key);
        final String etag = metadata.getETag();

        if (md5DigestStream != null
                && !skipMd5CheckStrategy.skipClientSideValidationPerUploadPartResponse(metadata)) {
            byte[] clientSideHash = md5DigestStream.getMd5Digest();
            byte[] serverSideHash = BinaryUtils.fromHex(etag);

            if (!Arrays.equals(clientSideHash, serverSideHash)) {
                final String info = "bucketName: " + bucketName + ", key: "
                        + key + ", uploadId: " + uploadId
                        + ", partNumber: " + partNumber + ", partSize: "
                        + partSize;
                throw new SdkClientException(
                     "Unable to verify integrity of data upload.  "
                    + "Client calculated content hash (contentMD5: "
                    + Base16.encodeAsString(clientSideHash)
                    + " in hex) didn't match hash (etag: "
                    + etag
                    + " in hex) calculated by Amazon S3.  "
                    + "You may need to delete the data stored in Amazon S3. "
                    + "(" + info + ")");
            }
        }
        publishProgress(listener, ProgressEventType.TRANSFER_PART_COMPLETED_EVENT);
        UploadPartResult result = new UploadPartResult();
        result.setETag(etag);
        result.setPartNumber(partNumber);
        result.setSSEAlgorithm(metadata.getSSEAlgorithm());
        result.setSSECustomerAlgorithm(metadata.getSSECustomerAlgorithm());
        result.setSSECustomerKeyMd5(metadata.getSSECustomerKeyMd5());
        result.setRequesterCharged(metadata.isRequesterCharged());
        return result;
    } catch (Throwable t) {
        publishProgress(listener, ProgressEventType.TRANSFER_PART_FAILED_EVENT);
        // Leaving this here in case anyone is depending on it, but it's
        // inconsistent with other methods which only generate one of
        // COMPLETED_EVENT_CODE or FAILED_EVENT_CODE.
        publishProgress(listener, ProgressEventType.TRANSFER_PART_COMPLETED_EVENT);
        throw failure(t);
    }
}

private String etag(byte[] bytes) {
    byte[] md5Hash = Md5Utils.computeMD5Hash(bytes);
    return BinaryUtils.toHex(md5Hash);
}

@Override
public Integer call() throws Exception {
    ObjectMetadata om = amazonS3Client.getObjectMetadata(bucket, key);
    contentLength = om.getContentLength();

    // this is the most up to date digest, it's initialized here but later holds the most up to date valid digest
    currentDigest = MessageDigest.getInstance("MD5");
    chunkSize = chunkSize == null ? DEFAULT_CHUNK_SIZE : chunkSize;
    fileParts = Parts.among(contentLength, chunkSize);
    for (Part fp : fileParts) {

        /*
         * We'll need to compute the digest on the full incoming stream for
         * each valid chunk that comes in. Invalid chunks will need to be
         * recomputed and fed through a copy of the MD5 that was valid up
         * until the latest chunk.
         */
        currentDigest = retryingGetWithRange(fp.start, fp.end);
    }

    // TODO fix total content length progress bar
    if(progressListener != null) {
        progressListener.progressChanged(new ProgressEvent(ProgressEventType.TRANSFER_STARTED_EVENT));
    }

    String fullETag = om.getETag();
    if (!fullETag.contains("-")) {
        byte[] expected = BinaryUtils.fromHex(fullETag);
        byte[] current = currentDigest.digest();
        if (!Arrays.equals(expected, current)) {
            throw new AmazonClientException("Unable to verify integrity of data download.  "
                    + "Client calculated content hash didn't match hash calculated by Amazon S3.  "
                    + "The data may be corrupt.");
        }
    } else {
        // TODO log warning that we can't validate the MD5
        if(verbose) {
            System.err.println("\nMD5 does not exist on AWS for file, calculated value: " + BinaryUtils.toHex(currentDigest.digest()));
        }
    }
    // TODO add ability to resume from previously downloaded chunks
    // TODO add rate limiter

    return 0;
}

private String getMD5(byte[] bytes) {
    return BinaryUtils.toHex(Md5Utils.computeMD5Hash(bytes));
}

/**
 * Calculates the MD5 hex code of a string (using the util class<br>
 * that the AWS SDK uses for MD5 calculation).
 * 
 * @param message Any text
 * @return Upper-case MD5 checksum in HEX string format
 */
public String getMD5(String message) {
  byte[] expectedMd5 = Md5Utils.computeMD5Hash(message.getBytes(UTF8));
  return BinaryUtils.toHex(expectedMd5);
}