Java 类com.amazonaws.util.Base64 实例源码

@Override
public PutObjectResult putObject(PutObjectRequest putObjectRequest)
        throws AmazonClientException, AmazonServiceException {
    String blobName = putObjectRequest.getKey();
    DigestInputStream stream = (DigestInputStream) putObjectRequest.getInputStream();

    if (blobs.containsKey(blobName)) {
        throw new AmazonS3Exception("[" + blobName + "] already exists.");
    }

    blobs.put(blobName, stream);

    // input and output md5 hashes need to match to avoid an exception
    String md5 = Base64.encodeAsString(stream.getMessageDigest().digest());
    PutObjectResult result = new PutObjectResult();
    result.setContentMd5(md5);

    return result;
}

private SecretValue getSecretValue(ToggleGroup valueSource, String value, String generated, File file) {
    Toggle current = valueSource.getSelectedToggle();

    String secretString;
    if (current.getUserData().equals("value")) {
        secretString = value;
    } else if (current.getUserData().equals("generated")) {
        Integer numBytesToGenerate = Integer.valueOf(generated);
        // TODO: store as plain bytes?
        byte[] random = Singleton.randomGenerator.generateRandom(numBytesToGenerate);
        secretString = Base64.encodeAsString(random);
    } else {
        String path = null;
        try {
            path = file.getCanonicalPath();
            return SecretValueConverter.inferEncoding(Files.readAllBytes(Paths.get(path)), SecretType.OPAQUE);
        } catch (IOException e) {
            throw new RuntimeException("Failed to read secret from file");
        }
    }

    return new SecretValue(secretString, SecretType.OPAQUE);
}

/**
 * <p>
 * Populates the specified request with the numerous attributes available in
 * <code>SSEWithCustomerKeyRequest</code>.
 * </p>
 *
 * @param request
 *            The request to populate with headers to represent all the
 *            options expressed in the
 *            <code>ServerSideEncryptionWithCustomerKeyRequest</code>
 *            object.
 * @param sseKey
 *            The request object for an S3 operation that allows server-side
 *            encryption using customer-provided keys.
 */
private static void populateSSE_C(Request<?> request, SSECustomerKey sseKey) {
    if (sseKey == null) return;

    addHeaderIfNotNull(request, Headers.SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM,
            sseKey.getAlgorithm());
    addHeaderIfNotNull(request, Headers.SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY,
            sseKey.getKey());
    addHeaderIfNotNull(request, Headers.SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5,
            sseKey.getMd5());
    // Calculate the MD5 hash of the encryption key and fill it in the
    // header, if the user didn't specify it in the metadata
    if (sseKey.getKey() != null
            && sseKey.getMd5() == null) {
        String encryptionKey_b64 = sseKey.getKey();
        byte[] encryptionKey = Base64.decode(encryptionKey_b64);
        request.addHeader(Headers.SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5,
                Md5Utils.md5AsBase64(encryptionKey));
    }
}

private static void populateSourceSSE_C(Request<?> request, SSECustomerKey sseKey) {
    if (sseKey == null) return;

    // Populate the SSE-C parameters for the source object
    addHeaderIfNotNull(request, Headers.COPY_SOURCE_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM,
            sseKey.getAlgorithm());
    addHeaderIfNotNull(request, Headers.COPY_SOURCE_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY,
            sseKey.getKey());
    addHeaderIfNotNull(request, Headers.COPY_SOURCE_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5,
            sseKey.getMd5());
    // Calculate the MD5 hash of the encryption key and fill it in the
    // header, if the user didn't specify it in the metadata
    if (sseKey.getKey() != null
            && sseKey.getMd5() == null) {
        String encryptionKey_b64 = sseKey.getKey();
        byte[] encryptionKey = Base64.decode(encryptionKey_b64);
        request.addHeader(Headers.COPY_SOURCE_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5,
                Md5Utils.md5AsBase64(encryptionKey));
    }
}

/**
 * Returns the json string in the latest format.
 */
private String toJsonString() {
    Map<String, String> map = new HashMap<String, String>();
    byte[] encryptedCEK = getEncryptedCEK();
    map.put(Headers.CRYPTO_KEY_V2, Base64.encodeAsString(encryptedCEK));
    byte[] iv = cipherLite.getIV();
    map.put(Headers.CRYPTO_IV, Base64.encodeAsString(iv));
    map.put(Headers.MATERIALS_DESCRIPTION, kekMaterialDescAsJson());
    // The CRYPTO_CEK_ALGORITHM, CRYPTO_TAG_LENGTH and
    // CRYPTO_KEYWRAP_ALGORITHM were not available in the Encryption Only
    // (EO) implementation
    ContentCryptoScheme scheme = getContentCryptoScheme();
    map.put(Headers.CRYPTO_CEK_ALGORITHM, scheme.getCipherAlgorithm());
    int tagLen = scheme.getTagLengthInBits();
    if (tagLen > 0)
        map.put(Headers.CRYPTO_TAG_LENGTH, String.valueOf(tagLen));
    String keyWrapAlgo = getKeyWrappingAlgorithm();
    if (keyWrapAlgo != null)
        map.put(Headers.CRYPTO_KEYWRAP_ALGORITHM, keyWrapAlgo);
    return Jackson.toJsonString(map);
}

public static String decrypt(String str, Region region) throws UnsupportedEncodingException {
  if (isJUnitTest()) {
    return str;
  }

  AWSKMS kms = AWSKMSClientBuilder.standard().withRegion(region.getName()).build();

  /*
   * The KMS ciphertext is base64 encoded and must be decoded before the request is made
   */
  String cipherString = str;
  byte[] cipherBytes = Base64.decode(cipherString);

  /*
   * Create decode request and decode
   */
  ByteBuffer cipherBuffer = ByteBuffer.wrap(cipherBytes);
  DecryptRequest req = new DecryptRequest().withCiphertextBlob(cipherBuffer);
  DecryptResult resp = kms.decrypt(req);

  /*
   * Convert the response plaintext bytes to a string
   */
  return new String(resp.getPlaintext().array(), Charset.forName("UTF-8"));
}

/**
* Adds encryption metadata to the StorageObjectMetadata object
*/
@Override
public void addEncryptionMetadata(StorageObjectMetadata meta,
                           MatDesc matDesc,
                           byte[] ivData,
                           byte[] encKeK,
                           long contentLength)
{
   meta.addUserMetadata(getMatdescKey(),
                        matDesc.toString());
   meta.addUserMetadata(AZ_ENCRYPTIONDATAPROP, buildEncryptionMetadataJSON(
                         Base64.encodeAsString(ivData),
                         Base64.encodeAsString(encKeK))
                       );
   meta.setContentLength(contentLength);
}

/**
* Adds encryption metadata to the StorageObjectMetadata object
*/
@Override
public void addEncryptionMetadata(StorageObjectMetadata meta,
                           MatDesc matDesc,
                           byte[] ivData,
                           byte[] encKeK,
                           long contentLength)
{
   meta.addUserMetadata(getMatdescKey(),
                        matDesc.toString());
   meta.addUserMetadata(AMZ_KEY,
                        Base64.encodeAsString(encKeK));
   meta.addUserMetadata(AMZ_IV,
                        Base64.encodeAsString(ivData));
   meta.setContentLength(contentLength);
}

/**
 * Decodes the encrypted token and attempts to decrypt it using AWS KMS. If
 * successful, the token is returned.
 *
 * @param kmsClient      KMS client
 * @param encryptedToken Token to decode and decrypt
 * @return Decrypted token
 */
protected VaultAuthResponse decryptToken(AWSKMS kmsClient, String encryptedToken) {
    byte[] decodedToken;

    try {
        decodedToken = Base64.decode(encryptedToken);
    } catch (IllegalArgumentException iae) {
        throw new VaultClientException("Encrypted token not Base64 encoded", iae);
    }

    final DecryptRequest request = new DecryptRequest().withCiphertextBlob(ByteBuffer.wrap(decodedToken));
    final DecryptResult result = kmsClient.decrypt(request);

    final String decryptedAuthData = new String(result.getPlaintext().array(), Charset.forName("UTF-8"));

    return gson.fromJson(decryptedAuthData, VaultAuthResponse.class);
}

@Test
public void testBrokenYaml() throws Exception{
    // a yaml list is not a valid taupage format. Map is required.
    final String yamlData = "- a\n- b\n- c\n";

    when(amazonEC2ClientMock.describeInstanceAttribute(any())).thenReturn(new DescribeInstanceAttributeResult().
            withInstanceAttribute(new InstanceAttribute()
                    .withUserData(Base64.encodeAsString(yamlData.getBytes()))));

    final FetchTaupageYaml fetchTaupageYaml = new FetchTaupageYamlImpl(clientProviderMock);

    final Optional<TaupageYaml> result = fetchTaupageYaml.getTaupageYaml(INSTANCE_ID, ACCOUNT, REGION);

    assertThat(result).isEmpty();

    verify(amazonEC2ClientMock).describeInstanceAttribute(any());
}

@Test
public void testApplyWithTaupageAmi() throws Exception {
    when(ec2InstanceContextMock.isTaupageAmi()).thenReturn(Optional.of(true));

    when(ec2InstanceContextMock.getInstanceId()).thenReturn(INSTANCE_ID);
    when(ec2InstanceContextMock.getClient(eq(AmazonEC2Client.class))).thenReturn(amazonEC2ClientMock);
    when(amazonEC2ClientMock.describeInstanceAttribute(any())).thenReturn(new DescribeInstanceAttributeResult().
            withInstanceAttribute(new InstanceAttribute()
                    .withUserData(Base64.encodeAsString("blub: fdsa".getBytes()))));

    final Optional<TaupageYaml> result = taupageYamlProvider.apply(ec2InstanceContextMock);

    assertThat(result).isPresent();


    verify(ec2InstanceContextMock).isTaupageAmi();
    verify(ec2InstanceContextMock).getInstanceId();
    verify(ec2InstanceContextMock).getClient(eq(AmazonEC2Client.class));
    verify(amazonEC2ClientMock).describeInstanceAttribute(any());
}

@Test
public void testApplyWithVersionSimilarToNumber() throws Exception {
    when(ec2InstanceContextMock.isTaupageAmi()).thenReturn(Optional.of(true));

    when(ec2InstanceContextMock.getInstanceId()).thenReturn(INSTANCE_ID);
    when(ec2InstanceContextMock.getClient(eq(AmazonEC2Client.class))).thenReturn(amazonEC2ClientMock);
    when(amazonEC2ClientMock.describeInstanceAttribute(any())).thenReturn(new DescribeInstanceAttributeResult().
            withInstanceAttribute(new InstanceAttribute()
                    .withUserData(Base64.encodeAsString("application_id: fdsa\napplication_version: 6478e18".getBytes()))));

    final Optional<TaupageYaml> result = taupageYamlProvider.apply(ec2InstanceContextMock);

    assertThat(result).isPresent();

    assertThat(result.get().getApplicationId()).isEqualTo("fdsa");
    assertThat(result.get().getApplicationVersion()).isEqualTo("6478e18");

    verify(ec2InstanceContextMock).isTaupageAmi();
    verify(ec2InstanceContextMock).getInstanceId();
    verify(ec2InstanceContextMock).getClient(eq(AmazonEC2Client.class));
    verify(amazonEC2ClientMock).describeInstanceAttribute(any());
}

@Test
public void testApplyWithVersionSimilarToNumber1() throws Exception {
    when(ec2InstanceContextMock.isTaupageAmi()).thenReturn(Optional.of(true));

    when(ec2InstanceContextMock.getInstanceId()).thenReturn(INSTANCE_ID);
    when(ec2InstanceContextMock.getClient(eq(AmazonEC2Client.class))).thenReturn(amazonEC2ClientMock);
    when(amazonEC2ClientMock.describeInstanceAttribute(any())).thenReturn(new DescribeInstanceAttributeResult().
            withInstanceAttribute(new InstanceAttribute()
                    .withUserData(Base64.encodeAsString("application_id: fdsa\napplication_version: '6478e18'".getBytes()))));

    final Optional<TaupageYaml> result = taupageYamlProvider.apply(ec2InstanceContextMock);

    assertThat(result).isPresent();

    assertThat(result.get().getApplicationId()).isEqualTo("fdsa");
    assertThat(result.get().getApplicationVersion()).isEqualTo("6478e18");

    verify(ec2InstanceContextMock).isTaupageAmi();
    verify(ec2InstanceContextMock).getInstanceId();
    verify(ec2InstanceContextMock).getClient(eq(AmazonEC2Client.class));
    verify(amazonEC2ClientMock).describeInstanceAttribute(any());
}

@Test
public void testApplyWithTaupageAmiButInvalidYaml() throws Exception {
    // a yaml list is not a valid taupage format. Map is required.
    final String yamlData = "- a\n- b\n- c\n";

    when(ec2InstanceContextMock.isTaupageAmi()).thenReturn(Optional.of(true));

    when(ec2InstanceContextMock.getInstanceId()).thenReturn(INSTANCE_ID);
    when(ec2InstanceContextMock.getClient(eq(AmazonEC2Client.class))).thenReturn(amazonEC2ClientMock);
    when(amazonEC2ClientMock.describeInstanceAttribute(any())).thenReturn(new DescribeInstanceAttributeResult().
            withInstanceAttribute(new InstanceAttribute()
                    .withUserData(Base64.encodeAsString(yamlData.getBytes()))));

    final Optional<TaupageYaml> result = taupageYamlProvider.apply(ec2InstanceContextMock);

    assertThat(result).isEmpty();


    verify(ec2InstanceContextMock).isTaupageAmi();
    verify(ec2InstanceContextMock).getInstanceId();
    verify(ec2InstanceContextMock).getClient(eq(AmazonEC2Client.class));
    verify(amazonEC2ClientMock).describeInstanceAttribute(any());
}

private void dumpTables() {
    for (String table : client.listTables().getTableNames()) {
        ScanResult scanResult;
        Map<String, AttributeValue> lastKey = null;
        do {
            scanResult = client.scan(new ScanRequest().withTableName(table).withExclusiveStartKey(lastKey));
            lastKey = scanResult.getLastEvaluatedKey();
            for (Map<String, AttributeValue> map : scanResult.getItems()) {
                for (Map.Entry<String, AttributeValue> item : map.entrySet()) {
                    System.out.print("item.put(\"");
                    System.out.print(item.getKey());
                    System.out.print("\", b642Av(\"");
                    System.out.print(Base64.encodeAsString(AttributeValueMarshaller.marshall(item.getValue()).array()));
                    System.out.println("\"));");
                }
                System.out.print("ddb.putItem(new PutItemRequest(\"");
                System.out.print(table);
                System.out.println("\", item));");
                System.out.println("item.clear();");
                System.out.println();
            }
        } while (lastKey != null);

    }
}

public static char[] decompressAndB64DecodeUTF8Bytes(byte[] b64EncodedCompressedBytes) throws Exception {

        byte[] input = Base64.decode(b64EncodedCompressedBytes);

        // Compressor with highest level of compression
        Inflater inflater = new Inflater();

        // Give the compressor the data to compress
        inflater.setInput(input);

        ByteArrayOutputStream stream = new ByteArrayOutputStream();
        byte[] buf = new byte[32];
        while (!inflater.finished()) {
            int count = inflater.inflate(buf);
            stream.write(buf, 0, count);
        }
        return new String(stream.toByteArray(),"UTF-8").toCharArray();
    }

private boolean walk(Iterator<S3ObjectSummary> iter, ObjectId file, String path) throws IOException {
    byte[] content;
    byte[] newHash;
    LOG.debug("Start processing file: {}", path);
    try (DigestInputStream is = new DigestInputStream(repository.open(file).openStream(), DigestUtils.getMd5Digest())) {
        // Get content
        content = IOUtils.toByteArray(is);
        // Get hash
        newHash = is.getMessageDigest().digest();
    }
    if (isUploadFile(iter, path, Hex.encodeHexString(newHash))) {
        LOG.info("Uploading file: {}", path);
        ObjectMetadata bucketMetadata = new ObjectMetadata();
        bucketMetadata.setContentMD5(Base64.encodeAsString(newHash));
        bucketMetadata.setContentLength(content.length);
        // Give Tika a few hints for the content detection
        Metadata tikaMetadata = new Metadata();
        tikaMetadata.set(Metadata.RESOURCE_NAME_KEY, FilenameUtils.getName(FilenameUtils.normalize(path)));
        // Fire!
        try (InputStream bis = TikaInputStream.get(content, tikaMetadata)) {
            bucketMetadata.setContentType(TIKA_DETECTOR.detect(bis, tikaMetadata).toString());
            s3.putObject(bucket.getName(), path, bis, bucketMetadata);
            return true;
        }
    }
    LOG.info("Skipping file (same checksum): {}", path);
    return false;
}

private SecretValue extractSecretValueOrThrow(boolean valueFromStdin, String generate, String valueFile) {
    String secretValue = "";
    int sum = booleanIfExists(valueFromStdin) + booleanIfExists(generate) + booleanIfExists(valueFile);

    SecretType secretType = SecretType.OPAQUE;

    if (sum == 0) {
        throw new RuntimeException("You must specify either --value-from-stdin, --value-from-file or --generate-value");
    }
    if (sum > 1) {
        throw new RuntimeException("You must specify one and only one of --value-from-stdin, --value-from-file and --generate-value");
    }

    if (generate != null) {
        secretValue = Base64.encodeAsString(randomGenerator.generateRandom(extractGenerate(generate)));
    }

    if (valueFile != null) {
        return SecretValueConverter.inferEncoding(extractValueFromFile(valueFile), secretType);
    }

    if (valueFromStdin) {
        return SecretValueConverter.inferEncoding(fromStdin(), secretType);
    }

    return new SecretValue(secretValue, secretType);
}

public ByteBuffer unmarshall(JsonUnmarshallerContext unmarshallerContext) throws Exception {
    String base64EncodedString = unmarshallerContext.readText();
    if (base64EncodedString == null) {
        return null;
    }
    byte[] decodedBytes = Base64.decode(base64EncodedString);
    return ByteBuffer.wrap(decodedBytes);

}

/**
 * Computes an RFC 2104-compliant HMAC signature for an array of bytes and
 * returns the result as a Base64 encoded string.
 */
protected String signAndBase64Encode(byte[] data, String key,
        SigningAlgorithm algorithm) throws SdkClientException {
    try {
        byte[] signature = sign(data, key.getBytes(UTF8), algorithm);
        return Base64.encodeAsString(signature);
    } catch (Exception e) {
        throw new SdkClientException(
                "Unable to calculate a request signature: "
                        + e.getMessage(), e);
    }
}

@Test
public void simpleObject_WithBinaryData_WritesAsBase64() throws IOException {
    byte[] data = new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10};
    jsonGenerator.writeStartObject();
    jsonGenerator.writeFieldName("binaryProp").writeValue(ByteBuffer.wrap(data));
    jsonGenerator.writeEndObject();
    JsonNode node = toJsonNode();
    assertEquals(Base64.encodeAsString(data), node.get("binaryProp").textValue());
}

/**
 * Returns the metadata in the latest format.
 */
private ObjectMetadata toObjectMetadata(ObjectMetadata metadata) {
    // If we generated a symmetric key to encrypt the data, store it in the
    // object metadata.
    byte[] encryptedCEK = getEncryptedCEK();
    metadata.addUserMetadata(Headers.CRYPTO_KEY_V2,
            Base64.encodeAsString(encryptedCEK));
    // Put the cipher initialization vector (IV) into the object metadata
    byte[] iv = cipherLite.getIV();
    metadata.addUserMetadata(Headers.CRYPTO_IV, Base64.encodeAsString(iv));
    // Put the materials description into the object metadata as JSON
    metadata.addUserMetadata(Headers.MATERIALS_DESCRIPTION,
            kekMaterialDescAsJson());
    // The CRYPTO_CEK_ALGORITHM, CRYPTO_TAG_LENGTH and
    // CRYPTO_KEYWRAP_ALGORITHM were not available in the Encryption Only
    // (EO) implementation
    ContentCryptoScheme scheme = getContentCryptoScheme();
    metadata.addUserMetadata(Headers.CRYPTO_CEK_ALGORITHM,
            scheme.getCipherAlgorithm());
    int tagLen = scheme.getTagLengthInBits();
    if (tagLen > 0)
        metadata.addUserMetadata(Headers.CRYPTO_TAG_LENGTH,
                String.valueOf(tagLen));
    String keyWrapAlgo = getKeyWrappingAlgorithm();
    if (keyWrapAlgo != null)
        metadata.addUserMetadata(Headers.CRYPTO_KEYWRAP_ALGORITHM,
                keyWrapAlgo);
    return metadata;
}

/**
 * Returns the metadata in backward compatibility (old) format, so it can be
 * read by older version of the AWS SDK.
 */
private ObjectMetadata toObjectMetadataEO(ObjectMetadata metadata) {
    // If we generated a symmetric key to encrypt the data, store it in the
    // object metadata.
    byte[] encryptedCEK = getEncryptedCEK();
    metadata.addUserMetadata(Headers.CRYPTO_KEY,
            Base64.encodeAsString(encryptedCEK));
    // Put the cipher initialization vector (IV) into the object metadata
    byte[] iv = cipherLite.getIV();
    metadata.addUserMetadata(Headers.CRYPTO_IV, Base64.encodeAsString(iv));
    // Put the materials description into the object metadata as JSON
    metadata.addUserMetadata(Headers.MATERIALS_DESCRIPTION,
            kekMaterialDescAsJson());
    return metadata;
}

private String toJsonStringEO() {
    Map<String, String> map = new HashMap<String, String>();
    byte[] encryptedCEK = getEncryptedCEK();
    map.put(Headers.CRYPTO_KEY, Base64.encodeAsString(encryptedCEK));
    byte[] iv = cipherLite.getIV();
    map.put(Headers.CRYPTO_IV, Base64.encodeAsString(iv));
    map.put(Headers.MATERIALS_DESCRIPTION, kekMaterialDescAsJson());
    return Jackson.toJsonString(map);
}

@Override
   public ServiceResponse<User> updateAvatar(User user) {

    User fromStorage = userRepository.findOne(user.getEmail());

       if(!StringUtils.isEmpty(user.getAvatar())
               && user.getAvatar().contains("data:image")) {

        String base64File = user.getAvatar();

        String fileExt = base64File.split(",")[0].split("/")[1].split(";")[0];
        String base64Content = base64File.split(",")[1];

        InputStream is = new ByteArrayInputStream(Base64.decode(base64Content.getBytes()));

        ServiceResponse<InputStream> resizeReponse =  cropAndResizeAvatar(is, fileExt);
        if (!resizeReponse.isOk()) {
               return ServiceResponseBuilder.<User>error()
                       .withMessages(resizeReponse.getResponseMessages())
                       .build();
        }

        is = resizeReponse.getResult();

           ServiceResponse<String> response = uploadService.upload(is, getUniqueFileName(), fileExt, true);
           if(!response.getStatus().equals(ServiceResponse.Status.OK)){
               return ServiceResponseBuilder.<User>error()
                       .withMessages(response.getResponseMessages())
                       .build();
           }
           user.setAvatar(response.getResult());
       } else {
           user.setAvatar(fromStorage.getAvatar());
       }

       return ServiceResponseBuilder.<User>ok()
               .withResult(user)
               .build();

}

@Override
public String downloadAsBase64(String filePath) throws Exception {
    if (!Optional.ofNullable(filePath).isPresent()) {
        throw new Exception(UploadService.Validations.INVALID_PATH.getCode());
    }
    try {
        return Base64.encodeAsString(IOUtils.toByteArray(downloadFile(filePath)));
    } catch (IOException e) {
        throw new Exception(e.getMessage());
    }
}

private void process(final List<Message> messages) {
  if (messages.size() == 0) return;

  final List<DeleteMessageBatchRequestEntry> toDelete = new ArrayList<>();
  int count = 0;
  for (Message message : messages) {
    final String deleteId = String.valueOf(count++);
    try {
      String stringBody = message.getBody();
      if (stringBody.isEmpty()) continue;
      // allow plain-text json, but permit base64 encoded thrift or json
      byte[] spans = stringBody.charAt(0) == '['
          ? stringBody.getBytes(UTF_8)
          : Base64.decode(stringBody);
      collector.acceptSpans(spans, DETECTING_DECODER, new Callback<Void>() {
        @Override public void onSuccess(Void value) {
          toDelete.add(new DeleteMessageBatchRequestEntry(deleteId, message.getReceiptHandle()));
        }

        @Override public void onError(Throwable t) {
          // don't delete messages. this will allow accept calls retry once the
          // messages are marked visible by sqs.
          logger.log(Level.WARNING, "collector accept failed", t);
        }
      });
    } catch (RuntimeException | Error e) {
      logger.log(Level.WARNING, "message decoding failed", e);
      toDelete.add(new DeleteMessageBatchRequestEntry(deleteId, message.getReceiptHandle()));
    }
  }

  delete(toDelete);
}

@Override public Call<Void> sendSpans(List<byte[]> list) {
  if (closeCalled) throw new IllegalStateException("closed");

  byte[] encodedSpans = BytesMessageEncoder.forEncoding(encoding()).encode(list);
  String body = encoding() == Encoding.JSON && isAscii(encodedSpans)
      ? new String(encodedSpans, UTF_8)
      : Base64.encodeAsString(encodedSpans);

  return new SQSCall(new SendMessageRequest(queueUrl(), body));
}

/**
 * Generates secret hash. Uses HMAC SHA256.
 *
 * @param userId       REQUIRED: User ID
 * @param clientId     REQUIRED: Client ID
 * @param clientSecret REQUIRED: Client secret
 * @return secret hash as a {@code String}, {@code null } if {@code clinetSecret if null}
 */
public static String getSecretHash(String userId, String clientId, String clientSecret) {
    // Arguments userId and clientId have to be not-null.
    if (userId == null) {
        throw new CognitoParameterInvalidException("user ID cannot be null");
    }

    if (clientId == null) {
        throw new CognitoParameterInvalidException("client ID cannot be null");
    }

    // Return null as secret hash if clientSecret is null.
    if (clientSecret == null) {
        return null;
    }

    SecretKeySpec signingKey = new SecretKeySpec(clientSecret.getBytes(StringUtils.UTF8),
            HMAC_SHA_256);

    try {
        Mac mac = Mac.getInstance(HMAC_SHA_256);
        mac.init(signingKey);
        mac.update(userId.getBytes(StringUtils.UTF8));
        byte[] rawHmac = mac.doFinal(clientId.getBytes(StringUtils.UTF8));
        return new String(Base64.encode(rawHmac));
    } catch (Exception e) {
        throw new CognitoInternalErrorException("errors in HMAC calculation");
    }
}

private static String convertBase64ToHex(String base64String){
  try{
    byte[] bytes = Base64.decode(base64String);

    final StringBuilder builder = new StringBuilder();
    for(byte b : bytes) {
      builder.append(String.format("%02x", b));
    }
    return builder.toString();
  // return empty string if input is not a valid Base64 string
  }catch(Exception e){
    return "";
  }
}

@Override
public Optional<TaupageYaml> getTaupageYaml(final String instanceId, final String account, final String region) {
    final AmazonEC2Client client = clientProvider.getClient(AmazonEC2Client.class,
            account,
            Region.getRegion(Regions.fromName(region)));

    try {
        final DescribeInstanceAttributeResult response = client.describeInstanceAttribute(
                new DescribeInstanceAttributeRequest()
                        .withInstanceId(instanceId)
                        .withAttribute(USER_DATA));


        return ofNullable(response)
                .map(DescribeInstanceAttributeResult::getInstanceAttribute)
                .map(InstanceAttribute::getUserData)
                .map(Base64::decode)
                .map(String::new)
                .map(TaupageYamlUtil::parseTaupageYaml);

    } catch (final AmazonClientException e) {
        log.warn("Could not get Taupage YAML for instance: " + instanceId, e);
        return empty();
    } catch (YAMLException | IllegalArgumentException s) {
        log.warn("Taupage YAML is not valid for instance: " + instanceId, s);
        return empty();
    }
}

@Test
public void testGetTaupageYaml() throws Exception {
    when(amazonEC2ClientMock.describeInstanceAttribute(any())).thenReturn(new DescribeInstanceAttributeResult().
            withInstanceAttribute(new InstanceAttribute()
                    .withUserData(Base64.encodeAsString("blub: fdsa".getBytes()))));
    final FetchTaupageYaml fetchTaupageYaml = new FetchTaupageYamlImpl(clientProviderMock);

    final Optional<TaupageYaml> result = fetchTaupageYaml.getTaupageYaml(INSTANCE_ID, ACCOUNT, REGION);

    assertThat(result).isPresent();

    verify(amazonEC2ClientMock).describeInstanceAttribute(any());
}

public String writeToS3(final String accountId, final String region, final Date instanceBootTime,
        final String logData, final String logType, final String instanceId) throws IOException {
    String fileName = null;

    final DateTime dateTime = new DateTime(instanceBootTime, UTC);

    final String keyName = Paths.get(
            accountId, region, dateTime.toString("YYYY"), dateTime.toString("MM"),
            dateTime.toString("dd"), instanceId + "-" + dateTime).toString();

    switch (LogType.valueOf(logType)) {

    case USER_DATA:
        fileName = TAUPAGE_YAML;
        break;

    case AUDIT_LOG:
        fileName = AUDIT_LOG_FILE_NAME + new DateTime(UTC) + LOG_GZ;
        break;

    default:
        log.error("Wrong logType given: " + logType);
        break;
    }

    final ObjectMetadata metadata = new ObjectMetadata();
    final byte[] decodedLogData = Base64.decode(logData);
    metadata.setContentLength(decodedLogData.length);

    final InputStream stream = new ByteArrayInputStream(decodedLogData);

    putObjectToS3(bucketName, fileName, keyName, metadata, stream);

    return Paths.get(bucketName, keyName, fileName).toString();
}

private Optional<TaupageYaml> getTaupageYaml(@Nonnull final EC2InstanceContext context) {

        if (context.isTaupageAmi().orElse(false)) {

            final String instanceId = context.getInstanceId();

            try {
                return Optional.of(context.getClient(AmazonEC2Client.class))
                        .map(client -> client.describeInstanceAttribute(new DescribeInstanceAttributeRequest()
                                .withInstanceId(instanceId)
                                .withAttribute(USER_DATA)))
                        .map(DescribeInstanceAttributeResult::getInstanceAttribute)
                        .map(InstanceAttribute::getUserData)
                        .map(Base64::decode)
                        .map(String::new)
                        .map(TaupageYamlUtil::parseTaupageYaml);

            } catch (final AmazonClientException e) {
                log.warn("Could not get Taupage YAML for instance: " + instanceId, e);
                return empty();
            } catch (YAMLException | IllegalArgumentException s)   {
                log.warn("Taupage YAML is not valid for instance: " + instanceId, s);
                return empty();
            }

        } else {
            return empty();
        }

    }

@Test
public void testBasicAuth()
        throws Exception
{
    String uri = "http://localhost:" + httpMockWebServer.getPort() + "/test";
    runWorkflow(folder, "acceptance/http/http.dig", ImmutableMap.of("test_uri", uri), ImmutableMap.of(
            "secrets.http.user", "test-user",
            "secrets.http.password", "test-pass"));
    assertThat(httpMockWebServer.getRequestCount(), is(1));
    RecordedRequest request = httpMockWebServer.takeRequest();

    assertThat(request.getHeader(AUTHORIZATION.asString()), is("Basic " + Base64.encodeAsString("test-user:test-pass".getBytes(UTF_8))));
}

/**
 * Synchronously or asynchronously invokes an AWS Lambda function.
 * If synchronously invoked, the AWS Lambda log is collected and the response payload is returned
 * @param invokeConfig AWS Lambda invocation configuration
 * @return response payload
 */
public String invokeLambdaFunction(InvokeConfig invokeConfig) throws LambdaInvokeException {
    InvokeRequest invokeRequest = new InvokeRequest()
            .withFunctionName(invokeConfig.getFunctionName())
            .withPayload(invokeConfig.getPayload());

    if(invokeConfig.isSynchronous()){
        invokeRequest
                .withInvocationType(InvocationType.RequestResponse)
                .withLogType(LogType.Tail);
    } else {
        invokeRequest
                .withInvocationType(InvocationType.Event);
    }
    logger.log("Lambda invoke request:%n%s%nPayload:%n%s%n", invokeRequest.toString(), invokeConfig.getPayload());

    InvokeResult invokeResult = client.invoke(invokeRequest);
    String payload = "";
    if(invokeResult.getPayload() != null){
        payload = new String(invokeResult.getPayload().array(), Charset.forName("UTF-8"));
    }
    logger.log("Lambda invoke response:%n%s%nPayload:%n%s%n", invokeResult.toString(), payload);

    if(invokeResult.getLogResult() != null){
        logger.log("Log:%n%s%n", new String(Base64.decode(invokeResult.getLogResult()), Charset.forName("UTF-8")));
    }

    if(StringUtils.isNotEmpty(invokeResult.getFunctionError())){
        throw new LambdaInvokeException("Function returned error of type: " + invokeResult.getFunctionError());
    }

    return payload;
}

public List<String> launch(String workerAMI, String instanceType, int num,
        double price, List<String> securityGroups, String keyName, String userData, String charset) throws UnsupportedEncodingException {
    RequestSpotInstancesRequest requestRequest = new RequestSpotInstancesRequest();
    requestRequest.setSpotPrice(Double.toString(price));
    requestRequest.setInstanceCount(Integer.valueOf(num));

    LaunchSpecification launchSpecification = new LaunchSpecification();
    launchSpecification.setImageId(workerAMI);
    launchSpecification.setInstanceType(instanceType);
    launchSpecification.setSecurityGroups(securityGroups);
    launchSpecification.setUserData(new String(Base64.encode(userData.getBytes(charset))));
    launchSpecification.setKeyName(keyName); //for test

    requestRequest.setLaunchSpecification(launchSpecification);
    RequestSpotInstancesResult requestResult = ec2
            .requestSpotInstances(requestRequest);
    List<SpotInstanceRequest> requestResponses = requestResult
            .getSpotInstanceRequests();

    List<String> spotInstanceRequestIds = new ArrayList<String>();

    for (SpotInstanceRequest requestResponse : requestResponses) {
        System.out.println("Created Spot Request: "
                + requestResponse.getSpotInstanceRequestId());
        spotInstanceRequestIds.add(requestResponse
                .getSpotInstanceRequestId());
    }
    return spotInstanceRequestIds;
}

@Override
public DecryptionMaterials getDecryptionMaterials(EncryptionContext context) {
    final Map<String, String> materialDescription = context.getMaterialDescription();

    final Map<String, String> ec = new HashMap<>();
    final String providedEncAlg = materialDescription.get(CONTENT_KEY_ALGORITHM);
    final String providedSigAlg = materialDescription.get(SIGNING_KEY_ALGORITHM);

    ec.put("*" + CONTENT_KEY_ALGORITHM + "*", providedEncAlg);
    ec.put("*" + SIGNING_KEY_ALGORITHM + "*", providedSigAlg);

    populateKmsEcFromEc(context, ec);

    DecryptRequest request = appendUserAgent(new DecryptRequest());
    request.setCiphertextBlob(ByteBuffer.wrap(Base64.decode(materialDescription.get(ENVELOPE_KEY))));
    request.setEncryptionContext(ec);
    final DecryptResult decryptResult = kms.decrypt(request);
    validateEncryptionKeyId(decryptResult.getKeyId(), context);

    final Hkdf kdf;
    try {
        kdf = Hkdf.getInstance(KDF_ALG);
    } catch (NoSuchAlgorithmException e) {
        throw new DynamoDBMappingException(e);
    }
    kdf.init(toArray(decryptResult.getPlaintext()));

    final String[] encAlgParts = providedEncAlg.split("/", 2);
    int encLength = encAlgParts.length == 2 ? Integer.parseInt(encAlgParts[1]) : 256;
    final String[] sigAlgParts = providedSigAlg.split("/", 2);
    int sigLength = sigAlgParts.length == 2 ? Integer.parseInt(sigAlgParts[1]) : 256;

    final SecretKey encryptionKey = new SecretKeySpec(kdf.deriveKey(KDF_ENC_INFO, encLength / 8), encAlgParts[0]);
    final SecretKey macKey = new SecretKeySpec(kdf.deriveKey(KDF_SIG_INFO, sigLength / 8), sigAlgParts[0]);

    return new SymmetricRawMaterials(encryptionKey, macKey, materialDescription);
}

/**
 * Extracts relevant information from {@code context} and uses it to populate fields in
 * {@code kmsEc}. Currently, these fields are:
 * <dl>
 * <dt>{@code HashKeyName}</dt>
 * <dd>{@code HashKeyValue}</dd>
 * <dt>{@code RangeKeyName}</dt>
 * <dd>{@code RangeKeyValue}</dd>
 * <dt>{@link #TABLE_NAME_EC_KEY}</dt>
 * <dd>{@code TableName}</dd>
 */
private static void populateKmsEcFromEc(EncryptionContext context, Map<String, String> kmsEc) {
    final String hashKeyName = context.getHashKeyName();
    if (hashKeyName != null) {
        final AttributeValue hashKey = context.getAttributeValues().get(hashKeyName);
        if (hashKey.getN() != null) {
            kmsEc.put(hashKeyName, hashKey.getN());
        } else if (hashKey.getS() != null) {
            kmsEc.put(hashKeyName, hashKey.getS());
        } else if (hashKey.getB() != null) {
            kmsEc.put(hashKeyName, Base64.encodeAsString(toArray(hashKey.getB())));
        } else {
            throw new UnsupportedOperationException("DirectKmsMaterialProvider only supports String, Number, and Binary HashKeys");
        }
    }
    final String rangeKeyName = context.getRangeKeyName();
    if (rangeKeyName != null) {
        final AttributeValue rangeKey = context.getAttributeValues().get(rangeKeyName);
        if (rangeKey.getN() != null) {
            kmsEc.put(rangeKeyName, rangeKey.getN());
        } else if (rangeKey.getS() != null) {
            kmsEc.put(rangeKeyName, rangeKey.getS());
        } else if (rangeKey.getB() != null) {
            kmsEc.put(rangeKeyName, Base64.encodeAsString(toArray(rangeKey.getB())));
        } else {
            throw new UnsupportedOperationException("DirectKmsMaterialProvider only supports String, Number, and Binary RangeKeys");
        }
    }

    final String tableName = context.getTableName();
    if (tableName != null) {
        kmsEc.put(TABLE_NAME_EC_KEY, tableName);
    }
}

@BeforeClass
public static void setUpBeforeClass() throws Exception {

    KeyGenerator macGen = KeyGenerator.getInstance("HmacSHA256");
    macGen.init(256, Utils.getRng());
    macKey = macGen.generateKey();

    KeyGenerator aesGen = KeyGenerator.getInstance("AES");
    aesGen.init(128, Utils.getRng());
    encryptionKey = aesGen.generateKey();

    keyStore = KeyStore.getInstance("jceks");
    keyStore.load(null, password.toCharArray());

    KeyFactory kf = KeyFactory.getInstance("RSA");
    PKCS8EncodedKeySpec rsaSpec = new PKCS8EncodedKeySpec(Base64.decode(keyPem));
    privateKey = kf.generatePrivate(rsaSpec);
    CertificateFactory cf = CertificateFactory.getInstance("X509");
    certificate = cf.generateCertificate(new ByteArrayInputStream(Base64.decode(certPem)));


    keyStore.setEntry("enc", new SecretKeyEntry(encryptionKey), passwordProtection);
    keyStore.setEntry("sig", new SecretKeyEntry(macKey), passwordProtection);
    keyStore.setEntry("enc-a", new PrivateKeyEntry(privateKey, new Certificate[] {certificate}), passwordProtection);
    keyStore.setEntry("sig-a", new PrivateKeyEntry(privateKey, new Certificate[] {certificate}), passwordProtection);
    keyStore.setCertificateEntry("trustedCert", certificate);
}