Java 类com.amazonaws.services.s3.AmazonS3 实例源码

@Test
void testExecutionExceptionOnClose() throws Exception {
    // test failed close
    final AmazonS3 mockedS3 = mock(AmazonS3.class);
    final S3WritableObjectChannelBuilder builder = defaultBuilder("id")
            .closeExecutorOnChannelClose(false)
            .amazonS3(mockedS3);
    s3channel = (S3AppendableObjectChannel) builder.build();
    when(mockedS3.completeMultipartUpload(any())).thenThrow(new TestException());

    assertThrows(TestException.class, () -> s3channel.close());

    s3channel.close(); // already closed, no effect, should we throw exception?

    while (s3channel.getCancellation() == null) {
        Thread.sleep(10);
    }
    s3channel.getCancellation().get();

    assertTrue(!s3channel.getCancellation().isCompletedExceptionally());
    verify(mockedS3, times(1)).abortMultipartUpload(any());
}

public static String webHookDump(InputStream stream, String school, String extension) {
    if (stream != null) {
        extension = extension == null || extension.isEmpty() ? ".xml" : extension.contains(".") ? extension : "." + extension;
        String fileName = "webhooks/" + school + "/" + school + "_" + Clock.getCurrentDateDashes() + "_" + Clock.getCurrentTime() + extension;
        AmazonS3 s3 = new AmazonS3Client();
        Region region = Region.getRegion(Regions.US_WEST_2);
        s3.setRegion(region);
        try {
            File file = CustomUtilities.inputStreamToFile(stream);
            s3.putObject(new PutObjectRequest(name, fileName, file));
            return CustomUtilities.fileToString(file);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    return "";
}

@Test
public void checkBucketIndex() throws Exception {
  testWithAUser(
      (v) -> {
        String userId = v.getUserId();
        AmazonS3 s3 =
            createS3(
                v.getS3Credentials().get(0).getAccessKey(),
                v.getS3Credentials().get(0).getSecretKey());
        String bucketName = userId.toLowerCase();

        // not exist
        RGW_ADMIN.checkBucketIndex(bucketName, true, true);

        s3.createBucket(bucketName);
        // Do not know how to check the behavior...
        Optional result = RGW_ADMIN.checkBucketIndex(bucketName, true, true);
        assertTrue(result.isPresent());
      });
}

@Test
void testFailedUploadPart() throws Exception {
    final AmazonS3 mocked = mock(AmazonS3.class);
    s3channel = (S3AppendableObjectChannel) defaultBuilder("id")
            .failedPartUploadRetries(3)
            .amazonS3(mocked)
            .build();
    when(mocked.uploadPart(any())).thenThrow(new TestException());

    s3channel.skip(MIN_PART_SIZE).write(ByteBuffer.allocate(123));
    while (s3channel.getCancellation() == null) {
        Thread.sleep(25);
    }
    s3channel.getCancellation().get();
    assertTrue(!s3channel.getCancellation().isCompletedExceptionally());
    assertFalse(s3channel.isOpen());

    //coverage
    s3channel.startWorker(new UploadPartRequest().withPartNumber(1), 0);

    assertThrows(IllegalStateException.class, () -> s3channel.write(ByteBuffer.allocate(1)));

    verify(mocked, times(1)).abortMultipartUpload(any());
}

@Autowired
public StorageService(AmazonS3 s3Client) throws IOException {
    isLocalFileSystem = s3Client == null;

    if (isLocalFileSystem) {
        this.s3Client = null;
        this.buckets = null;
        localFileSystemDirectory =
                new File(String.format("%s/%s", new File("build").getAbsolutePath(), "storage"));
        FileUtils.forceMkdir(localFileSystemDirectory);
    } else {
        this.s3Client = s3Client;
        this.buckets = new ConcurrentHashMap<>();
        this.s3Client.listBuckets().forEach(bucket -> buckets.put(bucket.getName(), bucket));
    }
}

@Test
public void copyCheckTransferManagerIsShutdown() throws Exception {
  client.putObject("source", "data", inputData);
  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();

  TransferManagerFactory mockedTransferManagerFactory = Mockito.mock(TransferManagerFactory.class);
  TransferManager mockedTransferManager = Mockito.mock(TransferManager.class);
  when(mockedTransferManagerFactory.newInstance(any(AmazonS3.class), eq(s3S3CopierOptions)))
      .thenReturn(mockedTransferManager);
  Copy copy = Mockito.mock(Copy.class);
  when(mockedTransferManager.copy(any(CopyObjectRequest.class), any(AmazonS3.class),
      any(TransferStateChangeListener.class))).thenReturn(copy);
  TransferProgress transferProgress = new TransferProgress();
  when(copy.getProgress()).thenReturn(transferProgress);
  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      mockedTransferManagerFactory, listObjectsRequestFactory, registry, s3S3CopierOptions);
  s3s3Copier.copy();
  verify(mockedTransferManager).shutdownNow();
}

@Test
public void copySafelyShutDownTransferManagerWhenNotInitialised() throws Exception {
  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();
  TransferManagerFactory mockedTransferManagerFactory = Mockito.mock(TransferManagerFactory.class);
  when(mockedTransferManagerFactory.newInstance(any(AmazonS3.class), eq(s3S3CopierOptions)))
      .thenThrow(new RuntimeException("error in instance"));
  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      mockedTransferManagerFactory, listObjectsRequestFactory, registry, s3S3CopierOptions);
  try {
    s3s3Copier.copy();
  } catch (RuntimeException e) {
    assertThat(e.getMessage(), is("error in instance"));
  }
}

@Test
public void copyCheckTransferManagerIsShutdownWhenSubmittingJobExceptionsAreThrown() throws Exception {
  client.putObject("source", "data", inputData);
  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();

  TransferManagerFactory mockedTransferManagerFactory = Mockito.mock(TransferManagerFactory.class);
  TransferManager mockedTransferManager = Mockito.mock(TransferManager.class);
  when(mockedTransferManagerFactory.newInstance(any(AmazonS3.class), eq(s3S3CopierOptions)))
      .thenReturn(mockedTransferManager);
  when(mockedTransferManager.copy(any(CopyObjectRequest.class), any(AmazonS3.class),
      any(TransferStateChangeListener.class))).thenThrow(new AmazonServiceException("MyCause"));
  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      mockedTransferManagerFactory, listObjectsRequestFactory, registry, s3S3CopierOptions);
  try {
    s3s3Copier.copy();
    fail("exception should have been thrown");
  } catch (CircusTrainException e) {
    verify(mockedTransferManager).shutdownNow();
    assertThat(e.getCause().getMessage(), startsWith("MyCause"));
  }
}

@Test
public void copyCheckTransferManagerIsShutdownWhenCopyExceptionsAreThrown() throws Exception {
  client.putObject("source", "data", inputData);
  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();

  TransferManagerFactory mockedTransferManagerFactory = Mockito.mock(TransferManagerFactory.class);
  TransferManager mockedTransferManager = Mockito.mock(TransferManager.class);
  when(mockedTransferManagerFactory.newInstance(any(AmazonS3.class), eq(s3S3CopierOptions)))
      .thenReturn(mockedTransferManager);
  Copy copy = Mockito.mock(Copy.class);
  when(copy.getProgress()).thenReturn(new TransferProgress());
  when(mockedTransferManager.copy(any(CopyObjectRequest.class), any(AmazonS3.class),
      any(TransferStateChangeListener.class))).thenReturn(copy);
  doThrow(new AmazonClientException("cause")).when(copy).waitForCompletion();
  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      mockedTransferManagerFactory, listObjectsRequestFactory, registry, s3S3CopierOptions);
  try {
    s3s3Copier.copy();
    fail("exception should have been thrown");
  } catch (CircusTrainException e) {
    verify(mockedTransferManager).shutdownNow();
    assertThat(e.getCause().getMessage(), is("cause"));
  }
}

@NotNull private SamInputResource getS3SamInputResource(BamFile bamFile) {
    final Bucket bucket = bucketManager.loadBucket(bamFile.getBucketId());
    Assert.notNull(bucket, getMessage(MessagesConstants.ERROR_S3_BUCKET));
    final AmazonS3 s3Client = new AmazonS3Client(new BasicAWSCredentials(bucket.getAccessKeyId(),
            bucket.getSecretAccessKey()));
    return SamInputResource.of(s3Client.generatePresignedUrl(bucket.getBucketName(), bamFile.getPath(),
            Utils.getTimeForS3URL()));
}

public Object get(Object[] params) {
    AmazonS3 s3Client = new AmazonS3Client(new ProfileCredentialsProvider());
    try {
        S3Object object = s3Client.getObject(new GetObjectRequest(bucketName, params[0].toString()));
        InputStream objectData = object.getObjectContent();
        byte[] bytes = IOUtils.toByteArray(is);
        ByteBuffer b = ByteBuffer.wrap(bytes);
        return b;
    } catch (AmazonServiceException ase) {
        System.out.println("Caught an AmazonServiceException, which " +
            "means your request made it " +
            "to Amazon S3, but was rejected with an error response" +
            " for some reason.");
        System.out.println("Error Message:" + ase.getMessage());
        System.out.println("HTTP Status Code:" + ase.getStatusCode());
        System.out.println("AWS Error Code:" + ase.getErrorCode());
        System.out.println("Error Type:" + ase.getErrorType());
        System.out.println("Request ID:" + ase.getRequestId());
    } catch (AmazonClientException ace) {
        System.out.println("Caught an AmazonClientException, which " +
            "means the client encountered " +
            "an internal error while trying to " +
            "communicate with S3, " +
            "such as not being able to access the network.");
        System.out.println("Error Message: " + ace.getMessage());
    }

}

/**
 * Creates the S3 client {@link Bean}.
 *
 * Uses the default client, but if a region is unspecified, uses {@code us-east-1}.
 *
 * @return The S3 client.
 */
@Bean
public AmazonS3 s3client() {
    try {
        return AmazonS3ClientBuilder.defaultClient();
    } catch (SdkClientException exception) {
        API_LOG.info("Default S3 client failed to build, trying again with region us-east-1", exception);
        return planB();
    }
}

@Override
public Map<String, BlobMetaData> listBlobsByPrefix(@Nullable String blobNamePrefix) throws IOException {
    return AccessController.doPrivileged((PrivilegedAction<Map<String, BlobMetaData>>) () -> {
        MapBuilder<String, BlobMetaData> blobsBuilder = MapBuilder.newMapBuilder();
        AmazonS3 client = blobStore.client();
        SocketAccess.doPrivilegedVoid(() -> {
            ObjectListing prevListing = null;
            while (true) {
                ObjectListing list;
                if (prevListing != null) {
                    list = client.listNextBatchOfObjects(prevListing);
                } else {
                    if (blobNamePrefix != null) {
                        list = client.listObjects(blobStore.bucket(), buildKey(blobNamePrefix));
                    } else {
                        list = client.listObjects(blobStore.bucket(), keyPath);
                    }
                }
                for (S3ObjectSummary summary : list.getObjectSummaries()) {
                    String name = summary.getKey().substring(keyPath.length());
                    blobsBuilder.put(name, new PlainBlobMetaData(name, summary.getSize()));
                }
                if (list.isTruncated()) {
                    prevListing = list;
                } else {
                    break;
                }
            }
        });
        return blobsBuilder.immutableMap();
    });
}

public CompleteMultipartUpload(String uploadId, AmazonS3 s3,
        PutObjectRequest putObjectRequest, List<Future<PartETag>> futures,
        List<PartETag> eTagsBeforeResume, ProgressListenerChain progressListenerChain,
        UploadMonitor monitor) {
    this.uploadId = uploadId;
    this.s3 = s3;
    this.origReq = putObjectRequest;
    this.futures = futures;
    this.eTagsBeforeResume = eTagsBeforeResume;
    this.listener = progressListenerChain;
    this.monitor = monitor;
}

private AmazonS3 cachedWrapper(AmazonS3 client) {
    TestAmazonS3 wrapper = clients.get(client);
    if (wrapper == null) {
        wrapper = new TestAmazonS3(client, settings);
        clients.put(client, wrapper);
    }
    return wrapper;
}

public CompleteMultipartCopy(String uploadId, AmazonS3 s3,
                             CopyObjectRequest copyObjectRequest, List<Future<PartETag>> futures,
                             ProgressListenerChain progressListenerChain, CopyMonitor monitor) {
    this.uploadId = uploadId;
    this.s3 = s3;
    this.origReq = copyObjectRequest;
    this.futures = futures;
    this.listener = progressListenerChain;
    this.monitor = monitor;
}

public static void storeObject(String contentType,
                               String content,
                               String destBucket,
                               String destKey) throws UnsupportedEncodingException {
    AmazonS3 s3Client = AmazonS3Provider.getS3Client();

    ObjectMetadata metadata = prepareObjectMetadata(contentType, content);

    s3Client.putObject(
            destBucket,
            destKey,
            new StringInputStream(content),
            metadata);
}

public static void storeObject(String contentType,
                               File content,
                               String destBucket,
                               String destKey) {
    AmazonS3 s3Client = AmazonS3Provider.getS3Client();

    ObjectMetadata metadata = prepareObjectMetadata(contentType);
    s3Client.putObject(new PutObjectRequest(destBucket, destKey, content).withMetadata(metadata));
}

@Override
public void run() {
    super.run();

    String awsCredentialsProfile = this.readStringArgument("awsProfile", "default");
    String bucket = this.readStringArgument("bucket");
    String objectKey = this.readStringArgument("objectKey");

    AmazonS3 s3Client = new AmazonS3Client(new ProfileCredentialsProvider(awsCredentialsProfile));
    ObjectMetadata metadata = s3Client.getObjectMetadata(
            new GetObjectMetadataRequest(bucket, objectKey));

    try {
        Date expirationTime = metadata.getExpirationTime();
        if (expirationTime != null) {
            this.writeOutput("expirationTime", metadata.getExpirationTime().getTime());
        } else {
            this.writeOutput("expirationTime", null);
        }
        this.writeOutput("lastModified", metadata.getLastModified().getTime());
        this.writeOutput("userMetadata", metadata.getUserMetadata());
        this.writeOutput("size", metadata.getContentLength());
        this.writeOutput("storageClass", metadata.getStorageClass());
        this.writeOutput("versionId", metadata.getVersionId());
    } catch (Exception ex) {
        throw new RuntimeException(String.format(
                "Failed to get object metadata for object key %s in bucket %s",
                objectKey,
                bucket), ex);
    }
}

public static BufferedReader getReaderFromObjectInfo(ObjectInfoSimple objectInfo) {
    AmazonS3 s3Client = AmazonS3Provider.getS3Client();

    S3Object object = s3Client.getObject(objectInfo.getBucket(), objectInfo.getKey());
    S3ObjectInputStream objectContentRawStream = object.getObjectContent();

    return new BufferedReader(new InputStreamReader(objectContentRawStream));
}

public ManifestWriter(AmazonS3 client, String destBucketName, String destPrefix, String srcBucket,
                      InventoryManifest originalManifest){
    this.s3Client = client;
    this.bucketName = destBucketName;
    String time = this.getTime();
    this.manifestKey = destPrefix + "/" + srcBucket + "/" + time + "/manifest.json";
    this.checksumKey = destPrefix + "/" + srcBucket + "/" + time + "/manifest.checksum";
    this.originalManifest = originalManifest;
}

public InventoryReportLineWriter(AmazonS3 client, String destBucketName, String destPrefix,
                                 String srcBucket, InventoryManifest inventoryManifest) throws IOException{
    this.s3Client = client;
    this.bucketName = destBucketName;
    String uuid = UUID.randomUUID().toString();
    this.outputInventoryReportKey = destPrefix + "/" + srcBucket + "/data/" + uuid + ".csv.gz";
    this.schema = CsvSchemaFactory.buildSchema(inventoryManifest);
}

@Bean
public AmazonS3 amazonS3Client(final AWSCredentialsProvider awsCredentialsProvider, 
    final ClientConfiguration awsClientConfig, final Region awsRegion) {
    return AmazonS3ClientBuilder.standard()
        .withCredentials(awsCredentialsProvider)
        .withClientConfiguration(awsClientConfig)
        .withRegion(awsRegion.getName())
        .build();
}

/**
 * Download an object data as a file
 *
 * @param remoteObjectName the name of object/key which contents should be downloaded
 * @param localFileName the location and file name on the local machine, where the file will be downloaded
 * @throws S3OperationException if there is an error during data transfer
 */
@PublicAtsApi
public void download( String remoteObjectName, String localFileName ) throws S3OperationException,
                                                                      IllegalArgumentException {

    AmazonS3 s3Client = getClient();

    localFileName = IoUtils.normalizeFilePath(localFileName);
    String localDirName = IoUtils.getFilePath(localFileName);
    String localFileOnlyName = IoUtils.getFileName(localFileName);
    File localDir = new File(localDirName);
    if (localDir.exists()) {
        if (localDir.isFile()) {
            throw new IllegalArgumentException("Could not create file " + localFileOnlyName + " into existing file "
                                               + localDirName);
        }
        // else dir exists
    } else {
        LOG.debug("Creating target directory path " + localDirName);
        if (!localDir.mkdirs()) {
            throw new S3OperationException("Could not create local directory path '" + localDirName
                                           + "' for local file specified '" + localFileName + "'");
        }
    }

    S3Object obj = s3Client.getObject(bucketName, remoteObjectName);
    try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(new File(localFileName)));
            S3ObjectInputStream s3is = obj.getObjectContent();) {
        byte[] readBuffArr = new byte[4096];
        int readBytes = 0;
        while ( (readBytes = s3is.read(readBuffArr)) >= 0) {
            bos.write(readBuffArr, 0, readBytes);
        }
    } catch (Exception e) {
        handleExeption(e, "Error while downloading object " + remoteObjectName + " to local file " + localFileName
                          + ". If error persists check your endpoint, credentials and permissions.");
    }
    LOG.info("S3 object '" + remoteObjectName + "; is downloaded successfully from bucket '" + bucketName
             + "' to file " + localFileName);
}

protected AmazonS3 initClient(@Nullable String accessKey, @Nullable String secretKey) throws IOException {
    if (accessKey == null || secretKey == null) {
        return new AmazonS3Client(DEFAULT_CREDENTIALS_PROVIDER_CHAIN, CLIENT_CONFIGURATION);
    }
    return new AmazonS3Client(
            new BasicAWSCredentials(accessKey, secretKey),
            CLIENT_CONFIGURATION
    );
}

@Test
@Ignore("Not a test")
public void subuserWithS3CredentialIncorporated() throws Exception {
  testWithUserAndS3(
      (user, s3) -> {
        createSomeObjects(s3);

        List<SubUser> subUser =
            RGW_ADMIN.createSubUser(
                user.getUserId(), "QQQ", ImmutableMap.of("key-type", "s3", "access", "full"));

        User userInfo = RGW_ADMIN.getUserInfo(user.getUserId()).get();

        S3Credential subUserKey =
            userInfo
                .getS3Credentials()
                .stream()
                .filter(v -> v.getUserId().equals(subUser.get(0).getId()))
                .findFirst()
                .get();

        AmazonS3 subUserS3 = createS3(subUserKey.getAccessKey(), subUserKey.getSecretKey());

        createSomeObjects(subUserS3);

        // The S3 bucket created by parent user and created by child subuser are incorporated.
        assertEquals(s3.listBuckets().size(), subUserS3.listBuckets().size());

        for (String bucketName :
            s3.listBuckets().stream().map(v -> v.getName()).collect(Collectors.toList())) {
          assertEquals(
              s3.listObjects(bucketName).getObjectSummaries().toString(),
              subUserS3.listObjects(bucketName).getObjectSummaries().toString());
          subUserS3.getBucketAcl(bucketName);
          subUserS3.setBucketAcl(bucketName, CannedAccessControlList.AuthenticatedRead);
        }
      });
}

@Test
public void removeBucket() throws Exception {
  String bucketName = "testremovebkbk" + UUID.randomUUID().toString();

  // remove bucket not exist
  Thread.sleep(3000);
  RGW_ADMIN.removeBucket(bucketName);

  testWithAUser(
      v -> {
        String userId = "testremovebk" + UUID.randomUUID().toString();

        User response = RGW_ADMIN.createUser(userId);
        AmazonS3 s3 =
            createS3(
                response.getS3Credentials().get(0).getAccessKey(),
                response.getS3Credentials().get(0).getSecretKey());
        s3.createBucket(bucketName);

        ByteArrayInputStream input = new ByteArrayInputStream("Hello World!".getBytes());
        s3.putObject(bucketName, "hello.txt", input, new ObjectMetadata());

        RGW_ADMIN.removeBucket(bucketName);

        try {
          s3.headBucket(new HeadBucketRequest(bucketName));
          fail();
        } catch (Exception e) {
          assertTrue("Not Found".equals(((AmazonS3Exception) e).getErrorMessage()));
        }
      });
}

@Test
public void itDoesntCount404AsFailure() throws InterruptedException {
  AmazonS3 s3 = FailsafeS3Decorator.decorate(new MissingS3Client());

  for (int i = 0; i < 100; i++ ) {
    try {
      s3.getObjectMetadata("test-bucket", "test-key");
    } catch (AmazonServiceException e) {
      assertThat(e.getStatusCode()).isEqualTo(404);
    }

    Thread.sleep(50);
  }
}

@Test
public void listBucket() throws Exception {
  testWithASubUser(
      v -> {
        AmazonS3 s3 =
            createS3(
                v.getS3Credentials().get(0).getAccessKey(),
                v.getS3Credentials().get(0).getSecretKey());
        for (int i = 0; i < 3; i++) {
          s3.createBucket(UUID.randomUUID().toString().toLowerCase());
        }
        List<String> response = RGW_ADMIN.listBucket(v.getUserId());
        assertEquals(3, response.size());
      });
}

@Test
public void userQuotaMaxObjects() throws Exception {
  testWithAUser(
      (v) -> {
        String userId = v.getUserId();
        Quota quota;

        // max object = 2
        RGW_ADMIN.setUserQuota(userId, 2, -1);
        quota = RGW_ADMIN.getUserQuota(userId).get();
        assertEquals(true, quota.getEnabled());

        AmazonS3 s3 =
            createS3(
                v.getS3Credentials().get(0).getAccessKey(),
                v.getS3Credentials().get(0).getSecretKey());
        String bucketName = userId.toLowerCase();
        s3.createBucket(bucketName);

        // allow 1st,2ed obj
        s3.putObject(bucketName, userId + "1", "qqqq");
        s3.putObject(bucketName, userId + "2", "qqqq");

        // deny 3rd obj
        try {
          s3.putObject(bucketName, userId + "3", "qqqq");
          fail();
        } catch (AmazonS3Exception e) {
          assertEquals("QuotaExceeded", e.getErrorCode());
        }
      });
}

@Test
public void userQuotaMaxSize() throws Exception {
  testWithAUser(
      (v) -> {
        String userId = v.getUserId();
        Quota quota;

        // max size = 6 bytes
        RGW_ADMIN.setUserQuota(userId, -1, 12);
        quota = RGW_ADMIN.getUserQuota(userId).get();
        assertEquals(true, quota.getEnabled());

        AmazonS3 s3 =
            createS3(
                v.getS3Credentials().get(0).getAccessKey(),
                v.getS3Credentials().get(0).getSecretKey());
        String bucketName = userId.toLowerCase();
        s3.createBucket(bucketName);

        // ok, ok, ok, since the total to used size exceed 12KiB
        s3.putObject(bucketName, userId + "1", createString(4096));
        s3.putObject(bucketName, userId + "2", createString(4096));
        s3.putObject(bucketName, userId + "3", createString(4096));

        // not ok, since the total to used size exceed 12KiB +1
        try {
          s3.putObject(bucketName, userId + "4", createString(1));
          fail();
        } catch (AmazonS3Exception e) {
          assertEquals("QuotaExceeded", e.getErrorCode());
        }
      });
}

@Test
public void removeObject() throws Exception {
  testWithAUser(
      (v) -> {
        String userId = v.getUserId();
        AmazonS3 s3 =
            createS3(
                v.getS3Credentials().get(0).getAccessKey(),
                v.getS3Credentials().get(0).getSecretKey());
        String bucketName = userId.toLowerCase();
        s3.createBucket(bucketName);
        String objectKey = userId.toLowerCase();
        s3.putObject(bucketName, objectKey, "qqq");

        // basic
        RGW_ADMIN.removeObject(bucketName, objectKey);
        try {
          s3.getObjectMetadata(bucketName, objectKey);
          fail();
        } catch (AmazonS3Exception e) {
          assertEquals(404, e.getStatusCode());
        }

        // not exist
        RGW_ADMIN.removeObject(bucketName, objectKey);
      });
}

protected static void createSomeObjects(AmazonS3 s3) {
  String bucketName = "bucket-" + UUID.randomUUID().toString().toLowerCase();
  s3.createBucket(bucketName);
  for (int i = 0; i < 3; i++) {

    s3.putObject(bucketName, "OBJECT-" + UUID.randomUUID(), createString(4096));
  }
  // Usage data are generated in the async way, hope it will be available after wait.
  try {
    Thread.sleep(5000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }
}

@Test
public void itDoesntCount404AsFailure() throws InterruptedException {
  AmazonS3 s3 = HystrixS3Decorator.decorate(new MissingS3Client());

  for (int i = 0; i < 100; i++ ) {
    try {
      s3.getObjectMetadata("test-bucket", "test-key");
    } catch (AmazonServiceException e) {
      assertThat(e.getStatusCode()).isEqualTo(404);
    }

    Thread.sleep(50);
  }
}

protected static AmazonS3 createS3(String accessKey, String secretKey) {
  AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
  ClientConfiguration clientConfig = new ClientConfiguration();
  clientConfig.setProtocol(Protocol.HTTP);
  clientConfig.withSignerOverride("S3SignerType");
  //noinspection deprecation
  AmazonS3 s3 = new AmazonS3Client(credentials, clientConfig);
  s3.setEndpoint(s3Endpoint);
  return s3;
}

protected S3WritableObjectChannel(String key, String bucket, String uploadId, int partSize, AmazonS3 s3,
                                  ExecutorService executor, boolean closeExecutorOnClose, int failedPartUploadRetries) {
    this.key = key;
    this.bucket = bucket;
    this.uploadId = uploadId;
    this.partSize = partSize;
    this.s3 = s3;
    this.executor = executor;
    this.closeExecutorOnClose = closeExecutorOnClose;
    this.failedPartUploadRetries = failedPartUploadRetries;
}

/**
 * Gets an Amazon S3 client from basic session credentials.
 *
 * @return an authenticated Amazon S3 amazonS3
 */
public AmazonS3 getAmazonS3Client() {
    if (amazonS3 == null) {
        amazonS3 = AmazonS3ClientBuilder.standard()
            .withEndpointConfiguration(new EndpointConfiguration(endpoint, region))
            .withClientConfiguration(new ClientConfiguration().withProtocol(Protocol.HTTP))
            .withCredentials(
                new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKeyId, accessKeySecret)))
            .build();
    }
    return amazonS3;
}

@Override
public AmazonS3 newInstance(AmazonS3URI uri, S3S3CopierOptions s3s3CopierOptions) {
  LOG.debug("trying to get a client for uri '{}'", uri);
  AmazonS3 globalClient = newGlobalInstance(s3s3CopierOptions);
  try {
    String bucketRegion = regionForUri(globalClient, uri);
    LOG.debug("Bucket region: {}", bucketRegion);
    return newInstance(bucketRegion, s3s3CopierOptions);
  } catch (IllegalArgumentException e) {
    LOG.warn("Using global (non region specific) client", e);
    return globalClient;
  }
}

public RepositoryS3(Bucket bucket, Repository repository, AmazonS3 s3, Branch branch) {
    this.s3 = s3;
    this.bucket = bucket;
    this.repository = repository;
    this.branch = branch;
    this.uri = new URIish().setScheme("amazon-s3").setHost(bucket.getName()).setPath(Constants.DOT_GIT);
}

private AmazonS3 newInstance(String region, S3S3CopierOptions s3s3CopierOptions) {
  HadoopAWSCredentialProviderChain credentialsChain = getCredentialsProviderChain();
  AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard().withCredentials(credentialsChain);
  URI s3Endpoint = s3s3CopierOptions.getS3Endpoint(region);
  if (s3Endpoint != null) {
    EndpointConfiguration endpointConfiguration = new EndpointConfiguration(s3Endpoint.toString(), region);
    builder.withEndpointConfiguration(endpointConfiguration);
  } else {
    builder.withRegion(region);
  }
  return builder.build();
}