Java 类com.amazonaws.services.s3.model.ObjectListing 实例源码

private List<String> resolveDirectoryResourceNames(ObjectListing objectListing) {
    ImmutableList.Builder<String> builder = ImmutableList.builder();
    if (objectListing.getCommonPrefixes() != null) {
        for (String prefix : objectListing.getCommonPrefixes()) {
            /**
             * The common prefixes will also include the prefix of the <code>ObjectListing</code>
             */
            String directChild = prefix.split(Pattern.quote(objectListing.getPrefix()))[1];
            if (directChild.endsWith("/")) {
                builder.add(directChild.substring(0, directChild.length() - 1));
            } else {
                builder.add(directChild);
            }
        }
        return builder.build();
    }
    return Collections.emptyList();
}

public List<String> listDirectChildren(URI parent) {
    S3RegionalResource s3RegionalResource = new S3RegionalResource(parent);
    String bucketName = s3RegionalResource.getBucketName();
    String s3BucketKey = s3RegionalResource.getKey();
    configureClient(s3RegionalResource);

    ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
        .withBucketName(bucketName)
        .withPrefix(s3BucketKey)
        .withMaxKeys(1000)
        .withDelimiter("/");
    ObjectListing objectListing = amazonS3Client.listObjects(listObjectsRequest);
    ImmutableList.Builder<String> builder = ImmutableList.builder();
    builder.addAll(resourceResolver.resolveResourceNames(objectListing));

    while (objectListing.isTruncated()) {
        objectListing = amazonS3Client.listNextBatchOfObjects(objectListing);
        builder.addAll(resourceResolver.resolveResourceNames(objectListing));
    }
    return builder.build();
}

@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }

    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());

        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}

@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }

    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());

        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}

@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }

    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());

        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}

@Test
public void testEmptyWithVersions() throws Exception
{
    // enable versioning on the bucket
    SetBucketVersioningConfigurationRequest req = new SetBucketVersioningConfigurationRequest(
                bucketName, new BucketVersioningConfiguration(BucketVersioningConfiguration.ENABLED));
    s3.setBucketVersioningConfiguration(req);

    // create some versions
    createVersionedObjects();

    // empty the bucket of it's contents but leave the bucket itself
    S3Cleaner.emptyBucket(s3, bucketName, false, offlineContext);

    // check the bucket is still present but empty
    assertTrue("Bucket '" + bucketName + "' should still be present", s3.doesBucketExistV2(bucketName));
    ObjectListing objects = s3.listObjects(bucketName);
    assertTrue("Bucket '" + bucketName + "' should be empty", objects.getObjectSummaries().isEmpty());
}

@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }

    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());

        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}

@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }

    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());

        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}

@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }

    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());

        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}

@Override
public List<URI> listUris(URI uri, Predicate<URI> uriPredicate) throws IOException {
    String bucketName = uri.getHost();
    if (client == null) {
        client = clientBuilder.client(uri);
    }
    String prefix = uri.getPath().length() > 1 ? uri.getPath().substring(1) : "";
    List<URI> uris = new ArrayList<>();
    ObjectListing list = client.listObjects(bucketName, prefix);
    addKeyUris(uris, list, uri, uriPredicate);
    while (list.isTruncated()) {
        list = client.listNextBatchOfObjects(list);
        addKeyUris(uris, list, uri, uriPredicate);
    }

    return uris;
}

@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest)
        throws SdkClientException, AmazonServiceException {
    listObjectsRequest = beforeClientExecution(listObjectsRequest);
    rejectNull(listObjectsRequest.getBucketName(), "The bucket name parameter must be specified when listing objects in a bucket");

    /**
     * This flag shows whether we need to url decode S3 key names. This flag is enabled
     * only when the customers don't explicitly call {@link ListObjectsRequest#setEncodingType(String)},
     * otherwise, it will be disabled for maintaining backwards compatibility.
     */
    final boolean shouldSDKDecodeResponse = listObjectsRequest.getEncodingType() == null;

    Request<ListObjectsRequest> request = createRequest(listObjectsRequest.getBucketName(), null, listObjectsRequest, HttpMethodName.GET);
    addParameterIfNotNull(request, "prefix", listObjectsRequest.getPrefix());
    addParameterIfNotNull(request, "marker", listObjectsRequest.getMarker());
    addParameterIfNotNull(request, "delimiter", listObjectsRequest.getDelimiter());
    if (listObjectsRequest.getMaxKeys() != null && listObjectsRequest.getMaxKeys().intValue() >= 0) request.addParameter("max-keys", listObjectsRequest.getMaxKeys().toString());
    request.addParameter("encoding-type", shouldSDKDecodeResponse ? Constants.URL_ENCODING : listObjectsRequest.getEncodingType());

    return invoke(request, new Unmarshallers.ListObjectsUnmarshaller(shouldSDKDecodeResponse), listObjectsRequest.getBucketName(), null);
}

@Override
public ObjectListing listNextBatchOfObjects(ListNextBatchOfObjectsRequest listNextBatchOfObjectsRequest)
        throws SdkClientException, AmazonServiceException {
    listNextBatchOfObjectsRequest = beforeClientExecution(listNextBatchOfObjectsRequest);
    rejectNull(listNextBatchOfObjectsRequest,
            "The request object parameter must be specified when listing the next batch of objects in a bucket");
    ObjectListing previousObjectListing = listNextBatchOfObjectsRequest.getPreviousObjectListing();

    if (!previousObjectListing.isTruncated()) {
        ObjectListing emptyListing = new ObjectListing();
        emptyListing.setBucketName(previousObjectListing.getBucketName());
        emptyListing.setDelimiter(previousObjectListing.getDelimiter());
        emptyListing.setMarker(previousObjectListing.getNextMarker());
        emptyListing.setMaxKeys(previousObjectListing.getMaxKeys());
        emptyListing.setPrefix(previousObjectListing.getPrefix());
        emptyListing.setEncodingType(previousObjectListing.getEncodingType());
        emptyListing.setTruncated(false);

        return emptyListing;
    }
    return listObjects(listNextBatchOfObjectsRequest.toListObjectsRequest());
}

/**
 * @see com.amazonaws.http.HttpResponseHandler#handle(com.amazonaws.http.HttpResponse)
 */
public AmazonWebServiceResponse<T> handle(HttpResponse response) throws Exception {
    AmazonWebServiceResponse<T> awsResponse = parseResponseMetadata(response);
    responseHeaders = response.getHeaders();

    if (responseUnmarshaller != null) {
        log.trace("Beginning to parse service response XML");
        T result = responseUnmarshaller.unmarshall(response.getContent());
        log.trace("Done parsing service response XML");
        awsResponse.setResult(result);

        if (result instanceof ObjectListing) {
            if (!StringUtils.isNullOrEmpty(responseHeaders.get(Headers.IBM_SSE_KP_ENABLED))){
                ((ObjectListing) result).setIBMSSEKPEnabled(Boolean.parseBoolean(responseHeaders.get(Headers.IBM_SSE_KP_ENABLED)));
            }
            if (!StringUtils.isNullOrEmpty(responseHeaders.get(Headers.IBM_SSE_KP_CUSTOMER_ROOT_KEY_CRN))) {
                ((ObjectListing) result).setIBMSSEKPCrk(responseHeaders.get(Headers.IBM_SSE_KP_CUSTOMER_ROOT_KEY_CRN));
            } 
        }
    }

    return awsResponse;
}

/**
 * Test the IBM_SSE_KP_ENABLED & IBM_SSE_KP_CRK are set in the ObjectLIsting
 * response object
 * @throws Exception 
 * 
 */ 
@Test
public void testHeadersAddedToObjectListing() throws Exception {

    Unmarshaller<ObjectListing, InputStream> unmarshaller = new Unmarshallers.ListObjectsUnmarshaller(false);
    S3XmlResponseHandler xmlResponseHandler = new S3XmlResponseHandler<ObjectListing>(unmarshaller);
    HttpResponse httpResponse = new HttpResponse(null, null);
    httpResponse.addHeader(Headers.IBM_SSE_KP_ENABLED, "True");
    httpResponse.addHeader(Headers.IBM_SSE_KP_CUSTOMER_ROOT_KEY_CRN, "123456");

    InputStream is = new ByteArrayInputStream(getXmlContent().getBytes());;
    httpResponse.setContent(is);

    AmazonWebServiceResponse<ObjectListing> objectListing = xmlResponseHandler.handle(httpResponse);

    assertEquals(objectListing.getResult().getIBMSSEKPCrk(), "123456");
    assertEquals(objectListing.getResult().getIBMSSEKPEnabled(), true);
}

/**
 * Test the IBM_SSE_KP_ENABLED & IBM_SSE_KP_CRK null headers are handled
 * 
 * @throws Exception 
 * 
 */ 
@Test
public void testNullKPHeadersAreHandled() throws Exception {

    Unmarshaller<ObjectListing, InputStream> unmarshaller = new Unmarshallers.ListObjectsUnmarshaller(false);
    S3XmlResponseHandler xmlResponseHandler = new S3XmlResponseHandler<ObjectListing>(unmarshaller);
    HttpResponse httpResponse = new HttpResponse(null, null);
    httpResponse.addHeader(Headers.IBM_SSE_KP_ENABLED, null);
    httpResponse.addHeader(Headers.IBM_SSE_KP_CRK, null);

    InputStream is = new ByteArrayInputStream(getXmlContent().getBytes());;
    httpResponse.setContent(is);

    AmazonWebServiceResponse<ObjectListing> objectListing = xmlResponseHandler.handle(httpResponse);

    assertEquals(objectListing.getResult().getIBMSSEKPCrk(), null);
    assertEquals(objectListing.getResult().getIBMSSEKPEnabled(), false);
}

/**
 * Test the IBM_SSE_KP_ENABLED & IBM_SSE_KP_CRK empty headers are handled
 * 
 * @throws Exception 
 * 
 */ 
@Test
public void testEmptyKPHeadersAreHandled() throws Exception {

    Unmarshaller<ObjectListing, InputStream> unmarshaller = new Unmarshallers.ListObjectsUnmarshaller(false);
    S3XmlResponseHandler xmlResponseHandler = new S3XmlResponseHandler<ObjectListing>(unmarshaller);
    HttpResponse httpResponse = new HttpResponse(null, null);

    InputStream is = new ByteArrayInputStream(getXmlContent().getBytes());;
    httpResponse.setContent(is);

    AmazonWebServiceResponse<ObjectListing> objectListing = xmlResponseHandler.handle(httpResponse);

    assertEquals(objectListing.getResult().getIBMSSEKPCrk(), null);
    assertEquals(objectListing.getResult().getIBMSSEKPEnabled(), false);
}

/**
 * Test the IBM_SSE_KP_CRK empty header is handled
 * 
 * @throws Exception 
 * 
 */ 
@Test
public void testOnlyKPEnabledHeaderIsSet() throws Exception {

    Unmarshaller<ObjectListing, InputStream> unmarshaller = new Unmarshallers.ListObjectsUnmarshaller(false);
    S3XmlResponseHandler xmlResponseHandler = new S3XmlResponseHandler<ObjectListing>(unmarshaller);
    HttpResponse httpResponse = new HttpResponse(null, null);
    httpResponse.addHeader(Headers.IBM_SSE_KP_ENABLED, "True");

    InputStream is = new ByteArrayInputStream(getXmlContent().getBytes());;
    httpResponse.setContent(is);

    AmazonWebServiceResponse<ObjectListing> objectListing = xmlResponseHandler.handle(httpResponse);

    assertEquals(objectListing.getResult().getIBMSSEKPCrk(), null);
    assertEquals(objectListing.getResult().getIBMSSEKPEnabled(), true);
}

/**
 * Test the IBM_SSE_KP_CRK empty header is handled
 * 
 * @throws Exception 
 * 
 */ 
@Test
public void testOnlyCRKHeaderIsSet() throws Exception {

    Unmarshaller<ObjectListing, InputStream> unmarshaller = new Unmarshallers.ListObjectsUnmarshaller(false);
    S3XmlResponseHandler xmlResponseHandler = new S3XmlResponseHandler<ObjectListing>(unmarshaller);
    HttpResponse httpResponse = new HttpResponse(null, null);
    httpResponse.addHeader(Headers.IBM_SSE_KP_CUSTOMER_ROOT_KEY_CRN, "34567");

    InputStream is = new ByteArrayInputStream(getXmlContent().getBytes());;
    httpResponse.setContent(is);

    AmazonWebServiceResponse<ObjectListing> objectListing = xmlResponseHandler.handle(httpResponse);

    assertEquals(objectListing.getResult().getIBMSSEKPCrk(), "34567");
    assertEquals(objectListing.getResult().getIBMSSEKPEnabled(), false);
}

/**
 * Tests if the list objects can be retrieved.
 */
@Test
public void shouldGetObjectListing() {
  final File uploadFile = new File(UPLOAD_FILE_NAME);
  s3Client.createBucket(BUCKET_NAME);
  s3Client.putObject(new PutObjectRequest(BUCKET_NAME, UPLOAD_FILE_NAME, uploadFile));

  final ObjectListing objectListingResult =
      s3Client.listObjects(BUCKET_NAME, UPLOAD_FILE_NAME);

  assertThat("ObjectListinig has no S3Objects.",
      objectListingResult.getObjectSummaries().size(),
      is(greaterThan(0)));
  assertThat("The Name of the first S3ObjectSummary item has not expected the key name.",
      objectListingResult.getObjectSummaries().get(0).getKey(),
      is(UPLOAD_FILE_NAME));
}

@Override
public void getFileList(String path, OutputStream out) throws Exception {
  String marker = null;
  do {
    ListObjectsRequest request = new ListObjectsRequest(bucketName, path, null, "/", 1000);
    ObjectListing listing = client.listObjects(request);
    for (S3ObjectSummary object : listing.getObjectSummaries()) {
      String line = object.getKey() + "\n";
      out.write(line.getBytes());
    }
    for (String commonPrefix : listing.getCommonPrefixes()) {
      getFileList(commonPrefix, out);
    }
    marker = listing.getNextMarker();
  } while (marker != null);
}

public void scanBucket() {
    ListObjectsRequest listReq = new ListObjectsRequest()
        .withPrefix(opt.prefix())
        .withBucketName(opt.bucket());

    Logger.Info("Scanning S3 bucket %s %s", opt.bucket(), opt.prefix());
    ObjectListing listing = s3.listObjects(listReq);
    boolean ok = processObjects(listing.getObjectSummaries());

    while (ok && listing.isTruncated()) {
        listing = s3.listNextBatchOfObjects(listing);
        ok = processObjects(listing.getObjectSummaries());
    }

    Logger.Info("Completed scan, added %s images to the processing queue.", numSeen.get());
}

@Override
@SuppressWarnings("unused")
public List<AwsFileMiniModel> list(String prefix) {

    AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider());
    List<AwsFileMiniModel> files = new ArrayList();

    ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix);
    ObjectListing objectListing;

    do {
        objectListing = s3client.listObjects(listObjectsRequest);

        for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
            System.out.println(" - " + objectSummary.getKey() + "  " + "(size = " + objectSummary.getSize() + ")"
                    + " (date = " + objectSummary.getLastModified() + ")");
            files.add(new AwsFileMiniModel(objectSummary.getKey(), objectSummary.getLastModified()));
        }
        listObjectsRequest.setMarker(objectListing.getNextMarker());
    } while (objectListing.isTruncated());

    return files;

}

/**
 * Test cleanup deletes everything in the bucket so obviously only use on Pairtree test buckets.
 *
 * @param aContext A test context
 */
@Override
@After
public void tearDown(final TestContext aContext) {
    super.tearDown(aContext);

    // Clean up our test resources in the S3 bucket
    final ObjectListing listing = myS3Client.listObjects(myTestBucket);
    final Iterator<S3ObjectSummary> iterator = listing.getObjectSummaries().iterator();

    while (iterator.hasNext()) {
        try {
            myS3Client.deleteObject(myTestBucket, iterator.next().getKey());
        } catch (final AmazonClientException details) {
            aContext.fail(details);
        }
    }
}

private Answer<ObjectListing> objectListingAnswer(@Nullable final String marker, final String... fileNames) {
    return new Answer<ObjectListing>() {
        @Override
        public ObjectListing answer(InvocationOnMock invocation)
                throws Throwable {
            ListObjectsRequest request = (ListObjectsRequest) invocation.getArguments()[0];

            ObjectListing objectListing = new ObjectListing();
            objectListing.setBucketName(request.getBucketName());
            objectListing.setPrefix(request.getPrefix());

            objectListing.setTruncated(marker != null);
            objectListing.setNextMarker(marker);

            for (String fileName : fileNames) {
                S3ObjectSummary objectSummary = new S3ObjectSummary();
                objectSummary.setKey(request.getPrefix() + fileName);
                objectSummary.setSize(100);
                objectListing.getObjectSummaries().add(objectSummary);
            }

            return objectListing;
        }
    };
}

public static List<S3ObjectSummary> listObjects(String bucket, String prefix, AmazonS3 s3) {
  List<S3ObjectSummary> objects = new ArrayList<>();
  ObjectListing listing;

  try {
    if (prefix == null) {
      listing = s3.listObjects(bucket);
    } else {
      listing = s3.listObjects(bucket, prefix);
    }

    objects.addAll(listing.getObjectSummaries());
    while (listing.isTruncated()) {
      listing = s3.listNextBatchOfObjects(listing);
      objects.addAll(listing.getObjectSummaries());
    }
  } catch (AmazonS3Exception e) {
   log.warn("listObjects for bucket '{}' prefix '{}' returned error code: {}", bucket, prefix, e.getStatusCode());
  }

  return objects;
}

@Override
public void deleteByTenant(final String tenant) {
    final String folder = sanitizeTenant(tenant);

    LOG.info("Deleting S3 object folder (tenant) from bucket {} and key {}", s3Properties.getBucketName(), folder);

    // Delete artifacts
    ObjectListing objects = amazonS3.listObjects(s3Properties.getBucketName(), folder + "/");
    do {
        for (final S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
            amazonS3.deleteObject(s3Properties.getBucketName(), objectSummary.getKey());
        }
        objects = amazonS3.listNextBatchOfObjects(objects);
    } while (objects.isTruncated());

}

public static void main(String[] args)
{
    final String USAGE = "\n" +
        "To run this example, supply the name of a bucket to list!\n" +
        "\n" +
        "Ex: ListObjects <bucket-name>\n";

    if (args.length < 1) {
        System.out.println(USAGE);
        System.exit(1);
    }

    String bucket_name = args[0];

    System.out.format("Objects in S3 bucket %s:\n", bucket_name);
    final AmazonS3 s3 = AmazonS3ClientBuilder.defaultClient();
    ObjectListing ol = s3.listObjects(bucket_name);
    List<S3ObjectSummary> objects = ol.getObjectSummaries();
    for (S3ObjectSummary os: objects) {
        System.out.println("* " + os.getKey());
    }
}

public void deleteAllKeysOnPartialPath(String path) {
    ObjectListing objectListing = s3Client.listObjects(s3Bucket, getFullPath(path));

    if (objectListing.getObjectSummaries().isEmpty()) {
        return;
    }

    List<DeleteObjectsRequest.KeyVersion> keys = objectListing
            .getObjectSummaries()
            .stream()
            .map(objectSummary -> new DeleteObjectsRequest.KeyVersion(objectSummary.getKey()))
            .collect(Collectors.toList());

    DeleteObjectsRequest request = new DeleteObjectsRequest(s3Bucket);
    request.setKeys(keys);
    s3Client.deleteObjects(request);
}

@Test
public void testGetKeysInPartialPath() {
    AmazonS3 client = mock(AmazonS3.class);
    S3StoreService service = new S3StoreService(client, S3_BUCKET, S3_PREFIX);

    String path = "path";
    String key = "my-key";

    S3ObjectSummary summary = new S3ObjectSummary();
    summary.setKey(S3_PREFIX + key);

    ObjectListing listing = mock(ObjectListing.class);

    when(listing.getObjectSummaries()).thenReturn(Lists.newArrayList(summary));
    when(client.listObjects(S3_BUCKET, S3_PREFIX + "/" + path)).thenReturn(listing);

    // invoke method under test
    Set<String> results = service.getKeysInPartialPath(path);

    assertEquals(1, results.size());
    assertEquals(key, results.iterator().next());
}

@Override
public ObjectListing listObjects(ListObjectsRequest request) throws AmazonClientException, AmazonServiceException {
    int currentRequestCount = requestCount.incrementAndGet();

    assertEquals("mycamelbucket", request.getBucketName());
    if (currentRequestCount == 2) {
        assertEquals("confidential", request.getPrefix());
    }

    ObjectListing response = new ObjectListing();
    response.setBucketName(request.getBucketName());
    response.setPrefix(request.getPrefix());

    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setBucketName(request.getBucketName());
    s3ObjectSummary.setKey("key");
    response.getObjectSummaries().add(s3ObjectSummary);

    return response;
}

@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }

    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());

        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}

@Override
public ObjectListing listObjects(ListObjectsRequest request) throws AmazonClientException, AmazonServiceException {
    int currentRequestCount = requestCount.incrementAndGet();

    assertEquals("mycamelbucket", request.getBucketName());
    if (currentRequestCount == 2) {
        assertEquals("confidential", request.getPrefix());
    }

    ObjectListing response = new ObjectListing();
    response.setBucketName(request.getBucketName());
    response.setPrefix(request.getPrefix());

    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setBucketName(request.getBucketName());
    s3ObjectSummary.setKey("key");
    response.getObjectSummaries().add(s3ObjectSummary);

    return response;
}

@Override
protected List<String> getFileNames(String lockPrefix) throws Exception
{
    ListObjectsRequest  request = new ListObjectsRequest();
    request.setBucketName(bucket);
    request.setPrefix(lockPrefix);
    ObjectListing objectListing = client.listObjects(request);

    return Lists.transform
    (
        objectListing.getObjectSummaries(),
        new Function<S3ObjectSummary, String>()
        {
            @Override
            public String apply(S3ObjectSummary summary)
            {
                return summary.getKey();
            }
        }
    );
}

@Test
public void getAllSummaries() {
    answer = true;
    AmazonS3Client client = mock(AmazonS3Client.class);
    ObjectListing objectListing = mock(ObjectListing.class);
    when(client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
    when(objectListing.isTruncated()).thenAnswer(__ -> {
        try {
            return answer;
        } finally {
            answer = false;
        }
    });
    S3Utils utils = new S3Utils(
                                client, null, null, false, null);
    utils.getAllSummaries(new ListObjectsRequest());
    verify(objectListing, times(2)).getObjectSummaries();
}

@Override
public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
  List<NoteInfo> infos = new LinkedList<>();
  NoteInfo info;
  try {
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
            .withBucketName(bucketName)
            .withPrefix(user + "/" + "notebook");
    ObjectListing objectListing;
    do {
      objectListing = s3client.listObjects(listObjectsRequest);
      for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        if (objectSummary.getKey().endsWith("note.json")) {
          info = getNoteInfo(objectSummary.getKey());
          if (info != null) {
            infos.add(info);
          }
        }
      }
      listObjectsRequest.setMarker(objectListing.getNextMarker());
    } while (objectListing.isTruncated());
  } catch (AmazonClientException ace) {
    throw new IOException("Unable to list objects in S3: " + ace, ace);
  }
  return infos;
}

@Override
public void remove(String noteId, AuthenticationInfo subject) throws IOException {
  String key = user + "/" + "notebook" + "/" + noteId;
  final ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
      .withBucketName(bucketName).withPrefix(key);

  try {
    ObjectListing objects = s3client.listObjects(listObjectsRequest);
    do {
      for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
        s3client.deleteObject(bucketName, objectSummary.getKey());
      }
      objects = s3client.listNextBatchOfObjects(objects);
    } while (objects.isTruncated());
  }
  catch (AmazonClientException ace) {
    throw new IOException("Unable to remove note in S3: " + ace, ace);
  }
}

public ArrayList<String> getAllChildren(String folderName) throws IOException {
    ListObjectsRequest listRequest = new ListObjectsRequest();
    listRequest.setBucketName(getBucketName());
    listRequest.setPrefix(folderName);

    ObjectListing listing = s3.listObjects(listRequest);

    ArrayList<String> list = new ArrayList<String>();

    System.out.println(listing.getObjectSummaries().size());

    for (S3ObjectSummary summ : listing.getObjectSummaries()) {
        list.add(summ.getKey());
    }

    return list;
}

public List<String> getAllChildren(String folderName, String bucket) throws IOException {
    ListObjectsRequest listRequest = new ListObjectsRequest();
    listRequest.setBucketName(bucket);

    if (!(folderName == null || folderName.equals(""))) {
        listRequest.setPrefix(folderName);
    }

    ObjectListing listing = s3.listObjects(listRequest);

    ArrayList<String> list = new ArrayList<String>();

    for (S3ObjectSummary summ : listing.getObjectSummaries()) {
        list.add(summ.getKey());
    }

    return list;
}

public List<String> listBucket(String bkt, String prefix, String delimiter) throws IOException {

        ListObjectsRequest listRequest = new ListObjectsRequest();
        listRequest.setBucketName(bkt);
        listRequest.setDelimiter(delimiter);
        listRequest.setPrefix(prefix);

        ObjectListing listing = s3.listObjects(listRequest);

        ArrayList<String> list = new ArrayList<String>();

        for (S3ObjectSummary summ : listing.getObjectSummaries()) {
            list.add(summ.getKey());
        }

        return list;
    }

public static void s3DeleteRecursively(AmazonS3 s3, String bucket, String prefix)
        throws Exception
{
    ListObjectsRequest request = new ListObjectsRequest()
            .withBucketName(bucket)
            .withPrefix(prefix);

    while (true) {
        ObjectListing listing = s3.listObjects(request);
        String[] keys = listing.getObjectSummaries().stream().map(S3ObjectSummary::getKey).toArray(String[]::new);
        for (String key : keys) {
            logger.info("delete s3://{}/{}", bucket, key);
        }
        retryExecutor()
                .retryIf(e -> e instanceof AmazonServiceException)
                .run(() -> s3.deleteObjects(new DeleteObjectsRequest(bucket).withKeys(keys)));
        if (listing.getNextMarker() == null) {
            break;
        }
    }
}