Java 类com.amazonaws.services.s3.model.ListObjectsV2Request 实例源码

private void clearDest(AWSCredentials credentials, RedshiftConnection.UnloadConfig unloadConfig)
{
    try {
        RetryExecutor.retryExecutor() .run(() -> {
            AmazonS3Client s3Client = new AmazonS3Client(credentials);
            ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(unloadConfig.s3Bucket).withPrefix(unloadConfig.s3Prefix);
            ListObjectsV2Result result;
            // This operation shouldn't be skipped since remaining files created by other operation can cause duplicated data
            do {
                result = s3Client.listObjectsV2(req);

                for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
                    s3Client.deleteObject(unloadConfig.s3Bucket, objectSummary.getKey());
                }
                req.setContinuationToken(result.getNextContinuationToken());
            } while (result.isTruncated());
        });
    }
    catch (RetryExecutor.RetryGiveupException e) {
        throw Throwables.propagate(e);
    }
}

@Override
public ListObjectsV2Result listObjectsV2(ListObjectsV2Request listObjectsV2Request)
        throws SdkClientException, AmazonServiceException {
    listObjectsV2Request = beforeClientExecution(listObjectsV2Request);
    rejectNull(listObjectsV2Request.getBucketName(), "The bucket name parameter must be specified when listing objects in a bucket");
    Request<ListObjectsV2Request> request = createRequest(listObjectsV2Request.getBucketName(), null, listObjectsV2Request, HttpMethodName.GET);

    /**
     * List type '2' is required to opt-in to listObjectsV2.
     */
    request.addParameter("list-type", "2");

    addParameterIfNotNull(request, "start-after", listObjectsV2Request.getStartAfter());
    addParameterIfNotNull(request, "continuation-token", listObjectsV2Request.getContinuationToken());
    addParameterIfNotNull(request, "delimiter", listObjectsV2Request.getDelimiter());
    addParameterIfNotNull(request, "max-keys", listObjectsV2Request.getMaxKeys());
    addParameterIfNotNull(request, "prefix", listObjectsV2Request.getPrefix());
    addParameterIfNotNull(request, "encoding-type", listObjectsV2Request.getEncodingType());
    request.addParameter("fetch-owner", Boolean.toString(listObjectsV2Request.isFetchOwner()));

    /**
     * If URL encoding has been requested from S3 we'll automatically decode the response.
     */
    final boolean shouldSDKDecodeResponse = listObjectsV2Request.getEncodingType() == Constants.URL_ENCODING;

    return invoke(request, new Unmarshallers.ListObjectsV2Unmarshaller(shouldSDKDecodeResponse), listObjectsV2Request.getBucketName(), null);
}

@Override public Collection<String> getFilenames() {
    Collection<String> names = new ArrayList<>();
    final ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucket);
    ListObjectsV2Result listing;
    do {
      listing = client.listObjectsV2(req);
      names.addAll(listing.getObjectSummaries().stream().map(S3ObjectSummary::getKey).collect(Collectors.toList()));
    } while (listing.isTruncated());

    return names;
}

@Override
public ListObjectsV2Result listObjectsV2(String bucketName)
        throws SdkClientException, AmazonServiceException {
    return listObjectsV2(new ListObjectsV2Request().withBucketName(bucketName));
}

@Override
public ListObjectsV2Result listObjectsV2(String bucketName, String prefix)
        throws SdkClientException, AmazonServiceException {
    return listObjectsV2(new ListObjectsV2Request().withBucketName(bucketName).withPrefix(prefix));
}

@Override
public ListObjectsV2Result listObjectsV2(ListObjectsV2Request listObjectsV2Request) throws SdkClientException, AmazonServiceException {
  return call(() -> getDelegate().listObjectsV2(listObjectsV2Request));
}

@Override
public Iterator<LogHandle> listAll() {
    final ListObjectsV2Request req = new ListObjectsV2Request()
            .withBucketName(connector.getBucket()).withPrefix(prefix);
    return new Iterator<LogHandle>() {
        ListObjectsV2Result result;
        Iterator<S3ObjectSummary> i;

        @Override
        public boolean hasNext() {
            if (i == null || !i.hasNext()) {
                return loadIfRequired();
            } else {
                return i.hasNext();
            }
        }

        @Override
        public LogHandle next() {
            S3ObjectSummary s = i.next();
            String key = s.getKey().replace(prefix, "");
            if (key.startsWith("-") || key.startsWith("/")) {
                key = key.substring(key.indexOf('/') + 1);
            }
            String[] parts = key.split("/");
            if (parts.length == 4) {
                return new LogHandle(parts[3], fromBase64(parts[0]), parts[1], parts[2]);
            } else {
                return LogHandle.INVALID_LOG;
            }
        }

        private boolean loadIfRequired() {
            if (result != null && (result.isTruncated() || result.getContinuationToken() == null)) {
                return false;
            } else {
                result = client().listObjectsV2(req);
                req.setContinuationToken(result.getNextContinuationToken());
                i = result.getObjectSummaries().iterator();
                return i.hasNext();
            }
        }
    };
}

@Override
public List<IndexKey> getArtifactKeys(String uploadPath) throws StorageException
{
    String prefixPath = getPath() + "/" + uploadPath + "/";
    IStorageTranslator st = getStorageTranslator(uploadPath);

    // Create a list for the index keys and start the timer.
    List<IndexKey> indices = new ArrayList<IndexKey>();

    ListObjectsV2Request lovr = new ListObjectsV2Request();
    lovr.setBucketName(bucketName);
    lovr.setPrefix(prefixPath);

    long start = System.currentTimeMillis();

    for (String delimiter : st.getDelimiters())
    {
        lovr.setDelimiter(delimiter);

        for (String file : client.listObjectsV2(lovr).getCommonPrefixes())
        {
            try
            {
                IndexKey key = st.generateIndexKey(file, prefixPath);

                if (!indices.contains(key))
                {
                    indices.add(key);
                }
            }
            catch (IndexException ie)
            {
                logger.error("Could not create key for " + prefixPath + "/" + file, ie);
            }
        }
    }

    long end = System.currentTimeMillis();
    logger.info(indices.size() + " Artifacts Indexed under " + getPath() + "/" + uploadPath
                    + " in " + (end - start) + "ms");

    return indices;
}

private void assertS3Contents(List<Map<String, Object>> expected)
        throws IOException
{
    ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(s3Bucket).withPrefix(s3ParentKey);
    ListObjectsV2Result result;
    List<String> lines = new ArrayList<>();
    do {
        result = s3Client.listObjectsV2(req);

        for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
            if (objectSummary.getKey().endsWith("_part_00")) {
                try (BufferedReader reader =
                        new BufferedReader(
                                new InputStreamReader(
                                        s3Client.getObject(
                                                objectSummary.getBucketName(),
                                                objectSummary.getKey()).getObjectContent()))) {
                    lines.addAll(reader.lines().collect(Collectors.toList()));
                }
            }
            else {
                assertThat(objectSummary.getKey(), endsWith("_manifest"));
            }

            try {
                s3Client.deleteObject(objectSummary.getBucketName(), objectSummary.getKey());
            }
            catch (Exception e) {
                logger.warn("Failed to delete S3 object: bucket={}, key={}", s3Bucket, objectSummary.getKey(), e);
            }
        }
        req.setContinuationToken(result.getNextContinuationToken());
    } while (result.isTruncated());

    List<ImmutableMap<String, ? extends Serializable>> actual =
            lines.stream()
                    .map(
                            l -> {
                                String[] values = l.split("\\|");
                                assertThat(values.length, is(3));
                                return ImmutableMap.of(
                                        "id", Integer.valueOf(values[0]),
                                        "name", values[1],
                                        "score", Float.valueOf(values[2]));
                            }
                    )
                    .sorted((o1, o2) -> ((Integer)o1.get("id")) - ((Integer) o2.get("id")))
                    .collect(Collectors.toList());
    assertThat(actual, is(expected));
}