Java 类com.amazonaws.services.s3.transfer.TransferProgress 实例源码

@Test
public void copyCheckTransferManagerIsShutdown() throws Exception {
  client.putObject("source", "data", inputData);
  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();

  TransferManagerFactory mockedTransferManagerFactory = Mockito.mock(TransferManagerFactory.class);
  TransferManager mockedTransferManager = Mockito.mock(TransferManager.class);
  when(mockedTransferManagerFactory.newInstance(any(AmazonS3.class), eq(s3S3CopierOptions)))
      .thenReturn(mockedTransferManager);
  Copy copy = Mockito.mock(Copy.class);
  when(mockedTransferManager.copy(any(CopyObjectRequest.class), any(AmazonS3.class),
      any(TransferStateChangeListener.class))).thenReturn(copy);
  TransferProgress transferProgress = new TransferProgress();
  when(copy.getProgress()).thenReturn(transferProgress);
  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      mockedTransferManagerFactory, listObjectsRequestFactory, registry, s3S3CopierOptions);
  s3s3Copier.copy();
  verify(mockedTransferManager).shutdownNow();
}

@Test
public void copyCheckTransferManagerIsShutdownWhenCopyExceptionsAreThrown() throws Exception {
  client.putObject("source", "data", inputData);
  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();

  TransferManagerFactory mockedTransferManagerFactory = Mockito.mock(TransferManagerFactory.class);
  TransferManager mockedTransferManager = Mockito.mock(TransferManager.class);
  when(mockedTransferManagerFactory.newInstance(any(AmazonS3.class), eq(s3S3CopierOptions)))
      .thenReturn(mockedTransferManager);
  Copy copy = Mockito.mock(Copy.class);
  when(copy.getProgress()).thenReturn(new TransferProgress());
  when(mockedTransferManager.copy(any(CopyObjectRequest.class), any(AmazonS3.class),
      any(TransferStateChangeListener.class))).thenReturn(copy);
  doThrow(new AmazonClientException("cause")).when(copy).waitForCompletion();
  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      mockedTransferManagerFactory, listObjectsRequestFactory, registry, s3S3CopierOptions);
  try {
    s3s3Copier.copy();
    fail("exception should have been thrown");
  } catch (CircusTrainException e) {
    verify(mockedTransferManager).shutdownNow();
    assertThat(e.getCause().getMessage(), is("cause"));
  }
}

public static void showTransferProgress(Transfer xfer)
{
    // print the transfer's human-readable description
    System.out.println(xfer.getDescription());
    // print an empty progress bar...
    printProgressBar(0.0);
    // update the progress bar while the xfer is ongoing.
    do {
        try {
            Thread.sleep(100);
        } catch (InterruptedException e) {
            return;
        }
        // Note: so_far and total aren't used, they're just for
        // documentation purposes.
        TransferProgress progress = xfer.getProgress();
        long so_far = progress.getBytesTransferred();
        long total = progress.getTotalBytesToTransfer();
        double pct = progress.getPercentTransferred();
        eraseProgressBar();
        printProgressBar(pct);
    } while (xfer.isDone() == false);
    // print the final state of the transfer.
    TransferState xfer_state = xfer.getState();
    System.out.println(": " + xfer_state);
}

AbstractTransfer(String description, TransferProgress transferProgress,
        ProgressListenerChain progressListenerChain, TransferStateChangeListener stateChangeListener) {
    this.description = description;
    this.listenerChain = progressListenerChain;
    this.transferProgress = transferProgress;
    addStateChangeListener(stateChangeListener);
}

@Deprecated
public DownloadImpl(String description, TransferProgress transferProgress,
        ProgressListenerChain progressListenerChain, S3Object s3Object, TransferStateChangeListener listener,
        GetObjectRequest getObjectRequest, File file) {
    this(description, transferProgress, progressListenerChain, s3Object, listener,
            getObjectRequest, file, null, false);
}

public DownloadImpl(String description, TransferProgress transferProgress,
        ProgressListenerChain progressListenerChain, S3Object s3Object, TransferStateChangeListener listener,
        GetObjectRequest getObjectRequest, File file,
        ObjectMetadata objectMetadata, boolean isDownloadParallel) {
    super(description, transferProgress, progressListenerChain, listener);
    this.s3Object = s3Object;
    this.objectMetadata = objectMetadata;
    this.getObjectRequest = getObjectRequest;
    this.file = file;
    this.progressListenerChain = progressListenerChain;
    this.persistableDownload = captureDownloadState(getObjectRequest, file);
    S3ProgressPublisher.publishTransferPersistable(progressListenerChain, persistableDownload);
}

public UploadCallable(TransferManager transferManager,
        ExecutorService threadPool, UploadImpl upload,
        PutObjectRequest origReq,
        ProgressListenerChain progressListenerChain, String uploadId,
        TransferProgress transferProgress) {
    this.s3 = transferManager.getAmazonS3Client();
    this.configuration = transferManager.getConfiguration();

    this.threadPool = threadPool;
    this.origReq = origReq;
    this.listener = progressListenerChain;
    this.upload = upload;
    this.multipartUploadId = uploadId;
    this.transferProgress = transferProgress;
}

/**
 * Logs transfer progress for an S3 file transfer operation.
 *
 * @param transferProgress the progress of an S3 transfer operation
 */
private void logTransferProgress(TransferProgress transferProgress)
{
    // If the total bytes to transfer is set to 0, we do not log the transfer progress.
    if (LOGGER.isInfoEnabled() && transferProgress.getTotalBytesToTransfer() > 0)
    {
        NumberFormat formatter = new DecimalFormat("#0.0");

        LOGGER.info("progressBytesTransferred={} totalBytesToTransfer={} progressPercentTransferred={}", transferProgress.getBytesTransferred(),
            transferProgress.getTotalBytesToTransfer(), formatter.format(transferProgress.getPercentTransferred()));
    }
}

public MessageDigest retryingGet()
        throws ExecutionException, RetryException {

    return (MessageDigest) RetryUtils.AWS_RETRYER.call(new Callable<Object>() {
        public MessageDigest call() throws Exception {

            GetObjectRequest req = new GetObjectRequest(bucket, key);

            S3Object s3Object = amazonS3Client.getObject(req);
            contentLength = s3Object.getObjectMetadata().getContentLength();
            fullETag = s3Object.getObjectMetadata().getETag();

            Progress progress = new TransferProgressWrapper(new TransferProgress());
            progress.setTotalBytesToTransfer(contentLength);
            if (progressListener != null) {
                progressListener.withTransferProgress(progress)
                        .withCompleted(0.0)
                        .withMultiplier(1.0);
            }

            InputStream input = null;
            try {
                // create the output file, now that we know it actually exists
                if(output == null) {
                    output = new RandomAccessFile(outputFile, "rw");
                }

                // seek to the start of the chunk in the file, just in case we're retrying
                output.seek(0);
                input = s3Object.getObjectContent();

                return copyAndHash(input, contentLength, progress);
            } finally {
                IOUtils.closeQuietly(input);
            }
        }
    });
}

public MessageDigest retryingGetWithRange(final long start, final long end)
        throws ExecutionException, RetryException {

    return (MessageDigest) RetryUtils.AWS_RETRYER.call(new Callable<Object>() {
        public MessageDigest call() throws Exception {

            long totalBytes = end - start + 1;
            Progress progress = new TransferProgressWrapper(new TransferProgress());
            progress.setTotalBytesToTransfer(totalBytes);

            if (progressListener != null) {
                progressListener.withTransferProgress(progress)
                        .withCompleted((100.0 * start) / contentLength)
                        .withMultiplier((1.0 * totalBytes / (Math.min(contentLength, chunkSize))) / fileParts.size());
            }

            GetObjectRequest req = new GetObjectRequest(bucket, key)
                    .withRange(start, end);

            S3Object s3Object = amazonS3Client.getObject(req);
            InputStream input = null;
            try {
                // create the output file, now that we know it actually exists
                if(output == null) {
                    output = new RandomAccessFile(outputFile, "rw");
                }

                // seek to the start of the chunk in the file, just in case we're retrying
                output.seek(start);
                input = s3Object.getObjectContent();

                return copyAndHash(input, totalBytes, progress);
            } finally {
                IOUtils.closeQuietly(input);
            }
        }
    });
}

public UploadImpl(String description, TransferProgress transferProgressInternalState,
        ProgressListenerChain progressListenerChain, TransferStateChangeListener listener) {
    super(description, transferProgressInternalState, progressListenerChain, listener);
}

public MultipleFileUploadImpl(String description, TransferProgress transferProgress,
        ProgressListenerChain progressListenerChain, String keyPrefix, String bucketName, Collection<? extends Upload> subTransfers) {
    super(description, transferProgress, progressListenerChain, subTransfers);
    this.keyPrefix = keyPrefix;
    this.bucketName = bucketName;
}

MultipleFileTransfer(String description, TransferProgress transferProgress,
        ProgressListenerChain progressListenerChain, Collection<? extends T> subTransfers) {
    super(description, transferProgress, progressListenerChain);
    this.subTransfers = subTransfers;
}

public TransferProgressUpdatingListener(TransferProgress transferProgress) {
    this.transferProgress = transferProgress;
}

AbstractTransfer(String description, TransferProgress transferProgress, ProgressListenerChain progressListenerChain) {
    this(description, transferProgress, progressListenerChain, null);
}

public CopyImpl(String description, TransferProgress transferProgress,
        com.amazonaws.event.ProgressListenerChain progressListenerChain,
        TransferStateChangeListener stateChangeListener) {
    super(description, transferProgress, progressListenerChain,
            stateChangeListener);
}

public MultipleFileDownloadImpl(String description, TransferProgress transferProgress,
        ProgressListenerChain progressListenerChain, String keyPrefix, String bucketName, Collection<? extends Download> downloads) {
    super(description, transferProgress, progressListenerChain, downloads);
    this.keyPrefix = keyPrefix;
    this.bucketName = bucketName;
}

@Test
public void testPerformTransferAssertErrorWhenTransferBytesMismatch() throws Exception
{
    S3Operations originalS3Operations = (S3Operations) ReflectionTestUtils.getField(s3Dao, "s3Operations");
    S3Operations mockS3Operations = mock(S3Operations.class);
    ReflectionTestUtils.setField(s3Dao, "s3Operations", mockS3Operations);

    // Shorten the sleep interval for faster tests
    long originalSleepIntervalsMillis = (long) ReflectionTestUtils.getField(s3Dao, "sleepIntervalsMillis");
    ReflectionTestUtils.setField(s3Dao, "sleepIntervalsMillis", 1l);

    try
    {
        S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
        s3FileTransferRequestParamsDto.setLocalPath("localPath");

        when(mockS3Operations.upload(any(), any())).then(new Answer<Upload>()
        {
            @Override
            public Upload answer(InvocationOnMock invocation) throws Throwable
            {
                Upload mockedUpload = mock(Upload.class);
                TransferProgress transferProgress = new TransferProgress();
                // bytesTransferred < totalBytesToTransfer should cause error
                ReflectionTestUtils.setField(transferProgress, "bytesTransferred", 0l);
                ReflectionTestUtils.setField(transferProgress, "totalBytesToTransfer", 1l);
                when(mockedUpload.getProgress()).thenReturn(transferProgress);
                when(mockedUpload.isDone()).thenReturn(true);
                when(mockedUpload.getState()).thenReturn(TransferState.Completed);
                return mockedUpload;
            }
        });

        try
        {
            s3Dao.uploadFile(s3FileTransferRequestParamsDto);
            fail();
        }
        catch (Exception e)
        {
            assertEquals(IllegalArgumentException.class, e.getClass());
            assertEquals("Actual number of bytes transferred is less than expected (actual: 0 bytes; expected: 1 bytes).", e.getMessage());
        }
    }
    finally
    {
        ReflectionTestUtils.setField(s3Dao, "s3Operations", originalS3Operations);
        ReflectionTestUtils.setField(s3Dao, "sleepIntervalsMillis", originalSleepIntervalsMillis);
    }
}

@Test
public void testPerformTransferAssertHandleFailedWithAmazonClientException() throws Exception
{
    S3Operations originalS3Operations = (S3Operations) ReflectionTestUtils.getField(s3Dao, "s3Operations");
    S3Operations mockS3Operations = mock(S3Operations.class);
    ReflectionTestUtils.setField(s3Dao, "s3Operations", mockS3Operations);

    // Shorten the sleep interval for faster tests
    long originalSleepIntervalsMillis = (long) ReflectionTestUtils.getField(s3Dao, "sleepIntervalsMillis");
    ReflectionTestUtils.setField(s3Dao, "sleepIntervalsMillis", 1l);

    try
    {
        S3FileCopyRequestParamsDto s3FileCopyRequestParamsDto = new S3FileCopyRequestParamsDto();
        s3FileCopyRequestParamsDto.setSourceBucketName("sourceBucketName");
        s3FileCopyRequestParamsDto.setSourceObjectKey("sourceObjectKey");
        s3FileCopyRequestParamsDto.setTargetBucketName("targetBucketName");
        s3FileCopyRequestParamsDto.setTargetObjectKey("targetObjectKey");
        s3FileCopyRequestParamsDto.setKmsKeyId("kmsKeyId");

        when(mockS3Operations.copyFile(any(), any())).then(new Answer<Copy>()
        {
            @Override
            public Copy answer(InvocationOnMock invocation) throws Throwable
            {
                Copy mockTransfer = mock(Copy.class);

                when(mockTransfer.getProgress()).thenReturn(new TransferProgress());
                when(mockTransfer.getState()).thenReturn(TransferState.Failed);
                when(mockTransfer.isDone()).thenReturn(true);
                when(mockTransfer.waitForException()).thenReturn(new AmazonClientException("message"));
                return mockTransfer;
            }
        });

        try
        {
            s3Dao.copyFile(s3FileCopyRequestParamsDto);
            fail();
        }
        catch (Exception e)
        {
            assertEquals(AmazonClientException.class, e.getClass());
            assertEquals("message", e.getMessage());
        }
    }
    finally
    {
        ReflectionTestUtils.setField(s3Dao, "s3Operations", originalS3Operations);
        ReflectionTestUtils.setField(s3Dao, "sleepIntervalsMillis", originalSleepIntervalsMillis);
    }
}

/**
 * {@inheritDoc}
 * <p/>
 * This implementation creates any directory that does not exist in the path to the destination directory.
 */
@Override
public MultipleFileDownload downloadDirectory(String bucketName, String keyPrefix, File destinationDirectory, TransferManager transferManager)
{
    LOGGER.debug("downloadDirectory(): bucketName = " + bucketName + ", keyPrefix = " + keyPrefix + ", destinationDirectory = " + destinationDirectory);

    MockS3Bucket mockS3Bucket = mockS3Buckets.get(bucketName);

    List<Download> downloads = new ArrayList<>();
    long totalBytes = 0;

    if (mockS3Bucket != null)
    {
        for (MockS3Object mockS3Object : mockS3Bucket.getObjects().values())
        {
            if (mockS3Object.getKey().startsWith(keyPrefix))
            {
                String filePath = destinationDirectory.getAbsolutePath() + "/" + mockS3Object.getKey();
                File file = new File(filePath);
                file.getParentFile().mkdirs(); // Create any directory in the path that does not exist.
                try (FileOutputStream fileOutputStream = new FileOutputStream(file))
                {
                    LOGGER.debug("downloadDirectory(): Writing file " + file);
                    fileOutputStream.write(mockS3Object.getData());
                    totalBytes += mockS3Object.getData().length;
                    downloads.add(new DownloadImpl(null, null, null, null, null, new GetObjectRequest(bucketName, mockS3Object.getKey()), file,
                        mockS3Object.getObjectMetadata(), false));
                }
                catch (IOException e)
                {
                    throw new RuntimeException("Error writing to file " + file, e);
                }
            }
        }
    }

    TransferProgress progress = new TransferProgress();
    progress.setTotalBytesToTransfer(totalBytes);
    progress.updateProgress(totalBytes);

    MultipleFileDownloadImpl multipleFileDownload = new MultipleFileDownloadImpl(null, progress, null, keyPrefix, bucketName, downloads);
    multipleFileDownload.setState(TransferState.Completed);
    return multipleFileDownload;
}

@Override
public MultipleFileUpload uploadFileList(String bucketName, String virtualDirectoryKeyPrefix, File directory, List<File> files,
    ObjectMetadataProvider metadataProvider, TransferManager transferManager)
{
    LOGGER.debug(
        "uploadFileList(): bucketName = " + bucketName + ", virtualDirectoryKeyPrefix = " + virtualDirectoryKeyPrefix + ", directory = " + directory +
            ", files = " + files);

    String directoryPath = directory.getAbsolutePath();

    long totalFileLength = 0;
    List<Upload> subTransfers = new ArrayList<>();
    for (File file : files)
    {
        // Get path to file relative to the specified directory
        String relativeFilePath = file.getAbsolutePath().substring(directoryPath.length());

        // Replace any backslashes (i.e. Windows separator) with a forward slash.
        relativeFilePath = relativeFilePath.replace("\\", "/");

        // Remove any leading slashes
        relativeFilePath = relativeFilePath.replaceAll("^/+", "");

        long fileLength = file.length();

        // Remove any trailing slashes
        virtualDirectoryKeyPrefix = virtualDirectoryKeyPrefix.replaceAll("/+$", "");

        String s3ObjectKey = virtualDirectoryKeyPrefix + "/" + relativeFilePath;
        totalFileLength += fileLength;

        PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, s3ObjectKey, file);

        ObjectMetadata objectMetadata = new ObjectMetadata();
        metadataProvider.provideObjectMetadata(null, objectMetadata);
        putObjectRequest.setMetadata(objectMetadata);

        putObject(putObjectRequest, transferManager.getAmazonS3Client());

        subTransfers.add(new UploadImpl(null, null, null, null));
    }

    TransferProgress progress = new TransferProgress();
    progress.setTotalBytesToTransfer(totalFileLength);
    progress.updateProgress(totalFileLength);

    MultipleFileUploadImpl multipleFileUpload = new MultipleFileUploadImpl(null, progress, null, virtualDirectoryKeyPrefix, bucketName, subTransfers);
    multipleFileUpload.setState(TransferState.Completed);
    return multipleFileUpload;
}

public TransferProgressWrapper(TransferProgress transferProgress) {
    this.transferProgress = transferProgress;
}

/**
 * Returns progress information about this transfer.
 *
 * @return The progress information about this transfer.
 */
public TransferProgress getProgress() {
    return transferProgress;
}