Java 类com.amazonaws.services.s3.transfer.Download 实例源码

@Override
public String retrieve(String file) throws Exception {
    LogUtils.debug(LOG_TAG, "Downloading file: " + file);
       TransferManager tm = new TransferManager(new DefaultAWSCredentialsProviderChain());
       // TransferManager processes all transfers asynchronously, 
       // so this call will return immediately.
       File downloadedFile = new File(Constants.MCSFS_WORKING_DIR + Constants.S3_WORKING_DIR + file + System.currentTimeMillis());
    downloadedFile.getParentFile().mkdirs();
    downloadedFile.createNewFile();
    Download download = tm.download(bucketName, file, downloadedFile);
       download.waitForCompletion();
       LogUtils.debug(LOG_TAG, "Successfully downloaded file from bucket.\nName: " + file + "\nBucket name: " +
               bucketName);
       tm.shutdownNow();
    return downloadedFile.getAbsolutePath();
}

private void downloadFile(TransferManager tx,
                          String bucketName,
                          String sourcePrefixKey,
                          String destinationFile) throws Exception{
    try {
        final File snapshotFile = new File(destinationFile);
        // Only create parent directory once, if it doesn't exist.
        final File parentDir = new File(snapshotFile.getParent());
        if (!parentDir.isDirectory()) {
            final boolean parentDirCreated = parentDir.mkdirs();
            if (!parentDirCreated) {
                LOGGER.error(
                        "Error creating parent directory for file: {}. Skipping to next",
                        destinationFile);
                return;
            }
        }
        snapshotFile.createNewFile();
        final Download download = tx.download(bucketName, sourcePrefixKey, snapshotFile);
        download.waitForCompletion();
    } catch (Exception e) {
        LOGGER.error("Error downloading the file {} : {}", destinationFile, e);
        throw new Exception(e);
    }
}

public static void downloadFile(String bucket_name, String key_name,
      String file_path, boolean pause)
{
    System.out.println("Downloading to file: " + file_path +
          (pause ? " (pause)" : ""));

    File f = new File(file_path);
    TransferManager xfer_mgr = new TransferManager();
    try {
        Download xfer = xfer_mgr.download(bucket_name, key_name, f);
        // loop with Transfer.isDone()
        XferMgrProgress.showTransferProgress(xfer);
        // or block with Transfer.waitForCompletion()
        XferMgrProgress.waitForCompletion(xfer);
    } catch (AmazonServiceException e) {
        System.err.println(e.getErrorMessage());
        System.exit(1);
    }
    xfer_mgr.shutdownNow();
}

@Test
public void getInputStreamSupplier()
        throws AmazonServiceException, AmazonClientException, InterruptedException, IOException {
    TransferManager transferManager = mock(TransferManager.class);
    Download download = mock(Download.class);

    when(transferManager.download(anyString(), anyString(), any())).thenReturn(download);

    File file = Files.createTempFile("micro-s3", "test")
                     .toFile();
    Assert.assertTrue(file.exists());
    ReadUtils utils = new ReadUtils(
                                    transferManager, "test");

    utils.getInputStream("", "", () -> file);

    Assert.assertFalse(file.exists());
}

/**
 * Method to download file from s3 to local file system
 * @param bucketName AWS S3 bucket name
 * @param key (example: android/apkFolder/ApkName.apk)
 * @param file (local file name)
 * @param pollingInterval (polling interval in sec for S3 download status determination)
 */
public void download(final String bucketName, final String key, final File file, long pollingInterval) {
    LOGGER.info("App will be downloaded from s3.");
    LOGGER.info(String.format("[Bucket name: %s] [Key: %s] [File: %s]", bucketName, key, file.getAbsolutePath()));
    DefaultAWSCredentialsProviderChain credentialProviderChain = new DefaultAWSCredentialsProviderChain();
    TransferManager tx = new TransferManager(
                   credentialProviderChain.getCredentials());
    Download appDownload = tx.download(bucketName, key, file);
    try {
        LOGGER.info("Transfer: " + appDownload.getDescription());
        LOGGER.info("   State: " + appDownload.getState());
        LOGGER.info("   Progress: ");
        // You can poll your transfer's status to check its progress
        while (!appDownload.isDone()) {
            LOGGER.info("       transferred: " +(int) (appDownload.getProgress().getPercentTransferred() + 0.5) + "%" );
            CommonUtils.pause(pollingInterval);
        }
        LOGGER.info("   State: " + appDownload.getState());
        //appDownload.waitForCompletion();
    } catch (AmazonClientException e) {
        throw new RuntimeException("File wasn't downloaded from s3. See log: ".concat(e.getMessage()));
    }
    //tx.shutdownNow();
}

private File downloadFile(String objectName) throws IOException {
    File tempFile = new File(System.getProperty("java.io.tmpdir"), objectName.replace('/', '-'));

    Download download = s3TransferManager.download(bucketName, objectName, tempFile);

    try {
        download.waitForCompletion();
    } catch(SdkBaseException | InterruptedException e) {
        throw new RuntimeException(e);
    }

    return tempFile;
}

public static void main(String[] args) throws Exception {
    // create the AWS S3 Client
    AmazonS3 s3 = AWSS3Factory.getS3Client();

    // retrieve the key value from user
    System.out.println( "Enter the object key:" );
    String key = new BufferedReader( new InputStreamReader( System.in ) ).readLine();

    // print start time
    Date start_date = new Date();
    System.out.println(start_date.toString());

    // file will be placed in temp dir with .tmp extension
    File file = File.createTempFile("read-large-object-tm", null);

    TransferManager tm = TransferManagerBuilder.standard()
            .withS3Client(s3)
            .build();

    // download the object to file
    Download download = tm.download(AWSS3Factory.S3_BUCKET, key, file);

    // block until download finished
    download.waitForCompletion();

    tm.shutdownNow();

    // print end time
    Date end_date = new Date();
    System.out.println(end_date.toString());
}

@Test
public void getInputStreamDefaultSupplier()
        throws AmazonServiceException, AmazonClientException, InterruptedException, IOException {
    TransferManager transferManager = mock(TransferManager.class);
    Download download = mock(Download.class);

    when(transferManager.download(anyString(), anyString(), any())).thenReturn(download);

    ReadUtils utils = new ReadUtils(
                                    transferManager, System.getProperty("java.io.tmpdir"));
    utils.getInputStream("", "");
    verify(download).waitForCompletion();
}

public MultipleFileDownloadImpl(String description, TransferProgress transferProgress,
        ProgressListenerChain progressListenerChain, String keyPrefix, String bucketName, Collection<? extends Download> downloads) {
    super(description, transferProgress, progressListenerChain, downloads);
    this.keyPrefix = keyPrefix;
    this.bucketName = bucketName;
}

private void processJobInput(@NotNull final AgentRunningBuild build) {
  if (myJobInputProcessed) return;
  myJobInputProcessed = true;

  final Map<String, String> params = build.getSharedConfigParameters();
  myJobID = getJobId(params);
  if (myJobID == null) {
    LOG.debug(msgForBuild("No AWS CodePipeline job found for the build", build));
    return;
  }

  AWSCommonParams.withAWSClients(params, new AWSCommonParams.WithAWSClients<Void, RuntimeException>() {
    @Nullable
    @Override
    public Void run(@NotNull AWSClients clients) throws RuntimeException {
      AWSCodePipelineClient codePipelineClient = null;
      try {
        codePipelineClient = clients.createCodePipeLineClient();
        final JobData jobData = getJobData(codePipelineClient, params);

        final PipelineContext pipelineContext = jobData.getPipelineContext();
        build.getBuildLogger().message(
          "This build is a part of an AWS CodePipeline pipeline: " + pipelineContext.getPipelineName() +
            "\nLink: https://console.aws.amazon.com/codepipeline/home?region=" + params.get(AWSCommonParams.REGION_NAME_PARAM) + "#/view/" + pipelineContext.getPipelineName() +
            "\nStage: " + pipelineContext.getStage().getName() +
            "\nAction: " + pipelineContext.getAction().getName() +
            "\nJob ID: " + myJobID);

        final List<Artifact> inputArtifacts = jobData.getInputArtifacts();
        if (inputArtifacts.isEmpty()) {
          LOG.debug(msgForBuild("No input artifacts provided for the job with ID: " + myJobID, build));
        } else {

          final File inputFolder = new File(params.get(ARTIFACT_INPUT_FOLDER_CONFIG_PARAM));
          FileUtil.createDir(inputFolder);

          final Collection<Download> downloads = S3Util.withTransferManager(getArtifactS3Client(jobData.getArtifactCredentials(), params), new S3Util.WithTransferManager<Download>() {
            @NotNull
            @Override
            public Collection<Download> run(@NotNull final TransferManager manager) throws Throwable {
              return CollectionsUtil.convertCollection(inputArtifacts, new Converter<Download, Artifact>() {
                @Override
                public Download createFrom(@NotNull Artifact artifact) {
                  final S3ArtifactLocation s3Location = artifact.getLocation().getS3Location();
                  final File destinationFile = getInputArtifactFile(inputFolder, s3Location.getObjectKey());

                  build.getBuildLogger().message("Downloading job input artifact " + s3Location.getObjectKey() + " to " + destinationFile.getAbsolutePath());
                  return manager.download(s3Location.getBucketName(), s3Location.getObjectKey(), destinationFile);
                }
              });
            }
          });
          // for backward compatibility, TW-47902
          for (Download d : downloads) {
            makeArtifactCopy(inputFolder, getInputArtifactFile(inputFolder, d.getKey()), d.getKey(), build);
          }
          if (!jobData.getOutputArtifacts().isEmpty()) {
            FileUtil.createDir(new File(params.get(ARTIFACT_OUTPUT_FOLDER_CONFIG_PARAM)));
          }
        }
      } catch (Throwable e) {
        failOnException(codePipelineClient, build, e);
      }
      return null;
    }
  });
}

@Override
public Download download(String s3BucketName, String s3Key, File file, TransferManager transferManager)
{
    return transferManager.download(s3BucketName, s3Key, file);
}

/**
 * {@inheritDoc}
 * <p/>
 * This implementation creates any directory that does not exist in the path to the destination directory.
 */
@Override
public MultipleFileDownload downloadDirectory(String bucketName, String keyPrefix, File destinationDirectory, TransferManager transferManager)
{
    LOGGER.debug("downloadDirectory(): bucketName = " + bucketName + ", keyPrefix = " + keyPrefix + ", destinationDirectory = " + destinationDirectory);

    MockS3Bucket mockS3Bucket = mockS3Buckets.get(bucketName);

    List<Download> downloads = new ArrayList<>();
    long totalBytes = 0;

    if (mockS3Bucket != null)
    {
        for (MockS3Object mockS3Object : mockS3Bucket.getObjects().values())
        {
            if (mockS3Object.getKey().startsWith(keyPrefix))
            {
                String filePath = destinationDirectory.getAbsolutePath() + "/" + mockS3Object.getKey();
                File file = new File(filePath);
                file.getParentFile().mkdirs(); // Create any directory in the path that does not exist.
                try (FileOutputStream fileOutputStream = new FileOutputStream(file))
                {
                    LOGGER.debug("downloadDirectory(): Writing file " + file);
                    fileOutputStream.write(mockS3Object.getData());
                    totalBytes += mockS3Object.getData().length;
                    downloads.add(new DownloadImpl(null, null, null, null, null, new GetObjectRequest(bucketName, mockS3Object.getKey()), file,
                        mockS3Object.getObjectMetadata(), false));
                }
                catch (IOException e)
                {
                    throw new RuntimeException("Error writing to file " + file, e);
                }
            }
        }
    }

    TransferProgress progress = new TransferProgress();
    progress.setTotalBytesToTransfer(totalBytes);
    progress.updateProgress(totalBytes);

    MultipleFileDownloadImpl multipleFileDownload = new MultipleFileDownloadImpl(null, progress, null, keyPrefix, bucketName, downloads);
    multipleFileDownload.setState(TransferState.Completed);
    return multipleFileDownload;
}

public static Download getFile(final ClientOptions clientOptions, final String bucketName, final String key, final File file) {
    LOGGER.debug(format("Receiving object %1$s as file %2$s from bucket %3$s", key, file.getAbsolutePath(), bucketName));

    return getTransferManager(clientOptions).download(bucketName, key, file);
}

public static Download getFile(final ClientOptions clientOptions, final GetObjectRequest getObjectRequest, final File file) {
    LOGGER.debug(format("Receiving object %1$s as file %2$s from bucket %3$s using GetObjectRequest", getObjectRequest.getKey(), file.getAbsolutePath(), getObjectRequest.getBucketName()));

    return getTransferManager(clientOptions).download(getObjectRequest, file);
}

/**
 * Method returns InputStream from S3Object. Multi-part download is used to
 * get file. s3.tmp.dir property used to store temporary files. You can
 * specify temporary file name by using tempFileSupplier object.
 * 
 * @param bucketName
 * @param key
 *            -
 * @param tempFileSupplier
 *            - Supplier providing temporary filenames
 * @return InputStream of
 * @throws AmazonServiceException
 * @throws AmazonClientException
 * @throws InterruptedException
 * @throws IOException
 */
public InputStream getInputStream(String bucketName, String key, Supplier<File> tempFileSupplier)
        throws AmazonServiceException, AmazonClientException, InterruptedException, IOException {
    File file = tempFileSupplier.get();
    try {
        Download download = transferManager.download(bucketName, key, file);
        download.waitForCompletion();
        return new ByteArrayInputStream(
                                        FileUtils.readFileToByteArray(file));
    } finally {
        file.delete();
    }
}

/**
 * Schedules a new transfer to download data from Amazon S3 and save it to the specified file.
 *
 * @param s3BucketName the S3 bucket name
 * @param s3Key the S3 key
 * @param file the destination file
 * @param transferManager the transfer manager implementation to use
 *
 * @return the object that represents an asynchronous download from Amazon S3
 */
public Download download(String s3BucketName, String s3Key, File file, TransferManager transferManager);