@Override public S3FileTransferResultsDto uploadDirectory(final S3FileTransferRequestParamsDto params) throws InterruptedException { LOGGER.info("Uploading local directory to S3... localDirectory=\"{}\" s3KeyPrefix=\"{}\" s3BucketName=\"{}\"", params.getLocalPath(), params.getS3KeyPrefix(), params.getS3BucketName()); // Perform the transfer. S3FileTransferResultsDto results = performTransfer(params, new Transferer() { @Override public Transfer performTransfer(TransferManager transferManager) { return s3Operations.uploadDirectory(params.getS3BucketName(), params.getS3KeyPrefix(), new File(params.getLocalPath()), params.isRecursive(), new ObjectMetadataProvider() { @Override public void provideObjectMetadata(File file, ObjectMetadata metadata) { prepareMetadata(params, metadata); } }, transferManager); } }); LOGGER.info("Uploaded local directory to S3. " + "localDirectory=\"{}\" s3KeyPrefix=\"{}\" s3BucketName=\"{}\" s3KeyCount={} totalBytesTransferred={} transferDuration=\"{}\"", params.getLocalPath(), params.getS3KeyPrefix(), params.getS3BucketName(), results.getTotalFilesTransferred(), results.getTotalBytesTransferred(), HerdDateUtils.formatDuration(results.getDurationMillis())); logOverallTransferRate(results); return results; }
@Override public MultipleFileUpload uploadDirectory(String bucketName, String virtualDirectoryKeyPrefix, File directory, boolean includeSubdirectories, ObjectMetadataProvider metadataProvider, TransferManager transferManager) { LOGGER.debug( "uploadDirectory(): bucketName = " + bucketName + ", virtualDirectoryKeyPrefix = " + virtualDirectoryKeyPrefix + ", directory = " + directory + ", includeSubdirectories = " + includeSubdirectories); List<File> files = new ArrayList<>(); listFiles(directory, files, includeSubdirectories); return uploadFileList(bucketName, virtualDirectoryKeyPrefix, directory, files, metadataProvider, transferManager); }
@Override protected void init() { tx = new TransferManager(getS3Client(), createDefaultExecutorService()); objectMetadataProvider = new ObjectMetadataProvider() { @Override public void provideObjectMetadata(File file, ObjectMetadata metadata) { metadata.setSSEAlgorithm("AES256"); metadata.setContentLength(file.length()); } }; }
@Override public MultipleFileUpload uploadDirectory(String s3BucketName, String virtualDirectoryKeyPrefix, File directory, boolean includeSubdirectories, ObjectMetadataProvider metadataProvider, TransferManager transferManager) { return transferManager.uploadDirectory(s3BucketName, virtualDirectoryKeyPrefix, directory, includeSubdirectories, metadataProvider); }
@Override public MultipleFileUpload uploadFileList(String s3BucketName, String virtualDirectoryKeyPrefix, File directory, List<File> files, ObjectMetadataProvider metadataProvider, TransferManager transferManager) { return transferManager.uploadFileList(s3BucketName, virtualDirectoryKeyPrefix, directory, files, metadataProvider); }
@Override public MultipleFileUpload uploadFileList(String bucketName, String virtualDirectoryKeyPrefix, File directory, List<File> files, ObjectMetadataProvider metadataProvider, TransferManager transferManager) { LOGGER.debug( "uploadFileList(): bucketName = " + bucketName + ", virtualDirectoryKeyPrefix = " + virtualDirectoryKeyPrefix + ", directory = " + directory + ", files = " + files); String directoryPath = directory.getAbsolutePath(); long totalFileLength = 0; List<Upload> subTransfers = new ArrayList<>(); for (File file : files) { // Get path to file relative to the specified directory String relativeFilePath = file.getAbsolutePath().substring(directoryPath.length()); // Replace any backslashes (i.e. Windows separator) with a forward slash. relativeFilePath = relativeFilePath.replace("\\", "/"); // Remove any leading slashes relativeFilePath = relativeFilePath.replaceAll("^/+", ""); long fileLength = file.length(); // Remove any trailing slashes virtualDirectoryKeyPrefix = virtualDirectoryKeyPrefix.replaceAll("/+$", ""); String s3ObjectKey = virtualDirectoryKeyPrefix + "/" + relativeFilePath; totalFileLength += fileLength; PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, s3ObjectKey, file); ObjectMetadata objectMetadata = new ObjectMetadata(); metadataProvider.provideObjectMetadata(null, objectMetadata); putObjectRequest.setMetadata(objectMetadata); putObject(putObjectRequest, transferManager.getAmazonS3Client()); subTransfers.add(new UploadImpl(null, null, null, null)); } TransferProgress progress = new TransferProgress(); progress.setTotalBytesToTransfer(totalFileLength); progress.updateProgress(totalFileLength); MultipleFileUploadImpl multipleFileUpload = new MultipleFileUploadImpl(null, progress, null, virtualDirectoryKeyPrefix, bucketName, subTransfers); multipleFileUpload.setState(TransferState.Completed); return multipleFileUpload; }
/** * Uploads all files in the directory given to the bucket named, optionally recursing for all subdirectories. * * @param s3BucketName the S3 bucket name * @param virtualDirectoryKeyPrefix the key prefix of the virtual directory to upload to * @param directory the directory to upload * @param includeSubdirectories specified whether to include subdirectories in the upload. If true, files found in subdirectories will be included with an * appropriate concatenation to the key prefix * @param metadataProvider the callback of type <code>ObjectMetadataProvider</code> which is used to provide metadata for each file being uploaded * @param transferManager the transfer manager implementation to use * * @return the multiple file upload information */ public MultipleFileUpload uploadDirectory(String s3BucketName, String virtualDirectoryKeyPrefix, File directory, boolean includeSubdirectories, ObjectMetadataProvider metadataProvider, TransferManager transferManager);
/** * Uploads all specified files to the bucket named, constructing relative keys depending on the common parent directory given. * * @param s3BucketName the S3 bucket name * @param virtualDirectoryKeyPrefix the key prefix of the virtual directory to upload to * @param directory the common parent directory of files to upload. The keys of the files in the list of files are constructed relative to this directory * and the virtualDirectoryKeyPrefix * @param files the list of files to upload. The keys of the files are calculated relative to the common parent directory and the virtualDirectoryKeyPrefix * @param metadataProvider the callback of type <code>ObjectMetadataProvider</code> which is used to provide metadata for each file being uploaded * @param transferManager the transfer manager implementation to use * * @return the multiple file upload information */ public MultipleFileUpload uploadFileList(String s3BucketName, String virtualDirectoryKeyPrefix, File directory, List<File> files, ObjectMetadataProvider metadataProvider, TransferManager transferManager);