private void initTransferManager() { TransferManagerConfiguration transferConfiguration = new TransferManagerConfiguration(); transferConfiguration.setMinimumUploadPartSize(partSize); transferConfiguration.setMultipartUploadThreshold(multiPartThreshold); transferConfiguration.setMultipartCopyPartSize(partSize); transferConfiguration.setMultipartCopyThreshold(multiPartThreshold); transfers = new TransferManager(mClient, unboundedThreadPool); transfers.setConfiguration(transferConfiguration); }
public PrestoS3OutputStream(AmazonS3 s3, TransferManagerConfiguration config, String host, String key, File tempFile, boolean sseEnabled) throws IOException { super(new BufferedOutputStream(new FileOutputStream(requireNonNull(tempFile, "tempFile is null")))); transferManager = new TransferManager(requireNonNull(s3, "s3 is null")); transferManager.setConfiguration(requireNonNull(config, "config is null")); this.host = requireNonNull(host, "host is null"); this.key = requireNonNull(key, "key is null"); this.tempFile = tempFile; this.sseEnabled = sseEnabled; log.debug("OutputStream for key '%s' using file: %s", key, tempFile); }
/** * Returns the optimal part size, in bytes, for each individual part upload * in a multipart upload. * * @param putObjectRequest * The request containing all the details of the upload. * @param configuration * Configuration values to use when calculating size. * * @return The optimal part size, in bytes, for each individual part upload * in a multipart upload. */ public static long calculateOptimalPartSize(PutObjectRequest putObjectRequest, TransferManagerConfiguration configuration) { double contentLength = TransferManagerUtils.getContentLength(putObjectRequest); double optimalPartSize = (double)contentLength / (double)MAXIMUM_UPLOAD_PARTS; // round up so we don't push the upload over the maximum number of parts optimalPartSize = Math.ceil(optimalPartSize); return (long)Math.max(optimalPartSize, configuration.getMinimumUploadPartSize()); }
/** * Calculates the optimal part size of each part request if the copy * operation is carried out as multi-part copy. * * @param copyObjectRequest * the original request. * @param configuration * configuration containing the default part size. * @param contentLengthOfSource * content length of the Amazon S3 object. * @return the optimal part size for a copy part request. */ public static long calculateOptimalPartSizeForCopy( CopyObjectRequest copyObjectRequest, TransferManagerConfiguration configuration, long contentLengthOfSource) { double optimalPartSize = (double) contentLengthOfSource / (double) MAXIMUM_UPLOAD_PARTS; // round up so we don't push the copy over the maximum number of parts optimalPartSize = Math.ceil(optimalPartSize); return (long) Math.max(optimalPartSize, configuration.getMultipartCopyPartSize()); }
/** * Returns true if the the specified request should be processed as a * multipart upload (instead of a single part upload). * * @param putObjectRequest * The request containing all the details of the upload. * @param configuration * Configuration settings controlling how transfer manager * processes requests. * * @return True if the the specified request should be processed as a * multipart upload. */ public static boolean shouldUseMultipartUpload(PutObjectRequest putObjectRequest, TransferManagerConfiguration configuration) { long contentLength = TransferManagerUtils.getContentLength(putObjectRequest); return (contentLength > configuration.getMultipartUploadThreshold()) && putObjectRequest.getTagging() == null; }