/** * <p> * Populates the specified request with the numerous attributes available in * <code>SSEWithCustomerKeyRequest</code>. * </p> * * @param request * The request to populate with headers to represent all the * options expressed in the * <code>ServerSideEncryptionWithCustomerKeyRequest</code> * object. * @param sseKey * The request object for an S3 operation that allows server-side * encryption using customer-provided keys. */ private static void populateSSE_C(Request<?> request, SSECustomerKey sseKey) { if (sseKey == null) return; addHeaderIfNotNull(request, Headers.SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, sseKey.getAlgorithm()); addHeaderIfNotNull(request, Headers.SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, sseKey.getKey()); addHeaderIfNotNull(request, Headers.SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, sseKey.getMd5()); // Calculate the MD5 hash of the encryption key and fill it in the // header, if the user didn't specify it in the metadata if (sseKey.getKey() != null && sseKey.getMd5() == null) { String encryptionKey_b64 = sseKey.getKey(); byte[] encryptionKey = Base64.decode(encryptionKey_b64); request.addHeader(Headers.SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, Md5Utils.md5AsBase64(encryptionKey)); } }
private static void populateSourceSSE_C(Request<?> request, SSECustomerKey sseKey) { if (sseKey == null) return; // Populate the SSE-C parameters for the source object addHeaderIfNotNull(request, Headers.COPY_SOURCE_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, sseKey.getAlgorithm()); addHeaderIfNotNull(request, Headers.COPY_SOURCE_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, sseKey.getKey()); addHeaderIfNotNull(request, Headers.COPY_SOURCE_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, sseKey.getMd5()); // Calculate the MD5 hash of the encryption key and fill it in the // header, if the user didn't specify it in the metadata if (sseKey.getKey() != null && sseKey.getMd5() == null) { String encryptionKey_b64 = sseKey.getKey(); byte[] encryptionKey = Base64.decode(encryptionKey_b64); request.addHeader(Headers.COPY_SOURCE_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, Md5Utils.md5AsBase64(encryptionKey)); } }
static S3Object getObject( AmazonS3 s3Client, String bucket, String objectKey, boolean useSSE, CredentialValue customerKey, CredentialValue customerKeyMd5 ) throws StageException { GetObjectRequest getObjectRequest = new GetObjectRequest(bucket, objectKey); if (useSSE) { SSECustomerKey sseCustomerKey = new SSECustomerKey(customerKey.get()); sseCustomerKey.setMd5(customerKeyMd5.get()); getObjectRequest.setSSECustomerKey(sseCustomerKey); } return s3Client.getObject(getObjectRequest); }
static S3Object getObjectRange( AmazonS3 s3Client, String bucket, String objectKey, long range, boolean useSSE, CredentialValue customerKey, CredentialValue customerKeyMd5 ) throws StageException { GetObjectRequest getObjectRequest = new GetObjectRequest(bucket, objectKey).withRange(0, range); if (useSSE) { SSECustomerKey sseCustomerKey = new SSECustomerKey(customerKey.get()); sseCustomerKey.setMd5(customerKeyMd5.get()); getObjectRequest.setSSECustomerKey(sseCustomerKey); } return s3Client.getObject(getObjectRequest); }
private cfData readToMemory(AmazonS3 s3Client, String bucket, String key, String aes256key, int retry, int retryseconds) throws Exception { // Let us run around the number of attempts int attempts = 0; while ( attempts < retry ){ try{ GetObjectRequest gor = new GetObjectRequest(bucket, key); if ( aes256key != null && !aes256key.isEmpty() ) gor.setSSECustomerKey( new SSECustomerKey(aes256key) ); S3Object s3object = s3Client.getObject(gor); String contentType = s3object.getObjectMetadata().getContentType(); ByteArrayOutputStream baos = new ByteArrayOutputStream( 32000 ); StreamUtil.copyTo(s3object.getObjectContent(), baos, false ); if ( contentType.indexOf("text") != -1 || contentType.indexOf("javascript") != -1 ){ return new cfStringData( baos.toString() ); }else{ return new cfBinaryData( baos.toByteArray() ); } }catch(Exception e){ cfEngine.log("Failed: AmazonS3Read(bucket=" + bucket + "; key=" + key + "; attempt=" + (attempts+1) + "; exception=" + e.getMessage() + ")"); attempts++; if ( attempts == retry ) throw e; else Thread.sleep( retryseconds*1000 ); } } return null; // should never }
public cfData execute( cfSession _session, cfArgStructData argStruct ) throws cfmRunTimeException{ AmazonKey amazonKey = getAmazonKey(_session, argStruct); AmazonS3 s3Client = getAmazonS3(amazonKey); String bucket = getNamedStringParam(argStruct, "bucket", null ); String srckey = getNamedStringParam(argStruct, "srckey", null ); String deskey = getNamedStringParam(argStruct, "destkey", null ); String aes256key = getNamedStringParam(argStruct, "aes256key", null ); if ( srckey != null && srckey.charAt( 0 ) == '/' ) srckey = srckey.substring(1); if ( deskey != null && deskey.charAt( 0 ) == '/' ) deskey = deskey.substring(1); CopyObjectRequest cor = new CopyObjectRequest(bucket, srckey, bucket, deskey); if ( aes256key != null && !aes256key.isEmpty() ){ cor.setSourceSSECustomerKey( new SSECustomerKey(aes256key) ); cor.setDestinationSSECustomerKey( new SSECustomerKey(aes256key) ); } try { s3Client.copyObject(cor); s3Client.deleteObject(new DeleteObjectRequest(bucket, srckey)); return cfBooleanData.TRUE; } catch (Exception e) { throwException(_session, "AmazonS3: " + e.getMessage() ); return cfBooleanData.FALSE; } }
private cfData readToFile(AmazonS3 s3Client, String bucket, String key, String localpath, boolean overwrite, String aes256key, int retry, int retryseconds ) throws Exception { File localFile = new File( localpath ); if ( localFile.isFile() ){ if ( !overwrite ) throw new Exception("The file specified exists: " + localpath ); else localFile.delete(); } // Let us run around the number of attempts int attempts = 0; while ( attempts < retry ){ try{ GetObjectRequest gor = new GetObjectRequest(bucket, key); if ( aes256key != null && !aes256key.isEmpty() ) gor.setSSECustomerKey( new SSECustomerKey(aes256key) ); S3Object s3object = s3Client.getObject(gor); FileOutputStream outStream = null; try{ outStream = new FileOutputStream( localFile ); StreamUtil.copyTo(s3object.getObjectContent(), outStream, false ); }finally{ StreamUtil.closeStream(outStream); } return new cfStringData( localFile.toString() ); }catch(Exception e){ cfEngine.log("Failed: AmazonS3Read(bucket=" + bucket + "; key=" + key + "; attempt=" + (attempts+1) + "; exception=" + e.getMessage() + ")"); attempts++; if ( attempts == retry ) throw e; else Thread.sleep( retryseconds*1000 ); } } return null; // should never }
public cfData execute( cfSession _session, cfArgStructData argStruct ) throws cfmRunTimeException{ AmazonKey amazonKey = getAmazonKey(_session, argStruct); AmazonS3 s3Client = getAmazonS3(amazonKey); String srcbucket = getNamedStringParam(argStruct, "srcbucket", null ); String srckey = getNamedStringParam(argStruct, "srckey", null ); String srcaes256key = getNamedStringParam(argStruct, "srcaes256key", null ); String destbucket = getNamedStringParam(argStruct, "destbucket", null ); String deskey = getNamedStringParam(argStruct, "destkey", null ); String destaes256key = getNamedStringParam(argStruct, "destaes256key", null ); String deststorageclass = getNamedStringParam(argStruct, "deststorageclass", null ); String destacl = getNamedStringParam(argStruct, "destacl", null ); if ( srckey != null && srckey.charAt( 0 ) == '/' ) srckey = srckey.substring(1); if ( deskey != null && deskey.charAt( 0 ) == '/' ) deskey = deskey.substring(1); CopyObjectRequest cor = new CopyObjectRequest(srcbucket, srckey, destbucket, deskey); if ( srcaes256key != null && !srcaes256key.isEmpty() ) cor.setSourceSSECustomerKey( new SSECustomerKey(srcaes256key) ); if ( destaes256key != null && !destaes256key.isEmpty() ) cor.setDestinationSSECustomerKey( new SSECustomerKey(destaes256key) ); if ( deststorageclass != null && !deststorageclass.isEmpty() ) cor.setStorageClass( amazonKey.getAmazonStorageClass(deststorageclass) ); if ( destacl != null && !destacl.isEmpty() ) cor.setCannedAccessControlList( amazonKey.getAmazonCannedAcl(destacl) ); try { s3Client.copyObject(cor); return cfBooleanData.TRUE; } catch (Exception e) { throwException(_session, "AmazonS3: " + e.getMessage() ); return cfBooleanData.FALSE; } }
private void writeData( AmazonKey amazonKey, String bucket, String key, Map<String, String> metadata, StorageClass storage, String mimetype, cfData data, int retry, int retryseconds, String acl, String aes256key, Map<String, String> customheaders ) throws Exception { if ( mimetype == null ) { if ( data.getDataType() == cfData.CFBINARYDATA ) mimetype = "application/unknown"; else if ( cfData.isSimpleValue( data ) ) mimetype = "text/plain"; else mimetype = "application/json"; // Check to see if the mime type is in the metadata if ( metadata != null && metadata.containsKey( "Content-Type" ) ) mimetype = metadata.get( "Content-Type" ); } InputStream ios = null; long size = 0; if ( data.getDataType() == cfData.CFSTRINGDATA ) { ios = new java.io.ByteArrayInputStream( data.getString().getBytes() ); size = data.getString().length(); } else if ( data.getDataType() == cfData.CFBINARYDATA ) { ios = new java.io.ByteArrayInputStream( ( (cfBinaryData) data ).getByteArray() ); size = ( (cfBinaryData) data ).getLength(); } else { serializejson json = new serializejson(); StringBuilder out = new StringBuilder(); json.encodeJSON( out, data, false, CaseType.MAINTAIN, DateType.LONG ); size = out.length(); mimetype = "application/json"; ios = new java.io.ByteArrayInputStream( out.toString().getBytes() ); } // Setup the object data ObjectMetadata omd = new ObjectMetadata(); if ( metadata != null ) omd.setUserMetadata( metadata ); omd.setContentType( mimetype ); omd.setContentLength( size ); AmazonS3 s3Client = getAmazonS3( amazonKey ); // Let us run around the number of attempts int attempts = 0; while ( attempts < retry ) { try { PutObjectRequest por = new PutObjectRequest( bucket, key, ios, omd ); por.setStorageClass( storage ); if ( aes256key != null && !aes256key.isEmpty() ) por.setSSECustomerKey( new SSECustomerKey( aes256key ) ); if ( acl != null && !acl.isEmpty() ) por.setCannedAcl( amazonKey.getAmazonCannedAcl( acl ) ); if ( customheaders != null && !customheaders.isEmpty() ) { Iterator<String> it = customheaders.keySet().iterator(); while ( it.hasNext() ) { String k = it.next(); por.putCustomRequestHeader( k, customheaders.get( k ) ); } } s3Client.putObject( por ); break; } catch ( Exception e ) { cfEngine.log( "Failed: AmazonS3Write(bucket=" + bucket + "; key=" + key + "; attempt=" + ( attempts + 1 ) + "; exception=" + e.getMessage() + ")" ); attempts++; if ( attempts == retry ) throw e; else Thread.sleep( retryseconds * 1000 ); } } }
private void writeFile( AmazonKey amazonKey, String bucket, String key, Map<String, String> metadata, StorageClass storage, String localpath, int retry, int retryseconds, boolean deletefile, boolean background, String callback, String callbackdata, String appname, String acl, String aes256key, Map<String, String> customheaders ) throws Exception { File localFile = new File( localpath ); if ( !localFile.isFile() ) throw new Exception( "The file specified does not exist: " + localpath ); // Push this to the background loader to handle and return immediately if ( background ) { BackgroundUploader.acceptFile( amazonKey, bucket, key, metadata, storage, localpath, retry, retryseconds, deletefile, callback, callbackdata, appname, acl, aes256key, customheaders ); return; } // Setup the object data ObjectMetadata omd = new ObjectMetadata(); if ( metadata != null ) omd.setUserMetadata( metadata ); AmazonS3 s3Client = getAmazonS3( amazonKey ); // Let us run around the number of attempts int attempts = 0; while ( attempts < retry ) { try { PutObjectRequest por = new PutObjectRequest( bucket, key, localFile ); por.setMetadata( omd ); por.setStorageClass( storage ); if ( acl != null && !acl.isEmpty() ) por.setCannedAcl( amazonKey.getAmazonCannedAcl( acl ) ); if ( aes256key != null && !aes256key.isEmpty() ) por.setSSECustomerKey( new SSECustomerKey( aes256key ) ); if ( customheaders != null && !customheaders.isEmpty() ) { Iterator<String> it = customheaders.keySet().iterator(); while ( it.hasNext() ) { String k = it.next(); por.putCustomRequestHeader( k, customheaders.get( k ) ); } } s3Client.putObject( por ); break; } catch ( Exception e ) { cfEngine.log( "Failed: AmazonS3Write(bucket=" + bucket + "key=" + key + "; file=" + localFile + "; attempt=" + ( attempts + 1 ) + "; exception=" + e.getMessage() + ")" ); attempts++; if ( attempts == retry ) throw e; else Thread.sleep( retryseconds * 1000 ); } } // delete the file now that it is a success if ( deletefile ) localFile.delete(); }
/** * Fetchs a remote object from S3; datasource, bucket, key, aes256key supported * * @param props * @param _Session * @throws cfmRunTimeException */ private void remoteFetchS3( cfStructData props, cfSession _Session ) throws cfmRunTimeException { if ( !props.containsKey( "datasource" ) || !props.containsKey( "bucket" ) || !props.containsKey( "key" ) ) throw newRunTimeException( "'remote'.type=s3; minimum keys are datasource, bucket and key" ); String datasource = props.getData( "datasource" ).getString(); String bucket = props.getData( "bucket" ).getString(); String key = props.getData( "key" ).getString(); // Get the Amazon datasource AmazonKey amazonKey = AmazonKeyFactory.getDS( datasource ); if ( amazonKey == null ) throw newRunTimeException( "Amazon Datasource [" + datasource + "] has not been registered; use AmazonRegisterDataSource()" ); amazonKey.setDataSource( datasource ); AmazonS3 s3Client = new AmazonBase().getAmazonS3( amazonKey ); GetObjectRequest gor = new GetObjectRequest( bucket, key ); if ( props.containsKey( "aes256key" ) ) { String aes256key = props.getData( "aes256key" ).getString(); if ( !aes256key.isEmpty() ) gor.setSSECustomerKey( new SSECustomerKey( aes256key ) ); } // Get the object try { S3Object s3object = s3Client.getObject( gor ); _Session.setContentType( s3object.getObjectMetadata().getContentType() ); InputStream in = s3object.getObjectContent(); byte[] buffer = new byte[65536]; int readCount = 0; while ( ( readCount = in.read( buffer ) ) != -1 ) { _Session.write( buffer, 0, readCount ); _Session.pageFlush(); } } catch ( Exception e ) { if ( e.getMessage().indexOf("404") != -1 ){ _Session.setStatus( 404 ); return; }else{ cfEngine.log( e.getMessage() ); throw newRunTimeException( e.getMessage() + "; key=" + key + "; bucket=" + bucket ); } } }
private void enableCustomerEncryption(PutObjectRequest uploadRequest) { SSECustomerKey sseKey = new SSECustomerKey(mConfig.getAwsSseCustomerKey()); uploadRequest.withSSECustomerKey(sseKey); }