private AmazonS3Client getAmazonS3Client(BackupRestoreContext ctx) throws URISyntaxException { final String accessKey = ctx.getAccountId(); final String secretKey = ctx.getSecretKey(); String endpoint = getEndpoint(ctx); LOGGER.info("endpoint: {}", endpoint); final BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(accessKey, secretKey); final AmazonS3Client amazonS3Client = new AmazonS3Client(basicAWSCredentials); amazonS3Client.setEndpoint(endpoint); if (ctx.usesEmc()) { final S3ClientOptions options = new S3ClientOptions(); options.setPathStyleAccess(true); amazonS3Client.setS3ClientOptions(options); } return amazonS3Client; }
private void connectToS3(String accessKey, String secretKey, String endpoint, boolean secure, boolean pathStyle) { LOG.info("'{}','{}','{}'", accessKey, secretKey, endpoint); AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey); ClientConfiguration clientConfiguration = new ClientConfiguration() .withProtocol(secure ? Protocol.HTTPS : Protocol.HTTP) .withUserAgentPrefix("s3browser") .withSignerOverride("S3SignerType"); s3Client = new AmazonS3Client(credentials, clientConfiguration); s3Client.setS3ClientOptions(S3ClientOptions.builder() .setPathStyleAccess(pathStyle) .disableChunkedEncoding() .build()); s3Client.setEndpoint(endpoint); }
@Override public void start(Map<String, String> props) throws ConnectException { readConfig(props); // Use default credentials provider that looks in Env + Java properties + profile + instance role AmazonS3 s3Client = new AmazonS3Client(); // If worker config sets explicit endpoint override (e.g. for testing) use that if (overrideS3Endpoint != "") { s3Client.setEndpoint(overrideS3Endpoint); } if (s3PathStyle) { s3Client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true)); } s3 = new S3Writer(bucket, prefix, s3Client); // Recover initial assignments Set<TopicPartition> assignment = context.assignment(); recoverAssignment(assignment); }
@Override public Storage newStorage(Config config) { AmazonS3Client client = new AmazonS3Client( buildCredentialsProvider(config), buildClientConfiguration(config)); if (config.has("endpoint")) { client.setEndpoint(config.get("endpoint", String.class)); } if (config.has("path-style-access")) { client.setS3ClientOptions( S3ClientOptions.builder().setPathStyleAccess( config.get("path-style-access", Boolean.class, false) ).build()); } String bucket = config.get("bucket", String.class); return new S3Storage(client, bucket); }
/** * @return S3 client */ private AmazonS3 buildS3Client() { AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey); ClientConfiguration clientConfiguration = new ClientConfiguration() .withProtocol(useHttp ? Protocol.HTTP : Protocol.HTTPS) .withUserAgent("s3pt") .withGzip(useGzip) .withTcpKeepAlive(useKeepAlive); if (signerOverride != null) { String signer = signerOverride.endsWith("Type") ? signerOverride : signerOverride + "Type"; clientConfiguration.setSignerOverride(signer); } AmazonS3 s3Client = new AmazonS3Client(credentials, clientConfiguration); s3Client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(usePathStyleAccess).disableChunkedEncoding().build()); s3Client.setEndpoint(endpointUrl); return s3Client; }
static AmazonS3Client configureClient(AmazonS3Client s3Client) { if (System.getProperty(S3_REGION) != null) { String region = System.getProperty(S3_REGION); Log.debug("S3 region specified: ", region); s3Client.setRegion(RegionUtils.getRegion(region)); } // Region overrides end-point settings if (System.getProperty(S3_END_POINT) != null) { String endPoint = System.getProperty(S3_END_POINT); Log.debug("S3 endpoint specified: ", endPoint); s3Client.setEndpoint(endPoint); } if (System.getProperty(S3_ENABLE_PATH_STYLE) != null && Boolean.valueOf(System.getProperty(S3_ENABLE_PATH_STYLE))) { Log.debug("S3 path style access enabled"); S3ClientOptions sco = new S3ClientOptions(); sco.setPathStyleAccess(true); s3Client.setS3ClientOptions(sco); } return s3Client; }
@Override protected void setUp() throws Exception { super.setUp(); final AWSCredentials credentials = new BasicAWSCredentials(ACCESS_KEY, SECRET_KEY); final ClientConfiguration clientConfiguration = new ClientConfiguration(); clientConfiguration.setPreemptiveBasicProxyAuth(true); final S3ClientOptions options = new S3ClientOptions(); options.setPathStyleAccess(true); final AmazonS3Client client = new AmazonS3Client(credentials, clientConfiguration); client.setS3ClientOptions(options); client.setEndpoint("http://localhost:8080"); _repo = new AmazonS3Repository(client); }
public void testDoubleDots() throws Exception { String filename = System.getProperty("user.home")+"/aws.properties"; PropertiesCredentials credentials = new PropertiesCredentials(new File(filename)); AmazonS3Client client = new AmazonS3Client(credentials); client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true)); client.setRegion(Region.getRegion(Regions.EU_WEST_1)); testPrefix(client,"",6); Collections.sort(prefixTestResults); for (String s : prefixTestResults) { System.out.println(s); } }
private void setAmazonS3ConnectionEndpoint() { S3ClientOptions.Builder clientOptionsBuilder = S3ClientOptions.builder(); Optional<URI> endpoint = s3ConnectionProperties.getEndpoint(); if (endpoint.isPresent()) { amazonS3Client.setEndpoint(endpoint.get().toString()); clientOptionsBuilder.setPathStyleAccess(true); } amazonS3Client.setS3ClientOptions(clientOptionsBuilder.build()); }
@Inject public S3DiscoveryCallback(final AmazonS3 s3, final Configuration configuration, final PluginExecutorService pluginExecutorService) { this.s3 = s3; this.s3.setEndpoint(configuration.getEndpoint()); this.s3.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(configuration.withPathStyleAccess())); this.configuration = configuration; this.pluginExecutorService = pluginExecutorService; this.bucketName = configuration.getBucketName(); }
@Bean @Profile("dev") public AmazonS3Client s3Ninja() { AWSCredentials credentials = new BasicAWSCredentials( "AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); AmazonS3Client newClient = new AmazonS3Client(credentials, new ClientConfiguration()); newClient.setS3ClientOptions(new S3ClientOptions() .withPathStyleAccess(true)); newClient.setEndpoint("http://localhost:9444/s3"); return newClient; }
@Bean @Profile("prod") public AmazonS3Client s3Amazon() { AWSCredentials credentials = new BasicAWSCredentials( "AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); AmazonS3Client newClient = new AmazonS3Client(credentials, new ClientConfiguration()); newClient.setS3ClientOptions(new S3ClientOptions() .withPathStyleAccess(true)); return newClient; }
@BeforeMethod public void setUp() throws Exception { s3mock = System.getProperty("s3mock"); s3mock = s3mock != null ? s3mock : "http://localhost:9444/s3"; client = new AmazonS3Client(new BasicAWSCredentials("AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")); client.setEndpoint(s3mock); client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true)); }
@Test public void testCustomRegionAndPathStyleAccess() throws Exception { Config config = newConfig(); config.set("path_style_access", true); config.set("_command", BUCKET + "/" + KEY); when(taskRequest.getConfig()).thenReturn(config); when(s3Secrets.getSecretOptional("region")).thenReturn(Optional.of(REGION)); when(s3Client.getObjectMetadata(objectMetadataRequestCaptor.capture())).thenThrow(NOT_FOUND_EXCEPTION); Operator operator = factory.newOperator(newContext(projectPath, taskRequest)); try { operator.run(); fail(); } catch (TaskExecutionException ignore) { } verify(s3Client).setS3ClientOptions(s3ClientOptionsCaptor.capture()); S3ClientOptions s3ClientOptions = s3ClientOptionsCaptor.getValue(); assertThat(s3ClientOptions.isPathStyleAccess(), is(true)); verify(s3Client).setRegion(RegionUtils.getRegion(REGION)); }
public static ViPRS3Client getS3Client(String S3_ACCESS_KEY_ID, String S3_SECRET_KEY, String S3_ENDPOINT, String S3_ViPR_NAMESPACE) { BasicAWSCredentials creds = new BasicAWSCredentials(S3_ACCESS_KEY_ID, S3_SECRET_KEY); S3ClientOptions opt = new S3ClientOptions(); opt.setPathStyleAccess(true); ViPRS3Client client = new ViPRS3Client(S3_ENDPOINT, creds); client.setS3ClientOptions(opt); if (S3_ViPR_NAMESPACE != null) { client.setNamespace(S3_ViPR_NAMESPACE); } return client; }
@Test public void testStartTwoServers() throws Exception { S3Server instanceA = null; S3Server instanceB = null; AmazonS3Client client = null; try { instanceA = S3Server.createHttpServer(); instanceB = S3Server.createHttpServer(); instanceA.start(); instanceB.start(); client = new AmazonS3Client(new StaticCredentialsProvider(new AnonymousAWSCredentials())); client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true)); client.setEndpoint(instanceA.getAddress()); BasicTestSuperclass.createDefaultBucket(client); S3Object response = client.getObject("bucketname", "asdf.txt"); String content = inputStreamToString(response.getObjectContent()); assertEquals("asdf",content); assertFalse(instanceA.getAddress().equals(instanceB.getAddress())); } finally { if (client!=null) client.shutdown(); if (instanceA!=null) instanceA.stop(); if (instanceB!=null) instanceB.stop(); } }
@Test public void testStartTwoHttpsServers() throws Exception { S3Server instanceA = null; S3Server instanceB = null; AmazonS3Client client = null; try { instanceA = S3Server.createHttpsServer( MultipleRequestTest.class.getResourceAsStream("/keystore.jks"), "password".toCharArray() ); instanceB = S3Server.createHttpsServer( MultipleRequestTest.class.getResourceAsStream("/keystore.jks"), "password".toCharArray() ); instanceA.start(); instanceB.start(); client = new AmazonS3Client(new StaticCredentialsProvider(new AnonymousAWSCredentials())); client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true)); client.setEndpoint(instanceA.getAddress()); BasicTestSuperclass.createDefaultBucket(client); S3Object response = client.getObject("bucketname", "asdf.txt"); String content = inputStreamToString(response.getObjectContent()); assertEquals("asdf",content); assertFalse(instanceA.getAddress().equals(instanceB.getAddress())); } finally { if (client!=null) client.shutdown(); if (instanceA!=null) instanceA.stop(); if (instanceB!=null) instanceB.stop(); } }
@Test public void testStartBothHttpsAndHttpServers() throws Exception { S3Server instanceA = null; S3Server instanceB = null; AmazonS3Client client = null; try { instanceA = S3Server.createHttpsServer( MultipleRequestTest.class.getResourceAsStream("/keystore.jks"), "password".toCharArray() ); instanceB = S3Server.createHttpServer(); instanceA.start(); instanceB.start(); client = new AmazonS3Client(new StaticCredentialsProvider(new AnonymousAWSCredentials())); client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true)); client.setEndpoint(instanceA.getAddress()); BasicTestSuperclass.createDefaultBucket(client); S3Object response = client.getObject("bucketname", "asdf.txt"); String content = inputStreamToString(response.getObjectContent()); assertEquals("asdf",content); assertFalse(instanceA.getAddress().equals(instanceB.getAddress())); } finally { if (client!=null) client.shutdown(); if (instanceA!=null) instanceA.stop(); if (instanceB!=null) instanceB.stop(); } }
@Before public void setUp() throws Exception { instance = S3Server.createHttpServer(); instance.start(); client = new AmazonS3Client(new StaticCredentialsProvider(new AnonymousAWSCredentials())); client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true)); client.setEndpoint(instance.getAddress()); createDefaultBucket(client); }
Boolean VersioningStatus(String access_key, String secret_key, String bucket, String endpoint, Boolean enable) { String message = null; boolean result = false; AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key); AmazonS3 s3Client = new AmazonS3Client(credentials, new ClientConfiguration()); if (endpoint.contains("amazonaws.com")) { String aws_endpoint = s3Client.getBucketLocation(new GetBucketLocationRequest(bucket)); if (aws_endpoint.contains("US")) { s3Client.setEndpoint("https://s3.amazonaws.com"); } else if (aws_endpoint.contains("us-west")) { s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com"); } else if (aws_endpoint.contains("eu-west")) { s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com"); } else if (aws_endpoint.contains("ap-")) { s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com"); } else if (aws_endpoint.contains("sa-east-1")) { s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com"); } else { s3Client.setEndpoint("https://s3." + aws_endpoint + ".amazonaws.com"); } } else { s3Client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build()); s3Client.setEndpoint(endpoint); } try { message = s3Client.getBucketVersioningConfiguration(bucket).getStatus().toString(); if (message.contains("Enabled") || message.contains("Suspended")) { result = true; } else { result = false; } } catch (Exception versioning) { } return result; }
Boolean LifeCycleStatus(String access_key, String secret_key, String bucket, String endpoint, Boolean enable) { String message = null; boolean result = false; AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key); AmazonS3 s3Client = new AmazonS3Client(credentials, new ClientConfiguration()); if (endpoint.contains("amazonaws.com")) { String aws_endpoint = s3Client.getBucketLocation(new GetBucketLocationRequest(bucket)); if (aws_endpoint.contains("US")) { s3Client.setEndpoint("https://s3.amazonaws.com"); } else if (aws_endpoint.contains("us-west")) { s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com"); } else if (aws_endpoint.contains("eu-west")) { s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com"); } else if (aws_endpoint.contains("ap-")) { s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com"); } else if (aws_endpoint.contains("sa-east-1")) { s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com"); } else { s3Client.setEndpoint("https://s3." + aws_endpoint + ".amazonaws.com"); } } else { s3Client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build()); s3Client.setEndpoint(endpoint); } try { message = s3Client.getBucketLifecycleConfiguration(bucket).getRules().toString(); if (message == null) { result = false; } else { result = true; } } catch (Exception lifecyclestatus) { } return result; }
void setBUCKETwebsite(String object, String access_key, String secret_key, String endpoint, String bucket) { try { AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key); AmazonS3 s3Client = new AmazonS3Client(credentials, new ClientConfiguration()); if (endpoint.contains("amazonaws.com")) { String aws_endpoint = s3Client.getBucketLocation(new GetBucketLocationRequest(bucket)); if (aws_endpoint.contains("US")) { s3Client.setEndpoint("https://s3.amazonaws.com"); } else if (aws_endpoint.contains("us-west")) { s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com"); } else if (aws_endpoint.contains("eu-west")) { s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com"); } else if (aws_endpoint.contains("ap-")) { s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com"); } else if (aws_endpoint.contains("sa-east-1")) { s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com"); } else { s3Client.setEndpoint("https://s3." + aws_endpoint + ".amazonaws.com"); } } else { s3Client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build()); s3Client.setEndpoint(endpoint); } BucketWebsiteConfiguration bucketWebsiteConfiguration = s3Client.getBucketWebsiteConfiguration(bucket); s3Client.setBucketAcl(bucket, CannedAccessControlList.PublicRead); s3Client.setBucketWebsiteConfiguration(bucket, new BucketWebsiteConfiguration("index.html", "error.html")); } catch (Exception setACLpublic) { mainFrame.jTextArea1.append("\nException occurred in ACL"); } }
String setACLurl(String object, String access_key, String secret_key, String endpoint, String bucket) { String URL = null; try { AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key); AmazonS3 s3Client = new AmazonS3Client(credentials, new ClientConfiguration()); if (endpoint.contains("amazonaws.com")) { String aws_endpoint = s3Client.getBucketLocation(new GetBucketLocationRequest(bucket)); if (aws_endpoint.contains("US")) { s3Client.setEndpoint("https://s3.amazonaws.com"); } else if (aws_endpoint.contains("us-west")) { s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com"); } else if (aws_endpoint.contains("eu-west")) { s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com"); } else if (aws_endpoint.contains("ap-")) { s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com"); } else if (aws_endpoint.contains("sa-east-1")) { s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com"); } else { s3Client.setEndpoint("https://s3." + aws_endpoint + ".amazonaws.com"); } } else { s3Client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build()); s3Client.setEndpoint(endpoint); } java.util.Date expiration = new java.util.Date(); long milliSeconds = expiration.getTime(); milliSeconds += 1000 * 60 * 1000; // Add 1 hour. expiration.setTime(milliSeconds); GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest(bucket, object); generatePresignedUrlRequest.setMethod(HttpMethod.GET); generatePresignedUrlRequest.setExpiration(expiration); URL url = s3Client.generatePresignedUrl(generatePresignedUrlRequest); URL = ("Pre-Signed URL = " + url.toString()); StringSelection stringSelection = new StringSelection(url.toString()); } catch (Exception setACLpublic) { } return URL; }
@Override public void setS3ClientOptions(S3ClientOptions clientOptions) { delegate.setS3ClientOptions(clientOptions); }
@Override public void setS3ClientOptions(S3ClientOptions clientOptions) { getDelegate().setS3ClientOptions(clientOptions); }
@Override public void setS3ClientOptions(S3ClientOptions clientOptions) { }
@Test public void testDefaults() throws Exception { Config config = newConfig(); config.set("_command", BUCKET + "/" + KEY); when(taskRequest.getConfig()).thenReturn(config); ObjectMetadata objectMetadata = new ObjectMetadata(); objectMetadata.setContentType(CONTENT_TYPE); objectMetadata.setContentLength(CONTENT_LENGTH); objectMetadata.setUserMetadata(USER_METADATA); Operator operator = factory.newOperator(newContext(projectPath, taskRequest)); when(s3Client.getObjectMetadata(objectMetadataRequestCaptor.capture())).thenReturn(objectMetadata); TaskResult taskResult = operator.run(); verify(s3ClientFactory).create(credentialsCaptor.capture(), clientConfigurationCaptor.capture()); ClientConfiguration clientConfiguration = clientConfigurationCaptor.getValue(); assertThat(clientConfiguration.getProxyHost(), is(nullValue())); assertThat(clientConfiguration.getProxyPort(), is(-1)); assertThat(clientConfiguration.getProxyUsername(), is(nullValue())); assertThat(clientConfiguration.getProxyPassword(), is(nullValue())); verify(s3Client).setS3ClientOptions(s3ClientOptionsCaptor.capture()); S3ClientOptions s3ClientOptions = s3ClientOptionsCaptor.getValue(); assertThat(s3ClientOptions.isPathStyleAccess(), is(false)); AWSCredentials credentials = credentialsCaptor.getValue(); assertThat(credentials.getAWSAccessKeyId(), is(ACCESS_KEY_ID)); assertThat(credentials.getAWSSecretKey(), is(SECRET_ACCESS_KEY)); GetObjectMetadataRequest objectMetadataRequest = objectMetadataRequestCaptor.getValue(); assertThat(objectMetadataRequest.getKey(), is(KEY)); assertThat(objectMetadataRequest.getBucketName(), is(BUCKET)); assertThat(objectMetadataRequest.getSSECustomerKey(), is(nullValue())); Config expectedStoreParams = newConfig(); expectedStoreParams .getNestedOrSetEmpty("s3") .getNestedOrSetEmpty("last_object") .set("metadata", objectMetadata.getRawMetadata()) .set("user_metadata", objectMetadata.getUserMetadata()); assertThat(taskResult.getStoreParams(), is(expectedStoreParams)); }
@Test public void testTransferManager() throws Exception { AmazonS3Client client = new AmazonS3Client(awsCreds, new ClientConfiguration().withSignerOverride("S3SignerType")); client.setEndpoint(s3Endpoint.toString()); client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true)); int numStreams = 2; int numUploadThreads = 2; int queueCapacity = 2; int partSize = 5; final StreamTransferManager manager = new StreamTransferManager(containerName, key, client, numStreams, numUploadThreads, queueCapacity, partSize) { @Override public void customiseUploadPartRequest(UploadPartRequest request) { /* Workaround from https://github.com/andrewgaul/s3proxy/commit/50a302436271ec46ce81a415b4208b9e14fcaca4 to deal with https://github.com/andrewgaul/s3proxy/issues/80 */ ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentType("application/unknown"); request.setObjectMetadata(metadata); } }; final List<MultiPartOutputStream> streams = manager.getMultiPartOutputStreams(); List<StringBuilder> builders = new ArrayList<StringBuilder>(numStreams); ExecutorService pool = Executors.newFixedThreadPool(numStreams); for (int i = 0; i < numStreams; i++) { final int streamIndex = i; final StringBuilder builder = new StringBuilder(); builders.add(builder); Runnable task = new Runnable() { @Override public void run() { MultiPartOutputStream outputStream = streams.get(streamIndex); for (int lineNum = 0; lineNum < 1000000; lineNum++) { String line = String.format("Stream %d, line %d\n", streamIndex, lineNum); outputStream.write(line.getBytes()); try { outputStream.checkSize(); } catch (InterruptedException e) { throw new RuntimeException(e); } builder.append(line); } outputStream.close(); } }; pool.submit(task); } pool.shutdown(); pool.awaitTermination(5, TimeUnit.SECONDS); manager.complete(); for (int i = 1; i < numStreams; i++) { builders.get(0).append(builders.get(i)); } String expectedResult = builders.get(0).toString(); S3ObjectInputStream objectContent = client.getObject(containerName, key).getObjectContent(); String result = IOUtils.toString(objectContent); IOUtils.closeQuietly(objectContent, null); Assert.assertEquals(expectedResult, result); }
/** * Instead of providing support bundle directly to user, upload it to StreamSets backend services. */ public void uploadNewBundle(List<String> generators) throws IOException { boolean enabled = configuration.get(Constants.UPLOAD_ENABLED, Constants.DEFAULT_UPLOAD_ENABLED); String accessKey = configuration.get(Constants.UPLOAD_ACCESS, Constants.DEFAULT_UPLOAD_ACCESS); String secretKey = configuration.get(Constants.UPLOAD_SECRET, Constants.DEFAULT_UPLOAD_SECRET); String bucket = configuration.get(Constants.UPLOAD_BUCKET, Constants.DEFAULT_UPLOAD_BUCKET); int bufferSize = configuration.get(Constants.UPLOAD_BUFFER_SIZE, Constants.DEFAULT_UPLOAD_BUFFER_SIZE); if(!enabled) { throw new IOException("Uploading support bundles was disabled by administrator."); } AWSCredentialsProvider credentialsProvider = new StaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey)); AmazonS3Client s3Client = new AmazonS3Client(credentialsProvider, new ClientConfiguration()); s3Client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true)); s3Client.setRegion(Region.getRegion(Regions.US_WEST_2)); // Object Metadata ObjectMetadata metadata = new ObjectMetadata(); for(Map.Entry<Object, Object> entry: getMetadata().entrySet()) { metadata.addUserMetadata((String)entry.getKey(), (String)entry.getValue()); } // Generate bundle SupportBundle bundle = generateNewBundle(generators); // Uploading part by part LOG.info("Initiating multi-part support bundle upload"); List<PartETag> partETags = new ArrayList<>(); InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucket, bundle.getBundleKey()); initRequest.setObjectMetadata(metadata); InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); try { byte[] buffer = new byte[bufferSize]; int partId = 1; int size = -1; while ((size = readFully(bundle.getInputStream(), buffer)) != -1) { LOG.debug("Uploading part {} of size {}", partId, size); UploadPartRequest uploadRequest = new UploadPartRequest() .withBucketName(bucket) .withKey(bundle.getBundleKey()) .withUploadId(initResponse.getUploadId()) .withPartNumber(partId++) .withInputStream(new ByteArrayInputStream(buffer)) .withPartSize(size); partETags.add(s3Client.uploadPart(uploadRequest).getPartETag()); } CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest( bucket, bundle.getBundleKey(), initResponse.getUploadId(), partETags ); s3Client.completeMultipartUpload(compRequest); LOG.info("Support bundle upload finished"); } catch (Exception e) { LOG.error("Support bundle upload failed", e); s3Client.abortMultipartUpload(new AbortMultipartUploadRequest( bucket, bundle.getBundleKey(), initResponse.getUploadId()) ); throw new IOException("Can't upload support bundle", e); } finally { // Close the client s3Client.shutdown(); } }
public static void main(String... args) throws IOException { Locale.setDefault(Locale.ENGLISH); final Properties userProperties = UserProperties.fromHome().load("s3pt"); final String accessKey = userProperties.getProperty("accessKey"); final String secretKey = userProperties.getProperty("secretKey"); final String endpoint = userProperties.getProperty("endpoint"); final String bucket = "testbucket"; final String key = "multipart_test.svg"; AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey); ClientConfiguration clientConfiguration = new ClientConfiguration() .withProtocol(Protocol.HTTP) .withUserAgent("MultiPartUpload") .withSignerOverride(Constants.S3_SIGNER_TYPE); AmazonS3 s3Client = new AmazonS3Client(credentials, clientConfiguration); s3Client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).disableChunkedEncoding().build()); s3Client.setEndpoint(endpoint); List<PartETag> partETags = new ArrayList<>(); InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucket, key); InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); File file = new File("/User/jns/flamegraph.svg"); long contentLength = file.length(); long partSize = 32 * 1024; try { // Step 2: Upload parts. long filePosition = 0; for (int i = 1; filePosition < contentLength; i++) { // Last part can be smaller. Adjust part size. partSize = Math.min(partSize, (contentLength - filePosition)); // Create request to upload a part. UploadPartRequest uploadRequest = new UploadPartRequest() .withBucketName(bucket).withKey(key) .withUploadId(initResponse.getUploadId()).withPartNumber(i) .withFileOffset(filePosition) .withFile(file) .withPartSize(partSize); // Upload part and add response to our list. partETags.add(s3Client.uploadPart(uploadRequest).getPartETag()); filePosition += partSize; } // Step 3: Complete. CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucket, key, initResponse.getUploadId(), partETags); s3Client.completeMultipartUpload(compRequest); } catch (Exception e) { s3Client.abortMultipartUpload(new AbortMultipartUploadRequest( bucket, key, initResponse.getUploadId())); } }
@Override public void setS3ClientOptions(S3ClientOptions clientOptions) { throw new TajoInternalError(new UnsupportedException()); }
@Override public void setS3ClientOptions(final S3ClientOptions arg0) { throw new UnsupportedOperationException(); }
public static AmazonS3Client getAmazonS3Client(S3Server s3Server) { AmazonS3Client client = new AmazonS3Client(); client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true)); client.setEndpoint(s3Server.getAddress()); return client; }
public void testCombinations() throws Exception { String filename = System.getProperty("user.home")+"/aws.properties"; PropertiesCredentials credentials = new PropertiesCredentials(new File(filename)); AmazonS3Client client = new AmazonS3Client(credentials); client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true)); client.setRegion(Region.getRegion(Regions.EU_WEST_1)); testBucketName(client,"doesnotrxist.mjt.me.uk"); testBucketName(client,"name-with-hyphens.mjt.me.uk"); testBucketName(client,"name_underscore.mjt.me.uk"); testBucketName(client,"-initial-hyphen.mjt.me.uk"); testBucketName(client,".initial-dot.mjt.me.uk"); testBucketName(client,"CAPITALLETTERS.mjt.me.uk"); testBucketName(client,"dashdota.-.mjt.me.uk"); testBucketName(client,"dashdotb-.mjt.me.uk"); testBucketName(client,"dashdotc.-mjt.me.uk"); testBucketName(client,"?.mjt.me.uk"); testBucketName(client,"§.mjt.me.uk"); testBucketName(client,"\u2605.mjt.me.uk"); testBucketName(client,"\u3070.mjt.me.uk"); testBucketName(client," "); testBucketName(client,"s"); testBucketName(client,""); testBucketName(client,"asdf"); testBucketName(client,"192.168.0.1.mjt.me.uk"); testBucketName(client,"192.168.0"); testBucketName(client,"192.168.0.1"); testBucketName(client,"192.168.0.1.2"); testBucketName(client,"292.168.0.1"); testBucketName(client,"392.168.0.1"); testBucketName(client,"1000.168.0.1"); testBucketName(client,"10000.2.3.4"); testBucketName(client,"100000.2.3.4"); testBucketName(client,"1000000.2.3.4"); testBucketName(client,"1-000.168.0.1"); testBucketName(client,"2001:0db8:85a3:0000:0000:8a2e:0370:7334"); testBucketName(client,"asdfasdfasdfadsfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf.mjt.me.uk"); testBucketName(client,"a"); testBucketName(client,"aa"); testBucketName(client,"aaa"); testBucketName(client,"aaaa"); testBucketName(client,"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); testBucketName(client,"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); testBucketName(client,"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); testBucketName(client,"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); testBucketName(client,"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); }
public void testCombinations() throws Exception { String filename = System.getProperty("user.home")+"/aws.properties"; PropertiesCredentials credentials = new PropertiesCredentials(new File(filename)); AmazonS3Client client = new AmazonS3Client(credentials); client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true)); client.setRegion(Region.getRegion(Regions.EU_WEST_1)); testKey(client,"asdf.txt"); testKey(client," "); testKey(client,"qwer.txt?acl"); testKey(client,"../../../.."); testKey(client,"../../../."); testKey(client,"../../../"); testKey(client,"../../.."); testKey(client,"../../."); testKey(client,"../../"); testKey(client,"../.."); testKey(client,"../."); testKey(client,"../"); testKey(client,".."); testKey(client,"."); testKey(client,"./"); testKey(client,"./."); testKey(client,"././"); testKey(client,"././."); testKey(client,"////"); testKey(client,"/"); testKey(client,"\\\\\\"); testKey(client,"\\"); testKey(client,""); testKey(client,"\0"); testKey(client,"~"); testKey(client,"#"); testKey(client,"@"); testKey(client,";"); testKey(client,":"); testKey(client,"}"); testKey(client,"]"); testKey(client,"{"); testKey(client,"["); testKey(client,"'"); testKey(client,"\""); testKey(client,"+"); testKey(client,"="); testKey(client,"%"); testKey(client,"\u00a5"); testKey(client,"\u030a"); testKey(client,"\r"); testKey(client,"\n"); testKey(client,"\r\t"); testKey(client,"\t"); testKey(client,";"); testKey(client,"?"); testKey(client,"\01"); testKey(client,"\b"); testKey(client,"\uFFFD"); testKey(client,"\u0080"); testKey(client,"\uDC80"); testKey(client,"\uD800"); testKey(client,"\u200F"); testKey(client,"\u009F"); testKey(client,"\u00a0"); testKey(client,"\u2060"); testKey(client,"\uFEFF"); testKey(client,"asdf../.."); testKey(client,"asdf/../.."); testKey(client,"asdf/../../"); testKey(client,"asdf/../../.."); testKey(client,"../..asdf"); testKey(client,"../../asdf"); testKey(client,"..asdf/.."); testKey(client,"..asdf/asdf.."); testKey(client,"../asdf/.."); }
public void run() { AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key); AmazonS3 s3Client = new AmazonS3Client(credentials, new ClientConfiguration()); if (endpoint.contains("amazonaws.com")) { String aws_endpoint = s3Client.getBucketLocation(new GetBucketLocationRequest(bucket)); if (aws_endpoint.contains("US")) { s3Client.setEndpoint("https://s3.amazonaws.com"); } else if (aws_endpoint.contains("us-west")) { s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com"); } else if (aws_endpoint.contains("eu-west")) { s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com"); } else if (aws_endpoint.contains("ap-")) { s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com"); } else if (aws_endpoint.contains("sa-east-1")) { s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com"); } else { s3Client.setEndpoint("https://s3." + aws_endpoint + ".amazonaws.com"); } } else { s3Client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build()); s3Client.setEndpoint(endpoint); } int converted_days = 0; if (!disabled) { converted_days = Integer.parseInt(days); } Transition transToArchive = new Transition() .withDays(converted_days) .withStorageClass(StorageClass.Glacier); BucketLifecycleConfiguration.Rule ruleArchiveAndExpire = null; if (!disabled) { ruleArchiveAndExpire = new BucketLifecycleConfiguration.Rule() .withPrefix(prefix) .withTransition(transToArchive) // .withExpirationInDays(converted_days + 1) .withStatus(BucketLifecycleConfiguration.ENABLED.toString()); } else { ruleArchiveAndExpire = new BucketLifecycleConfiguration.Rule() .withPrefix(prefix) .withTransition(transToArchive) //.withExpirationInDays(100) .withStatus(BucketLifecycleConfiguration.DISABLED.toString()); } List<BucketLifecycleConfiguration.Rule> rules = new ArrayList<BucketLifecycleConfiguration.Rule>(); rules.add(ruleArchiveAndExpire); try { BucketLifecycleConfiguration configuration = new BucketLifecycleConfiguration() .withRules(rules); s3Client.setBucketLifecycleConfiguration(bucket, configuration); } catch (Exception get) { mainFrame.jTextArea1.append("\n" + get.getMessage()); } if (!disabled) { mainFrame.jTextArea1.append("\nSent request to set bucket life cycle to tier to Glacier after: " + converted_days + " day(s). Please observe for any errors."); } else { mainFrame.jTextArea1.append("\nSent request to disable the bucket life cycle. Please observe for any errors."); } calibrate(); }
public void run() { AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key); AmazonS3 s3Client = new AmazonS3Client(credentials, new ClientConfiguration()); if (endpoint.contains("amazonaws.com")) { String aws_endpoint = s3Client.getBucketLocation(new GetBucketLocationRequest(bucket)); if (aws_endpoint.contains("US")) { s3Client.setEndpoint("https://s3.amazonaws.com"); } else if (aws_endpoint.contains("us-west")) { s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com"); } else if (aws_endpoint.contains("eu-west")) { s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com"); } else if (aws_endpoint.contains("ap-")) { s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com"); } else if (aws_endpoint.contains("sa-east-1")) { s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com"); } else { s3Client.setEndpoint("https://s3." + aws_endpoint + ".amazonaws.com"); } } else { s3Client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build()); s3Client.setEndpoint(endpoint); } try { if (version != null) { s3Client.deleteVersion(new DeleteVersionRequest(bucket, what, version)); } else { s3Client.deleteObject(new DeleteObjectRequest(bucket, what)); } if (!debug) { NewJFrame.jTextArea1.append("\nDeleted file: " + what); } calibrate(); } catch (AmazonServiceException ase) { if (NewJFrame.gui) { mainFrame.jTextArea1.append("\n\nError Message: " + ase.getMessage()); mainFrame.jTextArea1.append("\nHTTP Status Code: " + ase.getStatusCode()); mainFrame.jTextArea1.append("\nAWS Error Code: " + ase.getErrorCode()); mainFrame.jTextArea1.append("\nError Type: " + ase.getErrorType()); mainFrame.jTextArea1.append("\nRequest ID: " + ase.getRequestId()); calibrate(); } else { System.out.print("\n\nError Message: " + ase.getMessage()); System.out.print("\nHTTP Status Code: " + ase.getStatusCode()); System.out.print("\nAWS Error Code: " + ase.getErrorCode()); System.out.print("\nError Type: " + ase.getErrorType()); System.out.print("\nRequest ID: " + ase.getRequestId()); } } catch (Exception delete) { } }
String makeBucket(String access_key, String secret_key, String bucket, String endpoint, String region) { String message = null; AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key); AmazonS3 s3Client = new AmazonS3Client(credentials, new ClientConfiguration()); if (endpoint.contains("amazonaws.com")) { s3Client.setEndpoint(endpoint); if (region.length() > 5) { if (region.contains("us-east-1")) { s3Client.setEndpoint("https://s3.amazonaws.com"); } else if (region.contains("us-west")) { s3Client.setEndpoint("https://s3-" + region + ".amazonaws.com"); } else if (region.contains("eu-west")) { s3Client.setEndpoint("https://s3-" + region + ".amazonaws.com"); } else if (region.contains("ap-")) { s3Client.setEndpoint("https://s3-" + region + ".amazonaws.com"); } else if (region.contains("sa-east-1")) { s3Client.setEndpoint("https://s3-" + region + ".amazonaws.com"); } else { s3Client.setEndpoint("https://s3." + region + ".amazonaws.com"); } } } else { s3Client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build()); s3Client.setEndpoint(endpoint); } message = ("\nAttempting to create the bucket. Please view the Bucket list window for an update."); try { if (!(s3Client.doesBucketExist(bucket))) { s3Client.createBucket(new CreateBucketRequest(bucket)); } String bucketLocation = s3Client.getBucketLocation(new GetBucketLocationRequest(bucket)); } catch (AmazonServiceException ase) { if (ase.getMessage().contains("InvalidLocationConstraint")) { s3Client.createBucket(new CreateBucketRequest(bucket, region)); } else { if (NewJFrame.gui) { mainFrame.jTextArea1.append("\n\nError Message: " + ase.getMessage()); mainFrame.jTextArea1.append("\nHTTP Status Code: " + ase.getStatusCode()); mainFrame.jTextArea1.append("\nAWS Error Code: " + ase.getErrorCode()); mainFrame.jTextArea1.append("\nError Type: " + ase.getErrorType()); mainFrame.jTextArea1.append("\nRequest ID: " + ase.getRequestId()); calibrate(); } else { System.out.print("\n\nError Message: " + ase.getMessage()); System.out.print("\nHTTP Status Code: " + ase.getStatusCode()); System.out.print("\nAWS Error Code: " + ase.getErrorCode()); System.out.print("\nError Type: " + ase.getErrorType()); System.out.print("\nRequest ID: " + ase.getRequestId()); } } } return message; }