private String regionForUri(AmazonS3 client, AmazonS3URI uri) { String bucketRegion = client.getBucketLocation(uri.getBucket()); Region region = Region.fromValue(bucketRegion); // S3 doesn't have a US East 1 region, US East 1 is really the region // US Standard. US Standard places the data in either an east coast // or west coast data center geographically closest to you. // SigV4 requires you to mention a region while signing a request // and for the S3's US standard endpoints the value to be used is "us-east-1" // US West 1 has an endpoint and so is treated as a stand alone region, // US East 1 doesn't and so is bundled into US Standard if (region.equals(Region.US_Standard)) { bucketRegion = "us-east-1"; } else { bucketRegion = region.toString(); } return bucketRegion; }
private URI getCreateBucketEndpoint(String requestRegion) { // Route to the default endpoint if they're not trying to specify a different one in the request. if(requestRegion == null || requestRegion.equals(clientRegion) || !clientOptions.isForceGlobalBucketAccessEnabled()) { return endpoint; } // If they enabled global bucket access and they're trying to create a bucket in a region different than the default // one specified when they created the client, it will probably fail because only us-east-1 (actually the global // endpoint) is capable of creating buckets outside of its region. Override the endpoint to which the request // is routed so that it will succeed. com.amazonaws.regions.Region targetRegion = com.amazonaws.regions.Region.getRegion(Regions.fromName(requestRegion)); return new DefaultServiceEndpointBuilder(getEndpointPrefix(), clientConfiguration.getProtocol().toString()).withRegion(targetRegion) .getServiceEndpoint(); }
/** * Specifically made package access for testing. * Used for internal consumption of AWS SDK. * * Tries to determine the service endpoint for the bucket name. * Returns the endpoint configured in the client if the region cannot be determined. */ URI resolveServiceEndpoint(String bucketName) { if (getSignerRegion() != null || isSignerOverridden()) return endpoint; final String regionStr = fetchRegionFromCache(bucketName); final com.amazonaws.regions.Region region = RegionUtils.getRegion(regionStr); if (region == null) { log.warn("Region information for " + regionStr + " is not available. Please upgrade to latest version of AWS Java SDK"); } return region != null ? RuntimeHttpUtils.toUri(region.getServiceEndpoint(S3_SERVICE_NAME), clientConfiguration) : endpoint; }
@Test @Override public void test() throws Exception { for (int i = 1; i <= 2; i++) { Message<?> received = this.messageCollector.forChannel(this.channels.output()) .poll(10, TimeUnit.SECONDS); assertNotNull(received); assertThat(received, hasPayload(new File(this.config.getLocalDir(), i + ".test"))); } assertEquals(2, this.config.getLocalDir().list().length); AWSCredentialsProvider awsCredentialsProvider = TestUtils.getPropertyValue(this.amazonS3, "awsCredentialsProvider", AWSCredentialsProvider.class); AWSCredentials credentials = awsCredentialsProvider.getCredentials(); assertEquals(AWS_ACCESS_KEY, credentials.getAWSAccessKeyId()); assertEquals(AWS_SECRET_KEY, credentials.getAWSSecretKey()); assertEquals(Region.US_GovCloud, this.amazonS3.getRegion()); assertEquals(new URI("https://s3-us-gov-west-1.amazonaws.com"), TestUtils.getPropertyValue(this.amazonS3, "endpoint")); }
/** * Create an amazon bucket in the specified region * @param bucket - The s3 bucket name * @param region - The S3 region the bucket should be created in * @param accessList - The access control list settings for the bucket */ public void createBucket(final String bucket, final Region region, final CannedAccessControlList cannedACL, final AccessControlList accessList){ final CreateBucketRequest request = new CreateBucketRequest(bucket, region); if(cannedACL!=null){ request.withCannedAcl(cannedACL); } if(accessList!=null){ request.withAccessControlList(accessList); } this.client.createBucket(request); }
public Bucket createBucketForInstance(String instanceId, ServiceDefinition service, String planId, String organizationGuid, String spaceGuid) { String bucketName = getBucketNameForInstance(instanceId); logger.info("Creating bucket '{}' for serviceInstanceId '{}'", bucketName, instanceId); Bucket bucket = s3.createBucket(bucketName, Region.fromValue(region)); // TODO allow for additional, custom tagging options BucketTaggingConfiguration bucketTaggingConfiguration = new BucketTaggingConfiguration(); TagSet tagSet = new TagSet(); tagSet.setTag("serviceInstanceId", instanceId); tagSet.setTag("serviceDefinitionId", service.getId()); tagSet.setTag("planId", planId); tagSet.setTag("organizationGuid", organizationGuid); tagSet.setTag("spaceGuid", spaceGuid); bucketTaggingConfiguration.withTagSets(tagSet); s3.setBucketTaggingConfiguration(bucket.getName(), bucketTaggingConfiguration); return bucket; }
private AmazonS3 newGlobalInstance(S3S3CopierOptions s3s3CopierOptions) { HadoopAWSCredentialProviderChain credentialsChain = getCredentialsProviderChain(); AmazonS3ClientBuilder builder = AmazonS3ClientBuilder .standard() .withForceGlobalBucketAccessEnabled(Boolean.TRUE) .withCredentials(credentialsChain); URI s3Endpoint = s3s3CopierOptions.getS3Endpoint(); if (s3Endpoint != null) { EndpointConfiguration endpointConfiguration = new EndpointConfiguration(s3Endpoint.toString(), Region.US_Standard.getFirstRegionId()); builder.withEndpointConfiguration(endpointConfiguration); } return builder.build(); }
@Override public void validate(String name, String value) throws ParameterException { try { Region.fromValue(value); } catch (IllegalArgumentException e) { throw new ParameterException("Parameter " + name + " is not a valid AWS region (found " + value + ")", e); } }
@Test public void typical() { Configuration conf = new Configuration(); conf.set(ConfigurationVariable.REGION.getName(), "eu-west-1"); conf.setInt(ConfigurationVariable.UPLOAD_RETRY_COUNT.getName(), 7); conf.setLong(ConfigurationVariable.UPLOAD_RETRY_DELAY_MS.getName(), 333L); AmazonS3 client = factory.newInstance(conf); assertThat(client, is(instanceOf(AmazonS3Client.class))); assertThat(client.getRegion(), is(Region.EU_Ireland)); }
@Before public void setup() { EndpointConfiguration endpoint = new EndpointConfiguration(UNIT_STACK_URL + ":" + SNS_PORT, Region.EU_Frankfurt.name()); AWSCredentials credentials = new BasicAWSCredentials("key", "secret"); AWSCredentialsProvider credentialsProvider = new AWSStaticCredentialsProvider(credentials); sns = AmazonSNSAsyncClientBuilder.standard().withEndpointConfiguration(endpoint) .withCredentials(credentialsProvider).build(); }
/** * @deprecated use {@link AmazonS3ClientBuilder#setRegion(String)} */ @Override @Deprecated public synchronized void setRegion(com.amazonaws.regions.Region region) { super.setRegion(region); /* * We need to preserve the user provided region. This is because the * region might be mapped to a global s3 endpoint (e.g. when the client * is in accelerate mode), in which case we won't be able to extract the * region back from the endpoint during request signing phase. */ clientRegion = region.getName(); }
@Override public synchronized Region getRegion() { String authority = super.endpoint.getAuthority(); if (Constants.S3_HOSTNAME.equals(authority)) { return Region.US_Standard; } else { Matcher m = Region.S3_REGIONAL_ENDPOINT_PATTERN.matcher(authority); if (m.matches()) { return Region.fromValue(m.group(1)); } else { throw new IllegalStateException( "S3 client with invalid S3 endpoint configured: " + authority); } } }
@Override public String getRegionName() { String authority = super.endpoint.getAuthority(); if(Constants.S3_HOSTNAME.equals(authority)) { return "us-east-1"; } Matcher m = Region.S3_REGIONAL_ENDPOINT_PATTERN.matcher(authority); try { m.matches(); return RegionUtils.getRegion(m.group(1)).getName(); } catch (Exception e) { throw new IllegalStateException("No valid region has been specified. Unable to return region name", e); } }
/** * To reuse code for accountbased and external buckets * @param s3 * @param parentConf * @param bucketName * @throws IOException */ private FileSystem addBucketFSEntry(AmazonS3 s3, Configuration parentConf, String bucketName) throws AmazonS3Exception, IOException { final String bucketRegion = s3.getBucketLocation(bucketName); final String projectedBucketEndPoint = "s3." + bucketRegion + ".amazonaws.com"; String regionEndPoint = projectedBucketEndPoint; try { Region region = Region.fromValue(bucketRegion); com.amazonaws.regions.Region awsRegion = region.toAWSRegion(); if (awsRegion != null) { regionEndPoint = awsRegion.getServiceEndpoint("s3"); } } catch (IllegalArgumentException iae) { // try heuristic mapping if not found regionEndPoint = projectedBucketEndPoint; logger.warn("Unknown or unmapped region {} for bucket {}. Will use following fs.s3a.endpoint: {}", bucketRegion, bucketName, regionEndPoint); } // it could be null because no mapping from Region to aws region or there is no such region is the map of endpoints // not sure if latter is possible if (regionEndPoint == null) { logger.error("Could not get AWSRegion for bucket {}. Will use following fs.s3a.endpoint: " + "{} ", bucketName, projectedBucketEndPoint); } String location = S3_URI_SCHEMA + bucketName + "/"; Configuration bucketConf = new Configuration(parentConf); bucketConf.set(ENDPOINT, (regionEndPoint != null) ? regionEndPoint : projectedBucketEndPoint); FileSystem.setDefaultUri(bucketConf, new Path(location).toUri()); return FileSystem.get(bucketConf); }
public Region getTeamAvatarAWSS3BucketRegion() { if (!isTeamAvatarUsingAWSS3()) { throw new UnsupportedOperationException("Team avatar is not using AWS S3"); } if (teamAvatarAWSS3BucketRegion != null) { return teamAvatarAWSS3BucketRegion; } if (globalAWSS3Region != null) { return globalAWSS3Region; } throw new RuntimeException("Missing aws.global.s3.bucket.regionId or aws.teamAvatar.s3.bucket.regionId in"); }
public Region getSubmissionAWSS3BucketRegion() { if (!isSubmissionUsingAWSS3()) { throw new UnsupportedOperationException("Submission is not using AWS S3"); } if (submissionAWSS3BucketRegion != null) { return submissionAWSS3BucketRegion; } if (globalAWSS3Region != null) { return globalAWSS3Region; } throw new RuntimeException("Missing aws.global.s3.bucket.regionId or aws.submission.s3.bucket.regionId"); }
public Region getFileAWSS3BucketRegion() { if (!isFileUsingAWSS3()) { throw new UnsupportedOperationException("File is not using AWS S3"); } if (fileAWSS3BucketRegion != null) { return fileAWSS3BucketRegion; } if (globalAWSS3Region != null) { return globalAWSS3Region; } throw new RuntimeException("Missing aws.global.s3.bucket.regionId or aws.file.s3.bucket.regionId"); }
public S3ShortLinkRepository(final PropertyResolver props) { String awsRegion; String awsKey; String awsSecret; enabled = true; try { bucket = props.getProperty("shortlink.s3.bucket", String.class); log.info("Short linking will use S3 bucket: " + bucket); awsKey = props.getProperty("shortlink.s3.s3key", String.class); awsSecret = props.getProperty("shortlink.s3.s3secret", String.class); if (awsKey == null || awsSecret == null) { log.warn("No AWS key or Secret found. Using Anonymous access."); client = new AmazonS3Client(); } else { client = new AmazonS3Client(new BasicAWSCredentials(awsKey, awsSecret)); } boolean exists = client.doesBucketExist(bucket); if (! exists) { awsRegion = props.getProperty("aws.s3.region", String.class, Region.US_Standard.toString()); client.createBucket(bucket, awsRegion); } log.info("S3ShortLinkRepository initialized"); } catch (Exception e) { log.info("Failed to initialize the S3 client. Shortlinking disabled.", e); enabled = false; } }
public S3QueryCache(PropertyResolver props) { String awsRegion; String awsKey; String awsSecret; enabled = true; try { bucket = props.getProperty("query.cache.s3.bucket", String.class); awsKey = props.getProperty("query.cache.s3.s3key", String.class); awsSecret = props.getProperty("query.cache.s3.s3secret", String.class); if (awsKey == null || awsSecret == null) { log.warn("No AWS key or Secret found. Using Anonymous access."); client = new AmazonS3Client(); } else { client = new AmazonS3Client(new BasicAWSCredentials(awsKey, awsSecret)); } boolean exists = client.doesBucketExist(bucket); if (! exists) { awsRegion = props.getProperty("aws.s3.region", String.class, Region.US_Standard.toString()); client.createBucket(bucket, awsRegion); } } catch (Exception e) { log.info("Failed to initialize the S3 client. Caching disabled.", e); enabled = false; } }
@Override public Optional<Boolean> createBucket(Bucket bucket) { try { CreateBucketRequest bucketRequest = new CreateBucketRequest(bucket.getBucketName(), Region.US_West); bucketRequest.withCannedAcl(CannedAccessControlList.PublicRead); s3Client.createBucket(bucketRequest); return TRUE; } catch (Exception e) { log.error("Error creating bucket", e); return FALSE; } }
@Override public Bucket createBucket(String bucketName, Region region) throws AmazonClientException, AmazonServiceException { throw new UnsupportedOperationException(); }
void setRegion(String region) { Region.fromValue(region); this.region = region; }
@Override @Deprecated public Bucket createBucket(String bucketName, Region region) throws SdkClientException, AmazonServiceException { return createBucket(new CreateBucketRequest(bucketName, region)); }
@Override public Bucket createBucket(CreateBucketRequest createBucketRequest) throws SdkClientException, AmazonServiceException { createBucketRequest = beforeClientExecution(createBucketRequest); rejectNull(createBucketRequest, "The CreateBucketRequest parameter must be specified when creating a bucket"); String bucketName = createBucketRequest.getBucketName(); rejectNull(bucketName, "The bucket name parameter must be specified when creating a bucket"); bucketName = bucketName.trim(); String requestRegion = createBucketRequest.getRegion(); URI requestEndpoint = getCreateBucketEndpoint(requestRegion); BucketNameUtils.validateBucketName(bucketName); Request<CreateBucketRequest> request = createRequest(bucketName, null, createBucketRequest, HttpMethodName.PUT, requestEndpoint); //Add IBM Service Instance Id & Encryption to headers if ((null != this.awsCredentialsProvider ) && (this.awsCredentialsProvider.getCredentials() instanceof IBMOAuthCredentials)) { IBMOAuthCredentials oAuthCreds = (IBMOAuthCredentials)this.awsCredentialsProvider.getCredentials(); if (oAuthCreds.getServiceInstanceId() != null) { request.addHeader(Headers.IBM_SERVICE_INSTANCE_ID, oAuthCreds.getServiceInstanceId()); if (null != createBucketRequest.getEncryptionType()) { request.addHeader(Headers.IBM_SSE_KP_ENCRYPTION_ALGORITHM, createBucketRequest.getEncryptionType().getKmsEncryptionAlgorithm()); request.addHeader(Headers.IBM_SSE_KP_CUSTOMER_ROOT_KEY_CRN, createBucketRequest.getEncryptionType().getIBMSSEKMSCustomerRootKeyCrn()); } } } if (createBucketRequest.getAccessControlList() != null) { addAclHeaders(request, createBucketRequest.getAccessControlList()); } else if (createBucketRequest.getCannedAcl() != null) { request.addHeader(Headers.S3_CANNED_ACL, createBucketRequest.getCannedAcl().toString()); } /* * If we're talking to a region-specific endpoint other than the US, we * *must* specify a location constraint. Try to derive the region from * the endpoint. */ if (getSignerRegion() != null && !getSignerRegion().equals("us-east-1") && StringUtils.isNullOrEmpty(requestRegion)) { requestRegion = AwsHostNameUtils.parseRegion(requestEndpoint.getHost(), S3_SERVICE_NAME); } /* * We can only send the CreateBucketConfiguration if we're *not* * creating a bucket in the US region. */ if (requestRegion != null && !StringUtils.upperCase(requestRegion).equals(Region.US_Standard.toString())) { XmlWriter xml = new XmlWriter(); xml.start("CreateBucketConfiguration", "xmlns", Constants.XML_NAMESPACE); xml.start("LocationConstraint").value(requestRegion).end(); xml.end(); request.setContent(new ByteArrayInputStream(xml.getBytes())); } invoke(request, voidResponseHandler, bucketName, null); return new Bucket(bucketName); }
protected final void createBucket(final String name, final String policy) { this.s3.createBucket(new CreateBucketRequest(name, Region.fromValue(this.getRegion()))); this.s3.setBucketPolicy(name, policy); }
public Region getAvatarAWSS3BucketRegion() { if (!isAvatarUsingAWSS3()) { throw new UnsupportedOperationException("Avatar is not using AWS S3"); } return avatarAWSS3BucketRegion; }
private void build() { jophielBaseUrl = requireStringValue("jophiel.baseUrl"); jophielBaseDataDir = requireDirectoryValue("jophiel.baseDataDir"); jophielClientLabels = requireStringListValue("jophiel.client.labels"); jophielClientTargets = requireStringListValue("jophiel.client.targets"); idTokenPrivateKey = requireStringValue("jophiel.idToken.key.private"); noreplyName = requireStringValue("noreply.name"); noreplyEmail = requireStringValue("noreply.email"); requireStringValue("play.mailer.host"); requireIntegerValue("play.mailer.port"); requireStringValue("play.mailer.ssl"); requireStringValue("play.mailer.user"); requireStringValue("play.mailer.password"); avatarUsingAWSS3 = requireBooleanValue("aws.avatar.s3.use"); if (avatarUsingAWSS3) { avatarAWSUsingKeys = requireBooleanValue("aws.avatar.key.use"); avatarAWSAccessKey = requireStringValue("aws.avatar.key.access"); avatarAWSSecretKey = requireStringValue("aws.avatar.key.secret"); avatarAWSS3BucketName = requireStringValue("aws.avatar.s3.bucket.name"); avatarAWSS3BucketRegion = Region.fromValue(requireStringValue("aws.avatar.s3.bucket.regionId")); avatarAWSCloudFrontUrl = requireStringValue("aws.avatar.cloudFront.baseUrl"); } else { try { avatarLocalDir = new File(jophielBaseDataDir, "avatar"); FileUtils.forceMkdir(avatarLocalDir); } catch (IOException e) { throw new RuntimeException(e); } } registrationUsingRecaptcha = requireBooleanValue("recaptcha.registration.use"); if (registrationUsingRecaptcha) { registrationRecaptchaSiteKey = requireStringValue("recaptcha.registration.key.site"); registrationRecaptchaSecretKey = requireStringValue("recaptcha.registration.key.secret"); } }
@Test public void getUrl_existingObject_returnsUrlWithS3Prefix() throws Exception { AmazonS3Client amazonS3 = mock(AmazonS3Client.class); when(amazonS3.getRegion()).thenReturn(Region.EU_Ireland); //Act SimpleStorageResource simpleStorageResource = new SimpleStorageResource(amazonS3, "bucket", "object", new SyncTaskExecutor()); //Assert assertEquals(new URL("https://s3.eu-west-1.amazonaws.com/bucket/object"), simpleStorageResource.getURL()); }
/** * Create an amazon bucket. This defaults to US_STANDARD regions * @param bucket - The s3 bucket name */ public void createBucket(final String bucket){ this.createBucket(bucket, Region.US_Standard, null); }
/** * Create an amazon bucket. This defaults to US_STANDARD region * with the specified canned access control list. * * @param bucket - The s3 bucket name * @param cannedACL - Canned Access Control List */ public void createBucket(final String bucket, final CannedAccessControlList cannedACL){ this.createBucket(bucket, Region.US_Standard, cannedACL, null); }
/** * Create an amazon bucket within the given region with the specified canned access control list * * @param bucket - The s3 bucket name * @param region * @param cannedACL - Canned Access Control List */ public void createBucket(final String bucket, final Region region, final CannedAccessControlList cannedACL){ this.createBucket(bucket, region, cannedACL, null); }
/** * Create an amazon bucket. This defaults to US_STANDARD regions * * @param bucket - The s3 bucket name * @param accessControlList */ public void createBucket(final String bucket, final AccessControlList accessControlList){ this.createBucket(bucket, Region.US_Standard, null, accessControlList); }