protected void launchAWSConfigurationTest(Settings settings, Protocol expectedProtocol, String expectedProxyHost, int expectedProxyPort, String expectedProxyUsername, String expectedProxyPassword, String expectedSigner, int expectedReadTimeout) { ClientConfiguration configuration = AwsEc2ServiceImpl.buildConfiguration(logger, settings); assertThat(configuration.getResponseMetadataCacheSize(), is(0)); assertThat(configuration.getProtocol(), is(expectedProtocol)); assertThat(configuration.getProxyHost(), is(expectedProxyHost)); assertThat(configuration.getProxyPort(), is(expectedProxyPort)); assertThat(configuration.getProxyUsername(), is(expectedProxyUsername)); assertThat(configuration.getProxyPassword(), is(expectedProxyPassword)); assertThat(configuration.getSignerOverride(), is(expectedSigner)); assertThat(configuration.getSocketTimeout(), is(expectedReadTimeout)); }
public void testAWSConfigurationWithAwsSettingsBackcompat() { Settings settings = Settings.builder() .put(AwsS3Service.PROTOCOL_SETTING.getKey(), "http") .put(AwsS3Service.PROXY_HOST_SETTING.getKey(), "aws_proxy_host") .put(AwsS3Service.PROXY_PORT_SETTING.getKey(), 8080) .put(AwsS3Service.PROXY_USERNAME_SETTING.getKey(), "aws_proxy_username") .put(AwsS3Service.PROXY_PASSWORD_SETTING.getKey(), "aws_proxy_password") .put(AwsS3Service.SIGNER_SETTING.getKey(), "AWS3SignerType") .put(AwsS3Service.READ_TIMEOUT.getKey(), "10s") .build(); launchAWSConfigurationTest(settings, Settings.EMPTY, Protocol.HTTP, "aws_proxy_host", 8080, "aws_proxy_username", "aws_proxy_password", "AWS3SignerType", 3, false, 10000); assertSettingDeprecationsAndWarnings(new Setting<?>[]{ AwsS3Service.PROXY_USERNAME_SETTING, AwsS3Service.PROXY_PASSWORD_SETTING, AwsS3Service.PROTOCOL_SETTING, AwsS3Service.PROXY_HOST_SETTING, AwsS3Service.PROXY_PORT_SETTING, AwsS3Service.SIGNER_SETTING, AwsS3Service.READ_TIMEOUT}); }
@Bean public ClientConfiguration awsClientConfig(final ProxyDetails proxyDetails) { ClientConfiguration clientConfig = new ClientConfiguration(); if (useProxy) { clientConfig.setProxyHost(clientProxyHost); clientConfig.setProxyPort(clientProxyPort); } else if(proxyDetails != null) { clientConfig.setProxyHost(proxyDetails.getHost()); clientConfig.setProxyPort(proxyDetails.getPort()); } clientConfig.setProtocol(Protocol.valueOf(clientProtocol.toUpperCase())); clientConfig.setConnectionTimeout(clientConnectionTimeout); clientConfig.setMaxErrorRetry(clientMaxErrorRetry); return clientConfig; }
public static com.amazonaws.ClientConfiguration transformAndVerifyOrThrow(ClientConfiguration clientConfiguration) { com.amazonaws.ClientConfiguration awsClientConfig = new com.amazonaws.ClientConfigurationFactory().getConfig(); if (awsClientConfig.getProtocol() != Protocol.HTTPS) { throw new SecurityConfigurationException("Must use HTTPS protocol"); } clientConfiguration.proxy.ifPresent(p -> { awsClientConfig.setProxyHost(p.proxyHost); awsClientConfig.setProxyPort(p.proxyPort); if (!p.nonProxyHosts.isEmpty()) { awsClientConfig.setNonProxyHosts(String.join("|", p.nonProxyHosts)); } p.proxyUsername.ifPresent(awsClientConfig::setProxyUsername); p.proxyPassword.ifPresent(awsClientConfig::setProxyPassword); }); return verifyOrThrow(awsClientConfig); }
/** * Returns an URI for the given endpoint. * Prefixes the protocol if the endpoint given does not have it. * * @throws IllegalArgumentException if the inputs are null. */ public static URI toUri(String endpoint, Protocol protocol) { if (endpoint == null) { throw new IllegalArgumentException("endpoint cannot be null"); } /* * If the endpoint doesn't explicitly specify a protocol to use, then * we'll defer to the default protocol specified in the client * configuration. */ if (!endpoint.contains("://")) { endpoint = protocol.toString() + "://" + endpoint; } try { return new URI(endpoint); } catch (URISyntaxException e) { throw new IllegalArgumentException(e); } }
@Override public AmazonS3Client load(S3ClientKey clientKey) throws Exception { logger.debug("Opening S3 client connection for {}", clientKey); ClientConfiguration clientConf = new ClientConfiguration(); clientConf.setProtocol(clientKey.isSecure ? Protocol.HTTPS : Protocol.HTTP); // Proxy settings (if configured) clientConf.setProxyHost(clientKey.s3Config.get(Constants.PROXY_HOST)); if (clientKey.s3Config.get(Constants.PROXY_PORT) != null) { clientConf.setProxyPort(Integer.valueOf(clientKey.s3Config.get(Constants.PROXY_PORT))); } clientConf.setProxyDomain(clientKey.s3Config.get(Constants.PROXY_DOMAIN)); clientConf.setProxyUsername(clientKey.s3Config.get(Constants.PROXY_USERNAME)); clientConf.setProxyPassword(clientKey.s3Config.get(Constants.PROXY_PASSWORD)); clientConf.setProxyWorkstation(clientKey.s3Config.get(Constants.PROXY_WORKSTATION)); if (clientKey.accessKey == null){ return new AmazonS3Client(new AnonymousAWSCredentialsProvider(), clientConf); } else { return new AmazonS3Client(new BasicAWSCredentials(clientKey.accessKey, clientKey.secretKey), clientConf); } }
/** * S3 储存客户端 * * @return 客户端 */ @Bean @ConditionalOnProperty(value = "bigbug.storage.s3.enable", havingValue = "true") AmazonS3Client amazonS3Client() { ClientConfiguration clientConfig = new ClientConfiguration(); clientConfig.setProtocol(Protocol.HTTP); BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials( storageProperties.getStorage().getS3().getAccessKey(), storageProperties.getStorage().getS3().getSecretKey()); return (AmazonS3Client) AmazonS3ClientBuilder.standard() .withClientConfiguration(clientConfig) .withEndpointConfiguration( new AwsClientBuilder.EndpointConfiguration( storageProperties.getStorage().getS3().getEndpoint(), Regions.DEFAULT_REGION.getName())) .withCredentials(new AWSStaticCredentialsProvider(basicAWSCredentials)) .build(); }
private void connectToS3(String accessKey, String secretKey, String endpoint, boolean secure, boolean pathStyle) { LOG.info("'{}','{}','{}'", accessKey, secretKey, endpoint); AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey); ClientConfiguration clientConfiguration = new ClientConfiguration() .withProtocol(secure ? Protocol.HTTPS : Protocol.HTTP) .withUserAgentPrefix("s3browser") .withSignerOverride("S3SignerType"); s3Client = new AmazonS3Client(credentials, clientConfiguration); s3Client.setS3ClientOptions(S3ClientOptions.builder() .setPathStyleAccess(pathStyle) .disableChunkedEncoding() .build()); s3Client.setEndpoint(endpoint); }
/** * @return S3 client */ private AmazonS3 buildS3Client() { AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey); ClientConfiguration clientConfiguration = new ClientConfiguration() .withProtocol(useHttp ? Protocol.HTTP : Protocol.HTTPS) .withUserAgent("s3pt") .withGzip(useGzip) .withTcpKeepAlive(useKeepAlive); if (signerOverride != null) { String signer = signerOverride.endsWith("Type") ? signerOverride : signerOverride + "Type"; clientConfiguration.setSignerOverride(signer); } AmazonS3 s3Client = new AmazonS3Client(credentials, clientConfiguration); s3Client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(usePathStyleAccess).disableChunkedEncoding().build()); s3Client.setEndpoint(endpointUrl); return s3Client; }
public static ClientConfiguration getClientConfig(Parameters config){ ClientConfiguration clientConfig = new ClientConfiguration(); clientConfig.setProtocol(Protocol.valueOf(config.get("aws.protocol").toString().toUpperCase())); String host = config.get("aws.proxy.host").toString(); if (host != null && host.length() > 0){ clientConfig.setProxyHost(host); clientConfig.setProxyPort(Integer.valueOf(config.get("aws.proxy.port").toString())); String username = config.get("aws.proxy.username").toString(); String password = config.get("aws.proxy.password").toString(); if (username != null && username.length() > 0) clientConfig.setProxyUsername(username); if (password != null && password.length() > 0) clientConfig.setProxyPassword(password); } return clientConfig; }
private ClientConfiguration buildClientConfig() { final String userAgent = System.getProperty( PROP_S3_HANDLER_USER_AGENT, null); final String protocol = System.getProperty( PROP_S3_HANDLER_PROTOCOL, "https"); final String signerOverride = System.getProperty( PROP_S3_HANDLER_SIGNER_OVERRIDE, null); final ClientConfiguration clientConfig = new ClientConfiguration().withProtocol("https" .equalsIgnoreCase(protocol) ? Protocol.HTTPS : Protocol.HTTP); if (userAgent != null) { clientConfig.setUserAgent(userAgent); } if (signerOverride != null) { clientConfig.setSignerOverride(signerOverride); } return clientConfig; }
private AmazonS3 getS3Client() throws Exception{ this.bucketName = properties.getProperty( "usergrid.binary.bucketname" ); if(bucketName == null){ logger.error( "usergrid.binary.bucketname not properly set so amazon bucket is null" ); throw new AwsPropertiesNotFoundException( "usergrid.binary.bucketname" ); } final UsergridAwsCredentialsProvider ugProvider = new UsergridAwsCredentialsProvider(); AWSCredentials credentials = ugProvider.getCredentials(); ClientConfiguration clientConfig = new ClientConfiguration(); clientConfig.setProtocol(Protocol.HTTP); s3Client = new AmazonS3Client(credentials, clientConfig); if(regionName != null) s3Client.setRegion( Region.getRegion(Regions.fromName(regionName)) ); return s3Client; }
private void copyToS3( String fileName ) { String bucketName = ( String ) properties.get( BUCKET_PROPNAME ); String accessId = ( String ) properties.get( ACCESS_ID_PROPNAME ); String secretKey = ( String ) properties.get( SECRET_KEY_PROPNAME ); Properties overrides = new Properties(); overrides.setProperty( "s3" + ".identity", accessId ); overrides.setProperty( "s3" + ".credential", secretKey ); final Iterable<? extends Module> MODULES = ImmutableSet .of( new JavaUrlHttpCommandExecutorServiceModule(), new Log4JLoggingModule(), new NettyPayloadModule() ); AWSCredentials credentials = new BasicAWSCredentials(accessId, secretKey); ClientConfiguration clientConfig = new ClientConfiguration(); clientConfig.setProtocol( Protocol.HTTP); AmazonS3Client s3Client = new AmazonS3Client(credentials, clientConfig); s3Client.createBucket( bucketName ); File uploadFile = new File( fileName ); PutObjectResult putObjectResult = s3Client.putObject( bucketName, uploadFile.getName(), uploadFile ); logger.info("Uploaded file etag={}", putObjectResult.getETag()); }
/** * We test when only cloud.aws settings are set */ public void testRepositorySettingsGlobalOnly() { Settings nodeSettings = buildSettings(AWS); assertThat(AwsEc2Service.CLOUD_EC2.KEY_SETTING.get(nodeSettings), is("global-key")); assertThat(AwsEc2Service.CLOUD_EC2.SECRET_SETTING.get(nodeSettings), is("global-secret")); assertThat(AwsEc2Service.CLOUD_EC2.PROTOCOL_SETTING.get(nodeSettings), is(Protocol.HTTPS)); assertThat(AwsEc2Service.CLOUD_EC2.PROXY_HOST_SETTING.get(nodeSettings), is("global-proxy-host")); assertThat(AwsEc2Service.CLOUD_EC2.PROXY_PORT_SETTING.get(nodeSettings), is(10000)); assertThat(AwsEc2Service.CLOUD_EC2.PROXY_USERNAME_SETTING.get(nodeSettings), is("global-proxy-username")); assertThat(AwsEc2Service.CLOUD_EC2.PROXY_PASSWORD_SETTING.get(nodeSettings), is("global-proxy-password")); assertThat(AwsEc2Service.CLOUD_EC2.SIGNER_SETTING.get(nodeSettings), is("global-signer")); assertThat(AwsEc2Service.CLOUD_EC2.REGION_SETTING.get(nodeSettings), is("global-region")); assertThat(AwsEc2Service.CLOUD_EC2.ENDPOINT_SETTING.get(nodeSettings), isEmptyString()); }
/** * We test when cloud.aws settings are overloaded by cloud.aws.ec2 settings */ public void testRepositorySettingsGlobalOverloadedByEC2() { Settings nodeSettings = buildSettings(AWS, EC2); assertThat(AwsEc2Service.CLOUD_EC2.KEY_SETTING.get(nodeSettings), is("ec2-key")); assertThat(AwsEc2Service.CLOUD_EC2.SECRET_SETTING.get(nodeSettings), is("ec2-secret")); assertThat(AwsEc2Service.CLOUD_EC2.PROTOCOL_SETTING.get(nodeSettings), is(Protocol.HTTP)); assertThat(AwsEc2Service.CLOUD_EC2.PROXY_HOST_SETTING.get(nodeSettings), is("ec2-proxy-host")); assertThat(AwsEc2Service.CLOUD_EC2.PROXY_PORT_SETTING.get(nodeSettings), is(20000)); assertThat(AwsEc2Service.CLOUD_EC2.PROXY_USERNAME_SETTING.get(nodeSettings), is("ec2-proxy-username")); assertThat(AwsEc2Service.CLOUD_EC2.PROXY_PASSWORD_SETTING.get(nodeSettings), is("ec2-proxy-password")); assertThat(AwsEc2Service.CLOUD_EC2.SIGNER_SETTING.get(nodeSettings), is("ec2-signer")); assertThat(AwsEc2Service.CLOUD_EC2.REGION_SETTING.get(nodeSettings), is("ec2-region")); assertThat(AwsEc2Service.CLOUD_EC2.ENDPOINT_SETTING.get(nodeSettings), is("ec2-endpoint")); }
public void testAWSConfigurationWithAwsSettings() { Settings settings = Settings.builder() .put(AwsEc2Service.PROTOCOL_SETTING.getKey(), "http") .put(AwsEc2Service.PROXY_HOST_SETTING.getKey(), "aws_proxy_host") .put(AwsEc2Service.PROXY_PORT_SETTING.getKey(), 8080) .put(AwsEc2Service.PROXY_USERNAME_SETTING.getKey(), "aws_proxy_username") .put(AwsEc2Service.PROXY_PASSWORD_SETTING.getKey(), "aws_proxy_password") .put(AwsEc2Service.SIGNER_SETTING.getKey(), "AWS3SignerType") .put(AwsEc2Service.READ_TIMEOUT.getKey(), "10s") .build(); launchAWSConfigurationTest(settings, Protocol.HTTP, "aws_proxy_host", 8080, "aws_proxy_username", "aws_proxy_password", "AWS3SignerType", 10000); }
public void testAWSConfigurationWithAwsAndEc2Settings() { Settings settings = Settings.builder() .put(AwsEc2Service.PROTOCOL_SETTING.getKey(), "http") .put(AwsEc2Service.PROXY_HOST_SETTING.getKey(), "aws_proxy_host") .put(AwsEc2Service.PROXY_PORT_SETTING.getKey(), 8080) .put(AwsEc2Service.PROXY_USERNAME_SETTING.getKey(), "aws_proxy_username") .put(AwsEc2Service.PROXY_PASSWORD_SETTING.getKey(), "aws_proxy_password") .put(AwsEc2Service.SIGNER_SETTING.getKey(), "AWS3SignerType") .put(AwsEc2Service.CLOUD_EC2.PROTOCOL_SETTING.getKey(), "https") .put(AwsEc2Service.CLOUD_EC2.PROXY_HOST_SETTING.getKey(), "ec2_proxy_host") .put(AwsEc2Service.CLOUD_EC2.PROXY_PORT_SETTING.getKey(), 8081) .put(AwsEc2Service.CLOUD_EC2.PROXY_USERNAME_SETTING.getKey(), "ec2_proxy_username") .put(AwsEc2Service.CLOUD_EC2.PROXY_PASSWORD_SETTING.getKey(), "ec2_proxy_password") .put(AwsEc2Service.CLOUD_EC2.SIGNER_SETTING.getKey(), "NoOpSignerType") .put(AwsEc2Service.CLOUD_EC2.READ_TIMEOUT.getKey(), "10s") .build(); launchAWSConfigurationTest(settings, Protocol.HTTPS, "ec2_proxy_host", 8081, "ec2_proxy_username", "ec2_proxy_password", "NoOpSignerType", 10000); }
public void testAWSConfigurationWithAwsSettings() { MockSecureSettings secureSettings = new MockSecureSettings(); secureSettings.setString("s3.client.default.proxy.username", "aws_proxy_username"); secureSettings.setString("s3.client.default.proxy.password", "aws_proxy_password"); Settings settings = Settings.builder() .setSecureSettings(secureSettings) .put("s3.client.default.protocol", "http") .put("s3.client.default.proxy.host", "aws_proxy_host") .put("s3.client.default.proxy.port", 8080) .put("s3.client.default.read_timeout", "10s") .build(); launchAWSConfigurationTest(settings, Settings.EMPTY, Protocol.HTTP, "aws_proxy_host", 8080, "aws_proxy_username", "aws_proxy_password", null, 3, false, 10000); }
public void testAWSConfigurationWithAwsAndS3SettingsBackcompat() { Settings settings = Settings.builder() .put(AwsS3Service.PROTOCOL_SETTING.getKey(), "http") .put(AwsS3Service.PROXY_HOST_SETTING.getKey(), "aws_proxy_host") .put(AwsS3Service.PROXY_PORT_SETTING.getKey(), 8080) .put(AwsS3Service.PROXY_USERNAME_SETTING.getKey(), "aws_proxy_username") .put(AwsS3Service.PROXY_PASSWORD_SETTING.getKey(), "aws_proxy_password") .put(AwsS3Service.SIGNER_SETTING.getKey(), "AWS3SignerType") .put(AwsS3Service.READ_TIMEOUT.getKey(), "5s") .put(AwsS3Service.CLOUD_S3.PROTOCOL_SETTING.getKey(), "https") .put(AwsS3Service.CLOUD_S3.PROXY_HOST_SETTING.getKey(), "s3_proxy_host") .put(AwsS3Service.CLOUD_S3.PROXY_PORT_SETTING.getKey(), 8081) .put(AwsS3Service.CLOUD_S3.PROXY_USERNAME_SETTING.getKey(), "s3_proxy_username") .put(AwsS3Service.CLOUD_S3.PROXY_PASSWORD_SETTING.getKey(), "s3_proxy_password") .put(AwsS3Service.CLOUD_S3.SIGNER_SETTING.getKey(), "NoOpSignerType") .put(AwsS3Service.CLOUD_S3.READ_TIMEOUT.getKey(), "10s") .build(); launchAWSConfigurationTest(settings, Settings.EMPTY, Protocol.HTTPS, "s3_proxy_host", 8081, "s3_proxy_username", "s3_proxy_password", "NoOpSignerType", 3, false, 10000); assertSettingDeprecationsAndWarnings(new Setting<?>[] { AwsS3Service.PROXY_USERNAME_SETTING, AwsS3Service.PROXY_PASSWORD_SETTING, AwsS3Service.PROTOCOL_SETTING, AwsS3Service.PROXY_HOST_SETTING, AwsS3Service.PROXY_PORT_SETTING, AwsS3Service.SIGNER_SETTING, AwsS3Service.READ_TIMEOUT, AwsS3Service.CLOUD_S3.PROXY_USERNAME_SETTING, AwsS3Service.CLOUD_S3.PROXY_PASSWORD_SETTING, AwsS3Service.CLOUD_S3.PROTOCOL_SETTING, AwsS3Service.CLOUD_S3.PROXY_HOST_SETTING, AwsS3Service.CLOUD_S3.PROXY_PORT_SETTING, AwsS3Service.CLOUD_S3.SIGNER_SETTING, AwsS3Service.CLOUD_S3.READ_TIMEOUT}); }
public void testGlobalMaxRetries() { Settings settings = Settings.builder() .put(S3Repository.Repositories.MAX_RETRIES_SETTING.getKey(), 10) .build(); launchAWSConfigurationTest(settings, Settings.EMPTY, Protocol.HTTPS, null, -1, null, null, null, 10, false, 50000); }
public void testRepositoryMaxRetries() { Settings repositorySettings = generateRepositorySettings(null, null, null, 20); Settings settings = Settings.builder() .put(S3Repository.Repositories.MAX_RETRIES_SETTING.getKey(), 10) .build(); launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTPS, null, -1, null, null, null, 20, false, 50000); }
protected void launchAWSConfigurationTest(Settings settings, Settings singleRepositorySettings, Protocol expectedProtocol, String expectedProxyHost, int expectedProxyPort, String expectedProxyUsername, String expectedProxyPassword, String expectedSigner, Integer expectedMaxRetries, boolean expectedUseThrottleRetries, int expectedReadTimeout) { Integer maxRetries = S3Repository.getValue(singleRepositorySettings, settings, S3Repository.Repository.MAX_RETRIES_SETTING, S3Repository.Repositories.MAX_RETRIES_SETTING); Boolean useThrottleRetries = S3Repository.getValue(singleRepositorySettings, settings, S3Repository.Repository.USE_THROTTLE_RETRIES_SETTING, S3Repository.Repositories.USE_THROTTLE_RETRIES_SETTING); ClientConfiguration configuration = InternalAwsS3Service.buildConfiguration(logger, singleRepositorySettings, settings, "default", maxRetries, null, useThrottleRetries); assertThat(configuration.getResponseMetadataCacheSize(), is(0)); assertThat(configuration.getProtocol(), is(expectedProtocol)); assertThat(configuration.getProxyHost(), is(expectedProxyHost)); assertThat(configuration.getProxyPort(), is(expectedProxyPort)); assertThat(configuration.getProxyUsername(), is(expectedProxyUsername)); assertThat(configuration.getProxyPassword(), is(expectedProxyPassword)); assertThat(configuration.getSignerOverride(), is(expectedSigner)); assertThat(configuration.getMaxErrorRetry(), is(expectedMaxRetries)); assertThat(configuration.useThrottledRetries(), is(expectedUseThrottleRetries)); assertThat(configuration.getSocketTimeout(), is(expectedReadTimeout)); }
@Bean public ClientConfiguration awsClientConfig() { ClientConfiguration clientConfig = new ClientConfiguration(); if (useProxy) { clientConfig.setProxyHost(clientProxyHost); clientConfig.setProxyPort(clientProxyPort); } clientConfig.setProtocol(Protocol.valueOf(clientProtocol.toUpperCase())); clientConfig.setConnectionTimeout(clientConnectionTimeout); clientConfig.setMaxErrorRetry(clientMaxErrorRetry); return clientConfig; }
protected static AmazonS3 createS3(String accessKey, String secretKey) { AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey); ClientConfiguration clientConfig = new ClientConfiguration(); clientConfig.setProtocol(Protocol.HTTP); clientConfig.withSignerOverride("S3SignerType"); //noinspection deprecation AmazonS3 s3 = new AmazonS3Client(credentials, clientConfig); s3.setEndpoint(s3Endpoint); return s3; }
/** * Gets an Amazon S3 client from basic session credentials. * * @return an authenticated Amazon S3 amazonS3 */ public AmazonS3 getAmazonS3Client() { if (amazonS3 == null) { amazonS3 = AmazonS3ClientBuilder.standard() .withEndpointConfiguration(new EndpointConfiguration(endpoint, region)) .withClientConfiguration(new ClientConfiguration().withProtocol(Protocol.HTTP)) .withCredentials( new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKeyId, accessKeySecret))) .build(); } return amazonS3; }
private void assertHttps() { URI endpoint = this.endpoint; String scheme = endpoint == null ? null : endpoint.getScheme(); if (!Protocol.HTTPS.toString().equalsIgnoreCase(scheme)) { throw new IllegalArgumentException( "HTTPS must be used when sending customer encryption keys (SSE-C) to S3, in order to protect your encryption keys."); } }
public @NotNull ClientConfiguration createClientConfiguration(String proxyUrl) throws Exception { ClientConfiguration clientConfiguration = new ClientConfiguration(); URL url = new URL(proxyUrl); String userInfo = url.getUserInfo(); if (userInfo != null) { String username = url.getUserInfo(); final int password_idx = userInfo.indexOf(':'); if (password_idx != -1) { username = userInfo.substring(0, password_idx); final String password = userInfo.substring(userInfo.indexOf(':') + 1); clientConfiguration.setProxyPassword(password); } final int domain_idx = username.indexOf('\\'); if (domain_idx != -1) { final String domain = username.substring(0, domain_idx); username = username.substring(domain_idx + 1); clientConfiguration.setProxyDomain(domain); } clientConfiguration.setProxyUsername(username); } clientConfiguration.setProtocol(Protocol.HTTPS.toString().equals(url.getProtocol()) ? Protocol.HTTPS : Protocol.HTTP); clientConfiguration.setProxyHost(url.getHost()); clientConfiguration.setProxyPort(url.getPort()); return clientConfiguration; }
public byte[] getFromS3Proxy(String name, String key, String secret, String endpoint) throws Exception { System.setProperty("com.amazonaws.services.s3.disablePutObjectMD5Validation", "1"); AWSCredentials credentials = new BasicAWSCredentials(key, secret); ClientConfiguration clientConfig = new ClientConfiguration().withSignerOverride("S3SignerType"); clientConfig.setProtocol(Protocol.HTTP); AmazonS3 s3 = new AmazonS3Client(credentials, clientConfig); s3.setEndpoint(endpoint); S3Object object = s3.getObject( new GetObjectRequest(containerName, name)); InputStream objectData = object.getObjectContent(); return IOUtils.toByteArray(objectData); }
@Test public void testNoProxy() throws Exception { setUp(); clientConfig = storage.newClientConfiguration(connectorConfig); assertEquals(Protocol.HTTPS, clientConfig.getProtocol()); assertEquals(null, clientConfig.getProxyHost()); assertEquals(-1, clientConfig.getProxyPort()); assertEquals(null, clientConfig.getProxyUsername()); assertEquals(null, clientConfig.getProxyPassword()); }
@Test public void testProtocolOnly() throws Exception { localProps.put(S3SinkConnectorConfig.S3_PROXY_URL_CONFIG, "http://"); setUp(); clientConfig = storage.newClientConfiguration(connectorConfig); assertEquals(Protocol.HTTP, clientConfig.getProtocol()); assertEquals("", clientConfig.getProxyHost()); assertEquals(-1, clientConfig.getProxyPort()); assertEquals(null, clientConfig.getProxyUsername()); assertEquals(null, clientConfig.getProxyPassword()); }
@Test public void testProtocolHostOnly() throws Exception { localProps.put(S3SinkConnectorConfig.S3_PROXY_URL_CONFIG, "http://localhost"); setUp(); clientConfig = storage.newClientConfiguration(connectorConfig); assertEquals(Protocol.HTTP, clientConfig.getProtocol()); assertEquals("localhost", clientConfig.getProxyHost()); assertEquals(-1, clientConfig.getProxyPort()); assertEquals(null, clientConfig.getProxyUsername()); assertEquals(null, clientConfig.getProxyPassword()); }
@Test public void testProtocolIpOnly() throws Exception { localProps.put(S3SinkConnectorConfig.S3_PROXY_URL_CONFIG, "http://127.0.0.1"); setUp(); clientConfig = storage.newClientConfiguration(connectorConfig); assertEquals(Protocol.HTTP, clientConfig.getProtocol()); assertEquals("127.0.0.1", clientConfig.getProxyHost()); assertEquals(-1, clientConfig.getProxyPort()); assertEquals(null, clientConfig.getProxyUsername()); assertEquals(null, clientConfig.getProxyPassword()); }
@Test public void testUnknownHostOnly() throws Exception { localProps.put(S3SinkConnectorConfig.S3_PROXY_URL_CONFIG, "http://255.255.255.255"); setUp(); clientConfig = storage.newClientConfiguration(connectorConfig); assertEquals(Protocol.HTTP, clientConfig.getProtocol()); assertEquals("255.255.255.255", clientConfig.getProxyHost()); assertEquals(-1, clientConfig.getProxyPort()); assertEquals(null, clientConfig.getProxyUsername()); assertEquals(null, clientConfig.getProxyPassword()); }
@Test public void testProtocolPort() throws Exception { localProps.put(S3SinkConnectorConfig.S3_PROXY_URL_CONFIG, "http://:8080"); setUp(); clientConfig = storage.newClientConfiguration(connectorConfig); assertEquals(Protocol.HTTP, clientConfig.getProtocol()); assertEquals("", clientConfig.getProxyHost()); assertEquals(8080, clientConfig.getProxyPort()); assertEquals(null, clientConfig.getProxyUsername()); assertEquals(null, clientConfig.getProxyPassword()); }
@Test public void testProtocolHostPort() throws Exception { localProps.put(S3SinkConnectorConfig.S3_PROXY_URL_CONFIG, "http://localhost:8080"); setUp(); clientConfig = storage.newClientConfiguration(connectorConfig); assertEquals(Protocol.HTTP, clientConfig.getProtocol()); assertEquals("localhost", clientConfig.getProxyHost()); assertEquals(8080, clientConfig.getProxyPort()); assertEquals(null, clientConfig.getProxyUsername()); assertEquals(null, clientConfig.getProxyPassword()); }
@Test public void testProtocolHostPortAndPath() throws Exception { localProps.put(S3SinkConnectorConfig.S3_PROXY_URL_CONFIG, "http://localhost:8080/current"); setUp(); clientConfig = storage.newClientConfiguration(connectorConfig); assertEquals(Protocol.HTTP, clientConfig.getProtocol()); assertEquals("localhost", clientConfig.getProxyHost()); assertEquals(8080, clientConfig.getProxyPort()); assertEquals(null, clientConfig.getProxyUsername()); assertEquals(null, clientConfig.getProxyPassword()); }
@Test public void testProtocolHostPortUserOnUrlEmptyPass() throws Exception { localProps.put(S3SinkConnectorConfig.S3_PROXY_URL_CONFIG, "http://user:@localhost:8080"); setUp(); clientConfig = storage.newClientConfiguration(connectorConfig); assertEquals(Protocol.HTTP, clientConfig.getProtocol()); assertEquals("localhost", clientConfig.getProxyHost()); assertEquals(8080, clientConfig.getProxyPort()); assertEquals("user", clientConfig.getProxyUsername()); assertEquals("", clientConfig.getProxyPassword()); }