Java 类org.apache.http.impl.conn.DefaultProxyRoutePlanner 实例源码

@Override
public void customize(RestTemplate restTemplate) {
    HttpHost proxy = new HttpHost("proxy.example.com");
    HttpClient httpClient = HttpClientBuilder.create()
            .setRoutePlanner(new DefaultProxyRoutePlanner(proxy) {

                @Override
                public HttpHost determineProxy(HttpHost target,
                        HttpRequest request, HttpContext context)
                                throws HttpException {
                    if (target.getHostName().equals("192.168.0.5")) {
                        return null;
                    }
                    return super.determineProxy(target, request, context);
                }

            }).build();
    restTemplate.setRequestFactory(
            new HttpComponentsClientHttpRequestFactory(httpClient));
}

public void initialize(){
    requestConfig = RequestConfig.custom().setConnectTimeout(httpConnectTimeout).setSocketTimeout(httpReadTimeout).build();
    if (StringUtils.isNotBlank(proxyScheme) && StringUtils.isNotBlank(proxyHost) && proxyPort != null) {
        System.out.println("Initialising proxy settings");
        HttpHost proxy = new HttpHost(proxyHost, proxyPort, proxyScheme);
        DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
        httpClient = HttpClientBuilder.create().setDefaultRequestConfig(requestConfig).setRoutePlanner(routePlanner).build();           

    } else {
        httpClient = HttpClientBuilder.create().setDefaultRequestConfig(requestConfig).build();
    }

    String implementationVersion = this.getClass().getPackage().getImplementationVersion();
    if (implementationVersion != null && implementationVersion.length() > 0){
        clientVersion = implementationVersion;
    }
    customUserAgentHeaderValue = format("sentenial-java/%s", clientVersion);
}

/**
 * Get a HTTP client. With proxy if a proxy is provided in the constructor.
 * @return a HTTP client
 */
protected HttpClient getHttpClient(){

    HttpClientBuilder builder = HttpClientBuilder.create();

    if(proxyHost != null && proxyPort != 0) {
        HttpHost proxy = new HttpHost(proxyHost, proxyPort);
        DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
        builder.setRoutePlanner(routePlanner);
    }

    RequestConfig.Builder config = RequestConfig.custom()
            .setConnectTimeout(timeout)
            .setConnectionRequestTimeout(timeout)
            .setSocketTimeout(timeout);

    builder.setDefaultRequestConfig(config.build());

    return builder.build();
}

private QueryExecutor(String userAgent, ObjectMapper mapper, HttpHost proxy)
{
    checkNotNull(userAgent, "userAgent is null");
    checkNotNull(mapper, "mapper is null");

    this.userAgent = userAgent;
    this.mapper = mapper;

    HttpClientBuilder builder = HttpClients.custom();
    HttpAsyncClientBuilder asyncBuilder = HttpAsyncClients.custom();

    if (proxy != null) {
        DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
        builder.setRoutePlanner(routePlanner);
        asyncBuilder.setRoutePlanner(routePlanner);
    }

    this.httpClient = asyncBuilder.build();
    this.httpClient.start();
}

/**
 * Creates asynchronous Apache HTTP client.
 *
 * @param settings
 *         settings to use to create client.
 * @param conf
 *         configuration related to async connection.
 * @return Instance of {@link CloseableHttpAsyncClient}.
 */
private CloseableHttpAsyncClient createClient(HttpSettings settings, ApacheHttpClientConfiguration conf) {
    IOReactorConfig ioReactor = IOReactorConfig.custom().setIoThreadCount(conf.getMaxThreadCount()).build();
    HttpAsyncClientBuilder httpClientBuilder = HttpAsyncClients.custom()
            .useSystemProperties()
                    // allow POST redirects
            .setRedirectStrategy(new LaxRedirectStrategy()).setMaxConnTotal(conf.getMaxTotalConnectionCount()).setMaxConnPerRoute(conf.getMaxRouteConnectionCount()).setDefaultIOReactorConfig(ioReactor)
            .setKeepAliveStrategy(new DefaultConnectionKeepAliveStrategy()).setDefaultRequestConfig(createDefaultRequestConfig(settings));
    if (settings.getProxyUrl() != null) {
        DefaultProxyRoutePlanner routePlanner = createProxyRoutePlanner(settings, httpClientBuilder);
        httpClientBuilder.setRoutePlanner(routePlanner);
    }
    CloseableHttpAsyncClient httpClient = httpClientBuilder.build();
    httpClient.start();
    return httpClient;
}

/**
 * Creates a HttpClientBuilder with the default settings of GERBIL.
 * 
 * @return a HttpClientBuilder with the default settings of GERBIL.
 */
public HttpClientBuilder generateHttpClientBuilder() {
    HttpClientBuilder builder = HttpClientBuilder.create();
    builder.setUserAgent(userAgent);

    String proxyHost = GerbilConfiguration.getInstance().getString(PROXY_HOST_KEY);
    int proxyPort = GerbilConfiguration.getInstance().getInt(PROXY_PORT_KEY, DEFAULT_PROXY_PORT);

    if (proxyHost != null) {
        HttpHost proxy = new HttpHost(proxyHost, proxyPort);
        DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
        builder.setRoutePlanner(routePlanner);
    }

    return builder;
}

public static CloseableHttpClient getHttpClient (HasProxySettings proxySetting) {
    // http://hc.apache.org/httpcomponents-client-ga/tutorial/html/connmgmt.html#d5e475

    CloseableHttpClient httpclient = null ;
    if (proxySetting != null && proxySetting.isActive()) {
        logger.info("Set the http proxy (" + proxySetting.getHost() + ":" + proxySetting.getPort() + ")") ;
        CredentialsProvider credsProvider = Preconditions.checkNotNull(proxySetting.getCredentialsProvider()) ;
        HttpHost proxy = new HttpHost(proxySetting.getHost(), proxySetting.getPort());
        DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
        httpclient = HttpClients.custom()
                .setRoutePlanner(routePlanner).setDefaultCredentialsProvider(credsProvider)
                .build();
    } else {
        httpclient = HttpClients.createDefault();
    }
    return httpclient ;
}

protected void configureProxy(final HttpClientBuilder httpClientBuilder, final String proxy)
    throws InitializationException {
    final URL proxyUrl;
    try {
        proxyUrl = new URL(proxy);
    } catch (final MalformedURLException e) {
        throw new InitializationException("Invalid proxy url", e);
    }

    final String proxyHost = proxyUrl.getHost();
    final int proxyPort = proxyUrl.getPort() != -1
        ? proxyUrl.getPort()
        : proxyUrl.getDefaultPort();

    LOG.info("Set proxy server to '{}:{}'", proxyHost, proxyPort);
    httpClientBuilder.setRoutePlanner(new DefaultProxyRoutePlanner(new HttpHost(proxyHost, proxyPort)));

    final String userInfo = proxyUrl.getUserInfo();
    if (userInfo != null) {
        final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        credentialsProvider.setCredentials(new AuthScope(proxyHost, proxyPort), buildCredentials(userInfo));

        httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
    }
}

/**
 * プロパティの設定が終了したあとにHttpClientのインスタンスを生成し、プロキシの設定を行います。
 * {@inheritDoc}
 */
@Override
public void afterPropertiesSet() throws Exception {

    Assert.notNull(proxyHost, "プロキシホスト(proxyHost)は必須です。");
    Assert.notNull(proxyPort, "プロキシポート番号(proxyPort)は必須です。");

    PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
    connectionManager.setMaxTotal(maxTotal);
    connectionManager.setDefaultMaxPerRoute(defaultMaxPerRoute);

    HttpClientBuilder builder = HttpClients.custom();
    builder.setConnectionManager(connectionManager);
    if (authentication) {
        Assert.notNull(username, "ユーザ認証がtrueに設定された場合、ユーザ名(username)は必須です。");
        Assert.notNull(password, "ユーザ認証がtrueに設定された場合、パスワード(password)は必須です。");
        DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(new HttpHost(proxyHost, Integer.parseInt(proxyPort)));
        builder.setRoutePlanner(routePlanner);
        CredentialsProvider credsProvider = new BasicCredentialsProvider();
        credsProvider.setCredentials(new AuthScope(proxyHost, Integer.parseInt(proxyPort)), new UsernamePasswordCredentials(username, password));
        builder.setDefaultCredentialsProvider(credsProvider);
    }
    builder.setDefaultRequestConfig(RequestConfig.custom().setSocketTimeout(readTimeout).build());
    CloseableHttpClient client = builder.build();
    setHttpClient(client);
}

private RestTemplate createRestTemplate() {
    // jackson message converter
    MappingJackson2HttpMessageConverter mappingJacksonHttpMessageConverter = new MappingJackson2HttpMessageConverter();
    mappingJacksonHttpMessageConverter.setObjectMapper(objectMapper);

    // create message converters list
    List<HttpMessageConverter<?>> httpMessageConverters = new ArrayList<HttpMessageConverter<?>>();
    httpMessageConverters.add(mappingJacksonHttpMessageConverter);

    // create rest template
    RestTemplate restTemplate = new RestTemplate();
    restTemplate.setMessageConverters(httpMessageConverters);

    // configure proxy
    HttpHost httpHost = new HttpHost(System.getProperty("http.proxyHost"), Integer.parseInt(System.getProperty("http.proxyPort")), "http");
    DefaultProxyRoutePlanner defaultProxyRoutePlanner = new DefaultProxyRoutePlanner(httpHost);
    HttpClient httpClient = HttpClients.custom().setRoutePlanner(defaultProxyRoutePlanner).build();
    restTemplate.setRequestFactory(new HttpComponentsClientHttpRequestFactory(httpClient));

    return restTemplate;
}

public ClientHttpRequestFactory createRequestFactory(HttpProxyConfiguration httpProxyConfiguration, boolean
        trustSelfSignedCerts) {
    HttpClientBuilder httpClientBuilder = HttpClients.custom().useSystemProperties();

    if (trustSelfSignedCerts) {
        httpClientBuilder.setSslcontext(buildSslContext());
        httpClientBuilder.setHostnameVerifier(BROWSER_COMPATIBLE_HOSTNAME_VERIFIER);
    }

    if (httpProxyConfiguration != null) {
        HttpHost proxy = new HttpHost(httpProxyConfiguration.getProxyHost(), httpProxyConfiguration.getProxyPort());
        httpClientBuilder.setProxy(proxy);

        if (httpProxyConfiguration.isAuthRequired()) {
            BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
            credentialsProvider.setCredentials(
                    new AuthScope(httpProxyConfiguration.getProxyHost(), httpProxyConfiguration.getProxyPort()),
                    new UsernamePasswordCredentials(httpProxyConfiguration.getUsername(), httpProxyConfiguration
                            .getPassword()));
            httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
        }

        HttpRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
        httpClientBuilder.setRoutePlanner(routePlanner);
    }

    HttpClient httpClient = httpClientBuilder.build();
    HttpComponentsClientHttpRequestFactory requestFactory = new HttpComponentsClientHttpRequestFactory(httpClient);

    return requestFactory;
}

/**
 * Adds proxy to the {@link HttpClient} created by this builder
 *
 * @param proxy A {@link Proxy} object to get the connection info from
 * @return this object
 */
public SafechargeClientBuilder setProxy(Proxy proxy) {

    if (proxy != null) {
        HttpHost proxyHost = new HttpHost(proxy.getHost(), proxy.getPort(), proxy.getProtocol());
        this.routePlanner = new DefaultProxyRoutePlanner(proxyHost);
    }

    return this;
}

public CloseableHttpClient getHttpClient(HttpHost proxy) {
    CloseableHttpClient client;
    if (proxy == null) {
        client = HttpClients.createDefault();
    } else {
        DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
        client = HttpClients.custom().setRoutePlanner(routePlanner).build();
    }
    return client;
}

@Override
public HttpClientBuilder build(final TranscriptListener listener, final LoginCallback prompt) {
    final HttpClientBuilder builder = super.build(listener, prompt);
    // Add filter to inject custom headers to authenticate with proxy
    builder.setRequestExecutor(
            new CustomHeaderHttpRequestExecutor(headers)
    );
    // Set proxy router planer
    builder.setRoutePlanner(new DefaultProxyRoutePlanner(
            new HttpHost(proxy.getHostname(), proxy.getPort(), proxy.getProtocol().getScheme().name()),
            new DefaultSchemePortResolver()));
    return builder;
}

private HttpClient getHttpClient() {
    HttpClient client;
    if (proxyHost != null)
    {
        client = HttpClientBuilder.create().setRoutePlanner(new DefaultProxyRoutePlanner(proxyHost)).build();
    }
    else
    {
        client = HttpClientBuilder.create().build();
    }
    return client;
}

private void addProxy(RESTPool pool, HttpClientBuilder builder) {
    if (pool.getProxy() == null) return;

    Proxy proxy = pool.getProxy();

    if (proxy.getUsername() != null) {
        CredentialsProvider provider = makeProxyCredentialsProvider(proxy);
        builder.setDefaultCredentialsProvider(provider);
    }

    HttpHost proxyHost = new HttpHost(proxy.getHostname(), proxy.getPort());
    builder.setRoutePlanner(new DefaultProxyRoutePlanner(proxyHost));
}

private void addProxy(RESTPool pool, HttpAsyncClientBuilder builder) {
    if (pool.getProxy() == null) return;

    Proxy proxy = pool.getProxy();

    if (proxy.getUsername() != null) {
        CredentialsProvider provider = makeProxyCredentialsProvider(proxy);
        builder.setDefaultCredentialsProvider(provider);
    }

    HttpHost proxyHost = new HttpHost(proxy.getHostname(), proxy.getPort());
    builder.setRoutePlanner(new DefaultProxyRoutePlanner(proxyHost));
}

@Test
public void expectedSendGridBeanWithProxyCreated() {
    loadContext("spring.sendgrid.username:user", "spring.sendgrid.password:secret",
            "spring.sendgrid.proxy.host:localhost",
            "spring.sendgrid.proxy.port:5678");
    SendGrid sendGrid = this.context.getBean(SendGrid.class);
    assertThat(sendGrid).extracting("client").extracting("routePlanner")
            .hasOnlyElementsOfType(DefaultProxyRoutePlanner.class);
}

private  HttpClientPojo(ProxyPojo proxyPojo) {
            if (proxyPojo == null) {
                this.proxy = null;
//                logger.info("proxyPojo is null");
                this.httpClient = HttpClients.custom().build();
            } else {
                this.proxyPojo = proxyPojo;
                this.proxy = new HttpHost(proxyPojo.getIp(),proxyPojo.getPort());
                if(proxyPojo.isAuthEnable()){
                    //需要验证的
                    CredentialsProvider credsProvider = new BasicCredentialsProvider();
                    credsProvider.setCredentials(new AuthScope(proxyPojo.getIp(),
                            proxyPojo.getPort()), new UsernamePasswordCredentials(
                            proxyPojo.getUsername(), proxyPojo.getPassword()));
                    this.httpClient = HttpClients.custom().setDefaultCredentialsProvider(credsProvider).build();
                }else{
                    //无需验证的
                    DefaultProxyRoutePlanner routePlanner= new DefaultProxyRoutePlanner(this.proxy);
                    this.httpClient = HttpClients.custom().setRoutePlanner(routePlanner).build();
                }
                RequestConfig.Builder config_builder = RequestConfig.custom();
                config_builder.setProxy(proxy);
                config_builder.setSocketTimeout(StaticValue.http_connection_timeout);
                config_builder.setConnectTimeout(StaticValue.http_read_timeout);
                this.requestConfig = config_builder.build();
            }
        }

public static boolean TestProxyByBaidu(String proxyString){
        String testString = "030173";
        boolean reflag = true;
        String ip = proxyString.split(":")[0];
        int port = Integer.valueOf(proxyString.split(":")[1]);
        try{
            String url = "http://www.baidu.com/";
            HttpClient httpClient = HttpClients.custom().build();
            //211.144.72.154:8080
            HttpHost proxy = new HttpHost(ip, port);
            //无需验证的
            DefaultProxyRoutePlanner routePlanner= new DefaultProxyRoutePlanner(proxy);
            httpClient = HttpClients.custom().setRoutePlanner(routePlanner).build();
            RequestConfig.Builder config_builder = RequestConfig.custom();
            config_builder.setProxy(proxy);
            config_builder.setSocketTimeout(StaticValue.http_connection_timeout);
            config_builder.setConnectTimeout(StaticValue.http_read_timeout);
            RequestConfig requestConfig = config_builder.build();

            RequestBuilder rb = null;
            rb = RequestBuilder.get().setUri(URI.create(url));

            HttpUriRequest requestAll = null;
            rb.setConfig(requestConfig);
            requestAll = rb.build();
            CloseableHttpResponse response = (CloseableHttpResponse)httpClient.execute(requestAll);
            String re = Crawl4HttpClient.parserResponse_v2(response);
//            System.out.println(Crawl4HttpClient.parserResponse_v2(response).substring(0,100));
            if(!re.contains(testString)){
                reflag = false;
                System.out.println(re);
            }

        }catch (Exception ex){
            ex.printStackTrace();
            reflag = false;
        }

        return reflag;
    }

private HttpRoutePlanner getRoutePlanner() {
  proxyHostname = Strings.nullToEmpty(proxyHostname);
  if (proxyHostname.isEmpty()) {
    proxyEnabled = false;
  }
  if (proxyEnabled) {
    HttpHost proxy = new HttpHost(proxyHostname, proxyPort, proxyScheme);

    HttpRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy) {

      @Override
      public HttpRoute determineRoute(
              final HttpHost host,
              final HttpRequest request,
              final HttpContext context) throws HttpException {
        String hostname = host.getHostName();
        if (hostname.equals("127.0.0.1") || hostname.equalsIgnoreCase("localhost")) {
          // Return direct route
          return new HttpRoute(host);
        }
        return super.determineRoute(host, request, context);
      }
    };
    return routePlanner;
  }
  return null;
}

private HttpClient getHttpClient()
{
    HttpClient client = null;
    if (proxyHost != null)
    {
        client = HttpClientBuilder.create().setRoutePlanner(new DefaultProxyRoutePlanner(proxyHost)).build();
    }
    else
    {
        client = HttpClientBuilder.create().build();
    }
    return client;
}

@Test
public void expectedSendGridBeanWithProxyCreated() {
    loadContext("spring.sendgrid.username:user", "spring.sendgrid.password:secret",
            "spring.sendgrid.proxy.host:localhost",
            "spring.sendgrid.proxy.port:5678");
    SendGrid sendGrid = this.context.getBean(SendGrid.class);
    assertThat(sendGrid).extracting("client").extracting("routePlanner")
            .hasOnlyElementsOfType(DefaultProxyRoutePlanner.class);
}

public static ClientHttpRequestFactory createRequestFactory(HttpProxyConfiguration httpProxyConfiguration, boolean trustSelfSignedCerts, boolean disableRedirectHandling) {
    HttpClientBuilder httpClientBuilder = HttpClients.custom().useSystemProperties();

    if (trustSelfSignedCerts) {
        httpClientBuilder.setSslcontext(buildSslContext());
        httpClientBuilder.setHostnameVerifier(BROWSER_COMPATIBLE_HOSTNAME_VERIFIER);
    }

    if (disableRedirectHandling) {
        httpClientBuilder.disableRedirectHandling();
    }

    if (httpProxyConfiguration != null) {
        HttpHost proxy = new HttpHost(httpProxyConfiguration.getProxyHost(), httpProxyConfiguration.getProxyPort());
        httpClientBuilder.setProxy(proxy);

        if (httpProxyConfiguration.isAuthRequired()) {
            BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
            credentialsProvider.setCredentials(
                    new AuthScope(httpProxyConfiguration.getProxyHost(), httpProxyConfiguration.getProxyPort()),
                    new UsernamePasswordCredentials(httpProxyConfiguration.getUsername(), httpProxyConfiguration.getPassword()));
            httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
        }

        HttpRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
        httpClientBuilder.setRoutePlanner(routePlanner);
    }

    HttpClient httpClient = httpClientBuilder.build();
    HttpComponentsClientHttpRequestFactory requestFactory = new HttpComponentsClientHttpRequestFactory(httpClient);

    return requestFactory;
}

@Override
public void setProxy(String proxyhost, int proxyport) {
    if (StringUtils.isNotBlank(proxyhost)) {
        HttpHost proxy = new HttpHost(proxyhost, proxyport);
        DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
        httpclient = HttpClients.custom().setSSLSocketFactory(sslsf).setRoutePlanner(routePlanner).build();
        proxyOn = true;
    } else if (proxyOn){
        httpclient = HttpClients.custom().setSSLSocketFactory(sslsf).build();
        proxyOn = false;
    }
}

@Test
public void expectedSendGridBeanWithProxyCreated() {
    loadContext("spring.sendgrid.username:user", "spring.sendgrid.password:secret",
            "spring.sendgrid.proxy.host:localhost",
            "spring.sendgrid.proxy.port:5678");
    SendGrid sendGrid = this.context.getBean(SendGrid.class);
    CloseableHttpClient client = (CloseableHttpClient) ReflectionTestUtils
            .getField(sendGrid, "client");
    HttpRoutePlanner routePlanner = (HttpRoutePlanner) ReflectionTestUtils
            .getField(client, "routePlanner");
    assertThat(routePlanner, instanceOf(DefaultProxyRoutePlanner.class));
}

private HttpClient buildHttpClient() {
    HttpClientBuilder builder = HttpClientBuilder.create();
    builder.setUserAgent(CallfireClient.getClientConfig().getProperty(USER_AGENT_PROPERTY));
    String proxyAddress = CallfireClient.getClientConfig().getProperty(PROXY_ADDRESS_PROPERTY);
    String proxyCredentials = CallfireClient.getClientConfig().getProperty(PROXY_CREDENTIALS_PROPERTY);

    if (isNotBlank(proxyAddress)) {
        LOGGER.debug("Configuring proxy host for client: {} auth: {}", proxyAddress, proxyCredentials);
        String[] parsedAddress = proxyAddress.split(":");
        String[] parsedCredentials = StringUtils.split(defaultString(proxyCredentials), ":");
        HttpHost proxy = new HttpHost(parsedAddress[0],
            parsedAddress.length > 1 ? toInt(parsedAddress[1], DEFAULT_PROXY_PORT) : DEFAULT_PROXY_PORT);
        if (isNotBlank(proxyCredentials)) {
            if (parsedCredentials.length > 1) {
                CredentialsProvider provider = new BasicCredentialsProvider();
                provider.setCredentials(
                    new AuthScope(proxy),
                    new UsernamePasswordCredentials(parsedCredentials[0], parsedCredentials[1])
                );
                builder.setDefaultCredentialsProvider(provider);
            } else {
                LOGGER.warn("Proxy credentials have wrong format, must be username:password");
            }
        }

        builder.setRoutePlanner(new DefaultProxyRoutePlanner(proxy));
    }

    return builder.build();
}

public void prepareHttpClient() {
    HttpClientBuilder builder = HttpClients.custom();
    if (proxyHost != null && proxyPort != null) {
        HttpHost proxy = new HttpHost(proxyHost, proxyPort);
        builder.setRoutePlanner(new DefaultProxyRoutePlanner(proxy));
    }

    builder.setUserAgent("org.emonocot.ws.GetResourceClient");
    httpClient = builder.build();
}

@Override
public void configure(final Config conf) {
    this.maxContent = ConfUtils.getInt(conf, "http.content.limit",
            64 * 1024);
    String userAgent = getAgentString(
            ConfUtils.getString(conf, "http.agent.name"),
            ConfUtils.getString(conf, "http.agent.version"),
            ConfUtils.getString(conf, "http.agent.description"),
            ConfUtils.getString(conf, "http.agent.url"),
            ConfUtils.getString(conf, "http.agent.email"));

    this.responseTime = ConfUtils.getBoolean(conf,
            "http.store.responsetime", true);

    this.skipRobots = ConfUtils.getBoolean(conf, "http.skip.robots", false);

    robots = new HttpRobotRulesParser(conf);

    builder = HttpClients.custom().setUserAgent(userAgent)
            .setConnectionManager(CONNECTION_MANAGER)
            .setConnectionManagerShared(true).disableRedirectHandling();

    String proxyHost = ConfUtils.getString(conf, "http.proxy.host", null);
    int proxyPort = ConfUtils.getInt(conf, "http.proxy.port", 8080);

    boolean useProxy = (proxyHost != null && proxyHost.length() > 0);

    // use a proxy?
    if (useProxy) {
        HttpHost proxy = new HttpHost(proxyHost, proxyPort);
        DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(
                proxy);
        builder.setRoutePlanner(routePlanner);
    }

    int timeout = ConfUtils.getInt(conf, "http.timeout", 10000);
    requestConfig = RequestConfig.custom().setSocketTimeout(timeout)
            .setConnectTimeout(timeout).build();
}

/**
 * Creates default proxy route planner.
 *
 * @param settings
 *         settings to use.
 * @param httpClientBuilder
 *         http client builder.
 * @return Instance of {@link DefaultProxyRoutePlanner}.
 */
private DefaultProxyRoutePlanner createProxyRoutePlanner(HttpSettings settings, HttpAsyncClientBuilder httpClientBuilder) {
    HttpHost proxy = new HttpHost(settings.getProxyUrl().getHost(), settings.getProxyUrl().getPort());
    if (settings.getProxyUser() != null) {
        CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        String proxyUser = settings.getProxyUser();
        String proxyPassword = settings.getProxyPassword();
        UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(proxyUser, proxyPassword);
        credentialsProvider.setCredentials(new AuthScope(proxy), credentials);
        httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
    }
    return new DefaultProxyRoutePlanner(proxy);
}

private static CloseableHttpClient createClientWithProxy(String proxyStr) throws URISyntaxException {
  URI proxyUri = new URI(proxyStr);
  HttpHost proxy = new HttpHost(proxyUri.getHost(), proxyUri.getPort(), proxyUri.getScheme());
  DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
  return HttpClients.custom()
      .setRoutePlanner(routePlanner)
      .build();
}

private static HttpClient createProxyEnabledHttpClient() {
    String USERNAME = proxyUser; // username for proxy authentication
    String PASSWORD = proxyPass; // password for proxy authentication

    String PROXY_ADDRESS = "proxy.hs-karlsruhe.de"; // proxy (IP) address
    String PROXY_DOMAIN = "http"; // proxy domain

    HttpHost proxy = new HttpHost(PROXY_ADDRESS, 8888);
    DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);

    AuthCache authCache = new BasicAuthCache();
    CredentialsProvider credsProvider = new BasicCredentialsProvider();
    credsProvider.setCredentials(new AuthScope(PROXY_ADDRESS, 8888), new NTCredentials(USERNAME, PASSWORD, "", PROXY_DOMAIN));
    HttpClientContext context = HttpClientContext.create();
    context.setCredentialsProvider(credsProvider);
    context.setAuthCache(authCache);

    //String token = USERNAME + ":" + PASSWORD;
    //String auth = "Basic " + String.valueOf(Base64.encode(token.getBytes()));
    // TODO add headers to the request
    //request.addHeader("Proxy-Authorization", auth);
    //request.addHeader("Https-Proxy-Authorization", auth);

    RequestConfig defaultRequestConfig = RequestConfig.custom()
            .setSocketTimeout(5000)
            .setConnectTimeout(5000)
            .setConnectionRequestTimeout(5000)
            .setStaleConnectionCheckEnabled(true)
            .build();

    //Main client about to connect
    CloseableHttpClient httpclient = HttpClients.custom()
            .setDefaultCredentialsProvider(credsProvider)
            .setRoutePlanner(routePlanner)
            .setDefaultRequestConfig(defaultRequestConfig)
            .build();

    return httpclient;
}

private CloseableHttpClient buildHttpClient(HttpHost proxy) {
    CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
    credentialsProvider.setCredentials(ANY, noCredentials());
    Registry<AuthSchemeProvider> registry = RegistryBuilder.<AuthSchemeProvider> create().register(NTLM, new NTLMSchemeProvider()).build();
    RequestConfig requestConfig = RequestConfig.custom().setProxyPreferredAuthSchemes(singleton(NTLM)).build();
    return HttpClients.custom().setDefaultRequestConfig(requestConfig).setRoutePlanner(new DefaultProxyRoutePlanner(proxy)).setDefaultAuthSchemeRegistry(registry)
            .setDefaultCredentialsProvider(credentialsProvider).disableRedirectHandling().build();
}

@Test
public void shouldForwardRequestsToSecurePortUsingHttpClientViaHTTP_CONNECT() throws Exception {
    // given
    HttpClient httpClient = HttpClients
            .custom()
            .setSSLSocketFactory(new SSLConnectionSocketFactory(KeyStoreFactory.keyStoreFactory().sslContext(), NoopHostnameVerifier.INSTANCE))
            .setRoutePlanner(
                    new DefaultProxyRoutePlanner(
                            new HttpHost(
                                    System.getProperty("http.proxyHost"),
                                    Integer.parseInt(System.getProperty("http.proxyPort"))
                            )
                    )
            ).build();

    // when
    HttpPost request = new HttpPost(
            new URIBuilder()
                    .setScheme("https")
                    .setHost("localhost")
                    .setPort(getServerSecurePort())
                    .setPath("/test_headers_and_body")
                    .build()
    );
    request.setEntity(new StringEntity("an_example_body"));
    HttpResponse response = httpClient.execute(request);

    // then
    assertEquals(HttpStatusCode.OK_200.code(), response.getStatusLine().getStatusCode());
    assertEquals("an_example_body", new String(EntityUtils.toByteArray(response.getEntity()), StandardCharsets.UTF_8));

    // and
    getProxyClient().verify(
            request()
                    .withPath("/test_headers_and_body")
                    .withBody("an_example_body"),
            exactly(1)
    );
}

protected HttpClient createHttpClient() {
    return HttpClients
        .custom()
        .setSSLSocketFactory(new SSLConnectionSocketFactory(KeyStoreFactory.keyStoreFactory().sslContext(), NoopHostnameVerifier.INSTANCE))
        .setRoutePlanner(new DefaultProxyRoutePlanner(
            new HttpHost(
                System.getProperty("http.proxyHost", "localhost"),
                Integer.parseInt(System.getProperty("http.proxyPort", String.valueOf(getProxyPort())))
            )
        )).build();
}

protected void initialzeInternalClient() {

        if (!needsInternalClientInialization) {
            // internal client is already initialized
            return;
        }

        // release any resources if this client was already used
        close();

        // rebuild the client
        HttpClientBuilder httpClientBuilder = HttpClients.custom();

        // Add this interceptor to get the values of all HTTP headers in the request.
        // Some of them are provided by the user while others are generated by Apache HTTP Components.
        httpClientBuilder.addInterceptorLast(new HttpRequestInterceptor() {
            @Override
            public void process( HttpRequest request, HttpContext context ) throws HttpException,
                                                                            IOException {

                Header[] requestHeaders = request.getAllHeaders();
                actualRequestHeaders = new ArrayList<HttpHeader>();
                for (Header header : requestHeaders) {
                    addHeaderToList(actualRequestHeaders, header.getName(), header.getValue());
                }
                if (debugLevel != HttpDebugLevel.NONE) {
                    logHTTPRequest(requestHeaders, request);
                }
            }
        });

        // connect and read timeouts
        httpClientBuilder.setDefaultRequestConfig(RequestConfig.custom()
                                                               .setConnectTimeout(connectTimeoutSeconds
                                                                                  * 1000)
                                                               .setSocketTimeout(readTimeoutSeconds
                                                                                 * 1000)
                                                               .build());

        // socket buffer size
        if (this.socketBufferSize > 0) {
            httpClientBuilder.setDefaultSocketConfig(SocketConfig.custom()
                                                                 .setRcvBufSize(this.socketBufferSize)
                                                                 .setSndBufSize(this.socketBufferSize)
                                                                 .build());
        }

        // SSL
        if (isOverSsl) {
            setupSSL(httpClientBuilder);
        }

        // setup authentication
        if (!StringUtils.isNullOrEmpty(username)) {
            setupAuthentication(httpClientBuilder);
        }

        // set proxy
        if (AtsSystemProperties.SYSTEM_HTTP_PROXY_HOST != null
            && AtsSystemProperties.SYSTEM_HTTP_PROXY_PORT != null) {

            HttpHost proxy = new HttpHost(AtsSystemProperties.SYSTEM_HTTP_PROXY_HOST,
                                          Integer.parseInt(AtsSystemProperties.SYSTEM_HTTP_PROXY_PORT));
            DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
            httpClientBuilder.setRoutePlanner(routePlanner);
        }

        // now build the client after we have already set everything needed on the client builder
        httpClient = httpClientBuilder.build();

        // do not come here again until not needed
        needsInternalClientInialization = false;
    }

/**
 *
 * @param url   request url
 * @param proxyString   proxy ip:port
 * 发生异常返回exception,返回结果错误返回error,成功返回请求网页的源代码
 */
public static String downLoadPage(String url, String proxyString){
    String reString = "";
    String ip = proxyString.split(":")[0];
    int port = Integer.valueOf(proxyString.split(":")[1]);

    HttpClient httpClient = HttpClients.custom().build();

    HttpHost proxy = new HttpHost(ip, port);
    //无需验证的
    DefaultProxyRoutePlanner routePlanner= new DefaultProxyRoutePlanner(proxy);
    httpClient = HttpClients.custom().setRoutePlanner(routePlanner).build();
    RequestConfig.Builder config_builder = RequestConfig.custom();
    config_builder.setProxy(proxy);

    config_builder.setSocketTimeout(StaticValue.http_connection_timeout);
    config_builder.setConnectTimeout(StaticValue.http_read_timeout);
    config_builder.setConnectionRequestTimeout(StaticValue.http_getconnection_timeout);
    RequestConfig requestConfig = config_builder.build();

    RequestBuilder rb = null;
    rb = RequestBuilder.get().setUri(URI.create(url));

    HttpUriRequest requestAll = null;
    rb.setConfig(requestConfig);
    requestAll = rb.build();
    CloseableHttpResponse response = null;
    try {
        response = (CloseableHttpResponse) httpClient.execute(requestAll);
        int statusCode = response.getStatusLine().getStatusCode();

        //状态码不是200,不会发生异常,不同返回以区分
        if (statusCode == HttpStatus.SC_OK) {
            reString = Crawl4HttpClient.parserResponse_v2(response);
        } else {
            logger.error(response.getStatusLine());
            reString = "error";
        }
    }catch (Exception ex){
        reString = "exception";
        logger.error("execute request error:"+MySysLogger.formatException(ex));
        ex.printStackTrace();
    }finally {
        if(response != null){
            try{
                response.close();
            }catch(IOException e){
                logger.error("error:"+MySysLogger.formatException(e));
                e.printStackTrace();
            }
        }
        httpClient.getConnectionManager().shutdown();
    }
    return reString;
}

public RestCall(String appName, String appVersion, String machineName, String environmentUrl, String method, AvaTaxPath path, Object model, TypeToken<T> typeToken, String proxyHost, int proxyPort, String proxySchema) {
    this(appName, appVersion, machineName, environmentUrl, method, path, model, typeToken, HttpClients.custom()
            .setRoutePlanner(new DefaultProxyRoutePlanner(new HttpHost(proxyHost, proxyPort, proxySchema)))
            .build());
}

/**
 * Configure the proxy with the required credential if needed
 *
 * @param httpClientBuilder
 * @param credentialsProvider
 * @param url
 * @return
 */
private HttpClientBuilder configureProxy(HttpClientBuilder httpClientBuilder, CredentialsProvider credentialsProvider, String url) throws DSSException {
    if (proxyConfig == null) {
        return httpClientBuilder;
    }
    try {

        final String protocol = new URL(url).getProtocol();
        final boolean proxyHTTPS = Protocol.isHttps(protocol) && proxyConfig.getHttpsProperties() != null;
        final boolean proxyHTTP = Protocol.isHttp(protocol) && proxyConfig.getHttpProperties() != null;

        if (!proxyHTTPS && !proxyHTTP) {
            return httpClientBuilder;
        }

        ProxyProperties proxyProps = null;
        if (proxyHTTPS) {
            LOG.debug("Use proxy https parameters");
            proxyProps = proxyConfig.getHttpsProperties();
        } else if (proxyHTTP) {
            LOG.debug("Use proxy http parameters");
            proxyProps = proxyConfig.getHttpProperties();
        }

        String proxyHost = proxyProps.getHost();
        int proxyPort = proxyProps.getPort();
        String proxyUser = proxyProps.getUser();
        String proxyPassword = proxyProps.getPassword();
        String proxyExcludedHosts = proxyProps.getExcludedHosts();

        if (Utils.isStringNotEmpty(proxyUser) && Utils.isStringNotEmpty(proxyPassword)) {
            AuthScope proxyAuth = new AuthScope(proxyHost, proxyPort);
            UsernamePasswordCredentials proxyCredentials = new UsernamePasswordCredentials(proxyUser, proxyPassword);
            credentialsProvider.setCredentials(proxyAuth, proxyCredentials);
        }

        LOG.debug("proxy host/port: " + proxyHost + ":" + proxyPort);
        // TODO SSL peer shut down incorrectly when protocol is https
        final HttpHost proxy = new HttpHost(proxyHost, proxyPort, Protocol.HTTP.getName());

        if (Utils.isStringNotEmpty(proxyExcludedHosts)) {
            final String[] hosts = proxyExcludedHosts.split("[,; ]");

            HttpRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy) {
                @Override
                public HttpRoute determineRoute(final HttpHost host, final HttpRequest request, final HttpContext context) throws HttpException {

                    String hostname = (host != null ? host.getHostName() : null);

                    if ((hosts != null) && (hostname != null)) {
                        for (String h : hosts) {
                            if (hostname.equalsIgnoreCase(h)) {
                                // bypass proxy for that hostname
                                return new HttpRoute(host);
                            }
                        }
                    }
                    return super.determineRoute(host, request, context);
                }
            };

            httpClientBuilder.setRoutePlanner(routePlanner);
        }

        final HttpClientBuilder httpClientBuilder1 = httpClientBuilder.setProxy(proxy);
        return httpClientBuilder1;
    } catch (MalformedURLException e) {
        throw new DSSException(e);
    }
}

public String postRequestSepa(Cart cart, PaymentData paymentData) {
    String address = _env.getProperty("bfs.url");
    try {
        CloseableHttpClient httpClient;
        // on staging and production the requests has to be routed
        // through a proxy
        if (_env.getProperty("proxy.host") != null) {
            HttpHost proxy = new HttpHost(_env.getProperty("proxy.host"), Integer.parseInt(_env.getProperty("proxy.port")));
            DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
            httpClient = HttpClients.custom()
                                    .setRoutePlanner(routePlanner)
                                    .build();
        } else {
            httpClient = HttpClients.custom()
                                    .build();
        }

        HttpPost httpPost = new HttpPost(address);

        List<NameValuePair> urlParameters = new ArrayList<NameValuePair>();
        Map<String, String> params = createParams(cart, paymentData);
        for (Entry<String, String> entry : params.entrySet()) {
            urlParameters.add(new BasicNameValuePair(entry.getKey(), entry.getValue()));
        }

        httpPost.setEntity(new UrlEncodedFormEntity(urlParameters));

        HttpResponse response = httpClient.execute(httpPost);

        BufferedReader rd = new BufferedReader(new InputStreamReader(response.getEntity()
                                                                             .getContent()));
        StringBuffer result = new StringBuffer();
        String line = "";
        while ((line = rd.readLine()) != null) {
            result.append(line);
        }
        httpClient.close();
        return result.toString();
    } catch (final Exception e) {
        LOG.error("unable to do post request to '" + address + "'", e);
        return CONNECTION_ERROR;
    }
}