Java 类org.apache.http.ssl.TrustStrategy 实例源码

/**
 * Will create a certificate-ignoring {@link SSLContext}. Please use with utmost caution as it undermines security,
 * but may be useful in certain testing or development scenarios.
 *
 * @return The SSLContext
 */
public static SSLContext buildCertificateIgnoringSslContext() {
    try {
        return SSLContexts
            .custom()
            .loadTrustMaterial(new TrustStrategy() {
                @Override
                public boolean isTrusted(X509Certificate[] arg0, String arg1) throws CertificateException {
                    return true;
                }
            })
            .build();
    }
    catch (KeyManagementException | NoSuchAlgorithmException | KeyStoreException e) {
        throw new IllegalStateException("Unexpected exception while building the certificate-ignoring SSLContext.", e);
    }
}

/**
 * This creates a HTTP client instance for connecting the IFTTT server.
 * 
 * @return the HTTP client instance
 */
private CloseableHttpClient buildHttpClient ()
{
    if ( configuration.isIftttIgnoreServerCertificate() ) {
        try {
            SSLContextBuilder builder = new SSLContextBuilder();
            builder.loadTrustMaterial(new TrustStrategy() {
                @Override
                public boolean isTrusted (X509Certificate[] chain_, String authType_) throws CertificateException
                {
                    return true;
                }
            });
            SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(builder.build());
            return HttpClients.custom().setSSLSocketFactory(sslsf).build();
        }
        catch (Exception ex) {
            LOG.error(ex);
            // This should never happen, but we have to handle it
            throw new RuntimeException(ex);
        }
    }
    else {
        return HttpClients.createDefault();
    }
}

/**
 * @return
 */
private HttpClient createSSLHttpClient() {

    final TrustStrategy acceptingTrustStrategy = new TrustStrategy() {

        @Override
        public boolean isTrusted(final X509Certificate[] arg0, final String arg1) throws CertificateException {
            return true;
        }
    };

    SSLContext sslContext = null;
    try {
        sslContext = SSLContexts.custom().loadTrustMaterial(null, acceptingTrustStrategy).build();
    } catch (final Exception e) {
        System.out.println("Could not create SSLContext");
    }

    return HttpClientBuilder.create().setSSLContext(sslContext).build();
}

@Test
public void testPost() throws Exception {
    SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null,
            new TrustStrategy() {
                //信任所有
                public boolean isTrusted(X509Certificate[] chain, String authType)
                        throws CertificateException {
                    return true;
                }
            }).build();
    SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext);
    CloseableHttpClient httpClient = HttpClients.custom().setSSLSocketFactory(sslsf).build();

    HttpGet request = new HttpGet("https://www.baidu.com");
    request.addHeader(
            "User-Agent",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36");

    final CloseableHttpResponse response = httpClient.execute(request);

    System.out.println(response.getStatusLine().getStatusCode());

    System.out.println(EntityUtils.toString(response.getEntity()));
}

@Bean("httpClient")
@Profile("insecure")
@SneakyThrows
public HttpClient insecureHttpClient(@Value("${spring.application.name}") String userAgent) {
    // http://stackoverflow.com/a/41618092/1393467
    TrustStrategy trustStrategy = (X509Certificate[] chain, String authType) -> true;
    SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, trustStrategy).build();
    return configure(HttpClients.custom(), userAgent)
            .setSSLSocketFactory(new SSLConnectionSocketFactory(sslContext))
            .build();
}

/**
 * @see http://literatejava.com/networks/ignore-ssl-certificate-errors-apache-httpclient-4-4/
 * @return
 * @throws Exception
 */
public static synchronized HttpClient getHttpClient() throws Exception
{
   HttpClientBuilder b = HttpClientBuilder.create();

   // setup a Trust Strategy that allows all certificates.
   //
   SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy()
      {
         public boolean isTrusted(X509Certificate[] arg0, String arg1) throws CertificateException
         {
            return true;
         }
      }).build();
   b.setSslcontext(sslContext);

   // don't check Hostnames, either.
   //      -- use SSLConnectionSocketFactory.getDefaultHostnameVerifier(), if you don't want to weaken
   HostnameVerifier hostnameVerifier = SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER;

   // here's the special part:
   //      -- need to create an SSL Socket Factory, to use our weakened "trust strategy";
   //      -- and create a Registry, to register it.
   //
   SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(sslContext, hostnameVerifier);
   //Registry<ConnectionSocketFactory> socketFactoryRegistry = ;

   // now, we create connection-manager using our Registry.
   //      -- allows multi-threaded use
   PoolingHttpClientConnectionManager connMgr = new PoolingHttpClientConnectionManager(RegistryBuilder.<ConnectionSocketFactory> create().register("http", PlainConnectionSocketFactory.getSocketFactory()).register("https", sslSocketFactory).build());
   b.setConnectionManager(connMgr);

   // finally, build the HttpClient;
   //      -- done!
   HttpClient client = b.build();

   return client;
}

public HttpClient createHttpClient_AcceptsUntrustedCerts() throws KeyManagementException, NoSuchAlgorithmException, KeyStoreException {
    HttpClientBuilder b = HttpClientBuilder.create();

    // setup a Trust Strategy that allows all certificates.
    //
    SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {
        public boolean isTrusted(X509Certificate[] arg0, String arg1) throws CertificateException {
            return true;
        }
    }).build();
    b.setSslcontext( sslContext);

    // don't check Hostnames, either.
    //      -- use SSLConnectionSocketFactory.getDefaultHostnameVerifier(), if you don't want to weaken
    HostnameVerifier hostnameVerifier = SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER;

    // here's the special part:
    //      -- need to create an SSL Socket Factory, to use our weakened "trust strategy";
    //      -- and create a Registry, to register it.
    //
    SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(sslContext, hostnameVerifier);
    Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
            .register("http", PlainConnectionSocketFactory.getSocketFactory())
            .register("https", sslSocketFactory)
            .build();

    // now, we create connection-manager using our Registry.
    //      -- allows multi-threaded use
    PoolingHttpClientConnectionManager connMgr = new PoolingHttpClientConnectionManager( socketFactoryRegistry);
    b.setConnectionManager( connMgr);

    // finally, build the HttpClient;
    //      -- done!
    HttpClient client = b.build();
    return client;
}

public HttpClient createHttpClient_AcceptsUntrustedCerts() throws KeyManagementException, NoSuchAlgorithmException, KeyStoreException {
    HttpClientBuilder b = HttpClientBuilder.create();

    // setup a Trust Strategy that allows all certificates.
    //
    SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {
        public boolean isTrusted(X509Certificate[] arg0, String arg1) throws CertificateException {
            return true;
        }
    }).build();
    b.setSslcontext( sslContext);

    // don't check Hostnames, either.
    //      -- use SSLConnectionSocketFactory.getDefaultHostnameVerifier(), if you don't want to weaken
    HostnameVerifier hostnameVerifier = SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER;

    // here's the special part:
    //      -- need to create an SSL Socket Factory, to use our weakened "trust strategy";
    //      -- and create a Registry, to register it.
    //
    SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(sslContext, hostnameVerifier);
    Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
            .register("http", PlainConnectionSocketFactory.getSocketFactory())
            .register("https", sslSocketFactory)
            .build();

    // now, we create connection-manager using our Registry.
    //      -- allows multi-threaded use
    PoolingHttpClientConnectionManager connMgr = new PoolingHttpClientConnectionManager( socketFactoryRegistry);
    b.setConnectionManager( connMgr);

    // finally, build the HttpClient;
    //      -- done!
    HttpClient client = b.build();
    return client;
}

public void connect() {
    MQTT mqtt = new MQTT();

    try {
        mqtt.setHost(connectionInfo.getBiobrightUrl());
        mqtt.setUserName(connectionInfo.getBiobrightUserName());
        mqtt.setPassword(connectionInfo.getBiobrightPassword());

        // TODO change security policy that is actually disabled with this code.
        TrustStrategy acceptingTrustStrategy = (X509Certificate[] chain, String authType) -> true;

        SSLContext sslContext = org.apache.http.ssl.SSLContexts.custom()
                .loadTrustMaterial(null, acceptingTrustStrategy)
                .build();

        mqtt.setSslContext(sslContext);

        logger.info("Opening MQTT socket.. ");
        connection = mqtt.blockingConnection();
        logger.info("Opened MQTT socket, connecting.. ");
        connection.connect();           
        logger.info("Connected MQTT socket.. ");
    } catch (Exception e) {
        logger.error("connect()", e);
        if(connection != null) {
            connection = null;
        }
        throw new RuntimeException("Connection failed.", e);
    }
}

private static HttpClient createHttpClient()
        throws KeyStoreException, NoSuchAlgorithmException, KeyManagementException {
    HttpClientBuilder b = HttpClientBuilder.create();

    // setup a Trust Strategy that allows all certificates.
    //
    SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {
        public boolean isTrusted(X509Certificate[] arg0, String arg1) throws CertificateException {
            return true;
        }
    }).build();
    b.setSSLContext(sslContext);
    //b.setSSLHostnameVerifier(new NoopHostnameVerifier());

    // don't check Hostnames, either.
    //      -- use SSLConnectionSocketFactory.getDefaultHostnameVerifier(), if you don't want to weaken
    HostnameVerifier hostnameVerifier = SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER;

    // here's the special part:
    //      -- need to create an SSL Socket Factory, to use our weakened "trust strategy";
    //      -- and create a Registry, to register it.
    //
    SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(sslContext, hostnameVerifier);
    Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
            .register("http", PlainConnectionSocketFactory.getSocketFactory())
            .register("https", sslSocketFactory)
            .build();

    // now, we create connection-manager using our Registry.
    //      -- allows multi-threaded use
    PoolingHttpClientConnectionManager connMgr = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
    b.setConnectionManager(connMgr);

    // finally, build the HttpClient;
    //      -- done!
    CloseableHttpClient client = b.build();
    return client;
}

private static HttpClient createHttpClient()
        throws KeyStoreException, NoSuchAlgorithmException, KeyManagementException {
    HttpClientBuilder b = HttpClientBuilder.create();

    // setup a Trust Strategy that allows all certificates.
    //
    SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {
        public boolean isTrusted(X509Certificate[] arg0, String arg1) throws CertificateException {
            return true;
        }
    }).build();
    b.setSSLContext(sslContext);
    //b.setSSLHostnameVerifier(new NoopHostnameVerifier());

    // don't check Hostnames, either.
    //      -- use SSLConnectionSocketFactory.getDefaultHostnameVerifier(), if you don't want to weaken
    HostnameVerifier hostnameVerifier = SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER;

    // here's the special part:
    //      -- need to create an SSL Socket Factory, to use our weakened "trust strategy";
    //      -- and create a Registry, to register it.
    //
    SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(sslContext, hostnameVerifier);
    Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
            .register("http", PlainConnectionSocketFactory.getSocketFactory())
            .register("https", sslSocketFactory)
            .build();

    // now, we create connection-manager using our Registry.
    //      -- allows multi-threaded use
    PoolingHttpClientConnectionManager connMgr = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
    b.setConnectionManager(connMgr);

    // finally, build the HttpClient;
    //      -- done!
    CloseableHttpClient client = b.build();
    return client;
}

private static CloseableHttpClient getHttpClient()
    throws IOException {
  try {
    // Self sign SSL
    SSLContextBuilder builder = new SSLContextBuilder();
    builder.loadTrustMaterial(null, (TrustStrategy) new TrustSelfSignedStrategy());
    SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(builder.build());

    // Create client
    return HttpClients.custom().setSSLSocketFactory(sslsf).setDefaultCookieStore(new BasicCookieStore()).build();
  } catch (NoSuchAlgorithmException | KeyManagementException | KeyStoreException e) {
    throw new IOException("Issue with creating http client", e);
  }
}

public static HttpClientConnectionManager getConnectionManager() {

        // ConnectionSocketFactory plainsf = null;
        LayeredConnectionSocketFactory sslsf = null;

        RegistryBuilder<ConnectionSocketFactory> registryBuilder = RegistryBuilder.create();

        PlainConnectionSocketFactory plainsf = PlainConnectionSocketFactory.getSocketFactory();
        registryBuilder.register("http", plainsf);

        try {

            // Trust own CA and all self-signed certs
            SSLContext sslcontext = SSLContexts.custom().loadTrustMaterial(new TrustStrategy() {
                @Override
                public boolean isTrusted(X509Certificate[] chain, String authType)
                        throws CertificateException {
                    return true;
                }
            }).build();

            HostnameVerifier allowAllHostnameVerifier = NoopHostnameVerifier.INSTANCE;
            sslsf = new SSLConnectionSocketFactory(sslcontext, allowAllHostnameVerifier);

            registryBuilder.register("https", sslsf);

        } catch (Throwable e) {

            logger.error("https ssl init failed", e);
        }

        Registry<ConnectionSocketFactory> r = registryBuilder.build();

        PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(r);
        connManager.setMaxTotal(100);// 连接池最大并发连接数
        connManager.setDefaultMaxPerRoute(100);// 单路由最大并发数
        return connManager;

    }

public KF6Service() {
    gson = new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'").create();
    // client = HttpClientBuilder.create().build();
    try {
        TrustStrategy trustStrategy = new TrustSelfSignedStrategy();
        SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(trustStrategy).build();
        client =
                   HttpClients
                           .custom()
                           .setSSLContext(sslContext)
                           .build();
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    }
}

private static HttpClient getHttpClientWithoutSslVerify() throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException {
    final SSLContextBuilder builder = SSLContexts.custom();
    builder.loadTrustMaterial(null, new TrustStrategy() {
        @Override
        public boolean isTrusted(X509Certificate[] chain, String authType)
                throws CertificateException {
            return true;
        }
    });
    final SSLContext sslContext = builder.build();
    final SSLConnectionSocketFactory socketFactory = new SSLConnectionSocketFactory(
            builder.build());

    final Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder
            .<ConnectionSocketFactory>create().register("https", socketFactory)
            .build();

    final PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(
            socketFactoryRegistry, null, null, null, 30, TimeUnit.SECONDS
    );

    connectionManager.setMaxTotal(30);
    connectionManager.setDefaultMaxPerRoute(30);
    return HttpClients
            .custom()
            .setConnectionManager(connectionManager)
            .build();
}

private void initClient( String accessKey, String secretKey, List<Header> defaultHeadersOverride, int connectionRequestTimeout, int connectionTimeout, int socketTimeout, HttpHost proxy, boolean noSslValidation, String impersonateUsername ) {
    RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();

    requestConfigBuilder.setConnectionRequestTimeout( connectionRequestTimeout ).setConnectTimeout( connectionTimeout ).setSocketTimeout( socketTimeout );
    if( proxy != null )
        requestConfigBuilder.setProxy( proxy );

    SSLContext sslContext = null;
    // Note: this block of code disables SSL validation. It is only used during development/testing when testing through a proxy
    if( noSslValidation ) {
        try {
            sslContext = SSLContexts.custom().loadTrustMaterial( new TrustStrategy() {
                @Override
                public boolean isTrusted( X509Certificate[] chain, String authType ) throws CertificateException {
                    return true;
                }
            } )
                    .build();
        } catch( Exception e ) {
        }
    }

    //system properties
    Map<String, String> systemProperties = ManagementFactory.getRuntimeMXBean().getSystemProperties();

    if ( defaultHeadersOverride == null ) {
        defaultHeaders = new ArrayList<>( 3 );
        defaultHeaders.add( new BasicHeader( "X-ApiKeys", String.format( "accessKey=%s; secretKey=%s", accessKey, secretKey ) ) );
        defaultHeaders.add( new BasicHeader( "User-Agent", String.format( "TenableIOSDK Java/%s %s/%s/%s", systemProperties.get( "java.runtime.version" ), systemProperties.get( "os.name" ), systemProperties.get( "os.version" ), systemProperties.get( "os.arch" ) ) ) );
        defaultHeaders.add( new BasicHeader( "Accept", "*/*" ) );

        if ( impersonateUsername != null ) {
            defaultHeaders.add( new BasicHeader( "X-Impersonate", "username=" + impersonateUsername ) );
        }
    } else {
        defaultHeaders = defaultHeadersOverride;
    }

    asyncClient = HttpAsyncClients.custom()
            .setDefaultRequestConfig( requestConfigBuilder.build() )
            .setDefaultHeaders( defaultHeaders )
            .setSSLContext( sslContext )
            .build();

    asyncClient.start();
}

public HttpClient(String proxyUrl, String proxyUser, String proxyPass) {
    HttpClientBuilder builder = HttpClientBuilder.create()
            .setConnectionManagerShared(true);
    try {
        Optional<HttpHost> proxyHost = createProxyHttpHost(proxyUrl);
        if (proxyHost.isPresent()) {
            builder.setProxy(proxyHost.get());
            Optional<BasicCredentialsProvider> credentialsProvider = createBasicCredentialsProvider(
                    proxyUrl, proxyUser, proxyPass, proxyHost.get());
            if (credentialsProvider.isPresent()) {
                builder.setDefaultCredentialsProvider(
                        credentialsProvider.get());
            }
        }

        HostnameVerifier allHostsValid = new HostnameVerifier() {
            public boolean verify(String hostname, SSLSession session) {
                return true;
            }
        };
        SSLContext sslContext = SSLContexts.custom()
                .loadTrustMaterial(null, new TrustStrategy() {
                    @Override
                    public boolean isTrusted(X509Certificate[] chain,
                            String authType) throws CertificateException {
                        return true;
                    }
                }).build();
        SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(
                sslContext, allHostsValid);
        Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder
                .<ConnectionSocketFactory>create().register("https", sslsf)
                .register("http", new PlainConnectionSocketFactory())
                .build();
        PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(
                socketFactoryRegistry);

        builder.setConnectionManager(cm);
    } catch (Exception e) {
        throw new WebDriverManagerException(e);
    }

    closeableHttpClient = builder.useSystemProperties().build();
}

@Test
public void testGet2() {
    try {

        SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null,
                new TrustStrategy() {
                    @Override
                    public boolean isTrusted(X509Certificate[] chain, String authType)
                            throws CertificateException {
                        return true;
                    }
                }).build();
        SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext);
        HttpClient httpClient = HttpClients.custom().setSSLSocketFactory(sslsf).build();
        HttpPost httppost = new HttpPost("https://community.apache.org/contributors/");
        MultipartEntityBuilder mEntityBuilder = MultipartEntityBuilder.create();
        //params
        //mEntityBuilder.addTextBody("userName", "1234");

        httppost.setEntity(mEntityBuilder.build());
        //httppost.addHeader("Content-Type", "Application/JSON");

        int timeOut = 1000 * 50;
        // set Timeout
        RequestConfig requestConfig = RequestConfig.custom()
                .setConnectionRequestTimeout(timeOut).setConnectTimeout(timeOut)
                .setSocketTimeout(timeOut).build();
        httppost.setConfig(requestConfig);
        // get responce
        HttpResponse responce = httpClient.execute(httppost);
        // get http status code
        int status = responce.getStatusLine().getStatusCode();
        System.out.println("request code:" + status);
        String resultString = null;
        if (status == HttpStatus.SC_OK) {
            // get result data
            HttpEntity entity = responce.getEntity();
            resultString = EntityUtils.toString(entity);
        }
        System.out.println(resultString);

    }
    catch (Exception e) {
        e.printStackTrace();
    }

}

private static HttpClient createHttpClient0(CookieStore cookieStore) throws KeyStoreException, KeyManagementException, NoSuchAlgorithmException{
        RegistryBuilder<ConnectionSocketFactory> registryBuilder = RegistryBuilder.<ConnectionSocketFactory>create();
        ConnectionSocketFactory http = new PlainConnectionSocketFactory();
        registryBuilder.register("http", http);

        /*TrustManager trustManager = new X509TrustManager(){
            @Override
            public void checkClientTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {

            }

            @Override
            public void checkServerTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {

            }

            @Override
            public X509Certificate[] getAcceptedIssuers() {
                return null;
            }
        }; */
        /***
         * setConnectTimeout：设置连接超时时间，单位毫秒。
setConnectionRequestTimeout：设置从connect Manager获取Connection 超时时间，单位毫秒。这个属性是新加的属性，因为目前版本是可以共享连接池的。
setSocketTimeout：请求获取数据的超时时间，单位毫秒。 如果访问一个接口，多少时间内无法返回数据，就直接放弃此次调用。
         */
        RequestConfig reqConfig = createDefaultRequestConfig();
        KeyStore trustStory = KeyStore.getInstance(KeyStore.getDefaultType());
        TrustStrategy anyTrustStrategy = new TrustStrategy(){
            @Override
            public boolean isTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
                return true;
            }
        };
        SSLContext sslContext = SSLContexts.custom()
                                            .useProtocol("TLS")
                                            .loadTrustMaterial(trustStory, anyTrustStrategy)
                                            .build();
        LayeredConnectionSocketFactory https = new SSLConnectionSocketFactory(sslContext);
        registryBuilder.register("https", https);

        Registry<ConnectionSocketFactory> registry = registryBuilder.build();
        PoolingHttpClientConnectionManager poolMgr = new PoolingHttpClientConnectionManager(registry);
        return HttpClientBuilder.create()
                                .setDefaultCookieStore(cookieStore)
                                .setConnectionManager(poolMgr)
                                .setDefaultRequestConfig(reqConfig)
                                .build();
    }

public HttpClientDownloader() {

    cookieContext = HttpClientContext.create();
    cookieContext.setCookieStore(new BasicCookieStore());

    Registry<ConnectionSocketFactory> socketFactoryRegistry = null;
    try {
        //构造一个信任所有ssl证书的httpclient
        SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() {
            @Override
            public boolean isTrusted(X509Certificate[] chain, String authType) throws CertificateException {
                return true;
            }
        }).build();
        SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext);
        socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
                   .register("http", PlainConnectionSocketFactory.getSocketFactory())  
                   .register("https", sslsf)  
                   .build();
    } catch(Exception ex) {
        socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
           .register("http", PlainConnectionSocketFactory.getSocketFactory())
           .register("https", SSLConnectionSocketFactory.getSocketFactory())
           .build();
    }
    RequestConfig clientConfig = RequestConfig.custom().setRedirectsEnabled(false).build();
    PoolingHttpClientConnectionManager syncConnectionManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
    syncConnectionManager.setMaxTotal(1000);
    syncConnectionManager.setDefaultMaxPerRoute(50);
    httpClient = HttpClientBuilder.create()
            .setDefaultRequestConfig(clientConfig)
            .setConnectionManager(syncConnectionManager)
            .setRetryHandler(new HttpRequestRetryHandler() {
                @Override
                public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
                    int retryCount = SpiderThreadLocal.get().getEngine().getRetry();
                    boolean retry = (executionCount <= retryCount);
                    if(log.isDebugEnabled() && retry) {
                        log.debug("retry : " + executionCount);
                    }
                    return retry;
                }
            }).build();
}

public ScanAndScroll(String url,String username, String password,String outputFilePath,int outputType,String outputFile,int runTika,String htmlField) throws FileNotFoundException, UnsupportedEncodingException{

    SSLContext sslContext;
       try {
           sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {
               public boolean isTrusted(X509Certificate[] arg0, String arg1) throws CertificateException {
                   return true;
               }
           }).build();
       } catch (NoSuchAlgorithmException | KeyManagementException | KeyStoreException e) {
           throw new IllegalStateException(e);
       }

       // Skip hostname checks
       HostnameVerifier hostnameVerifier = NoopHostnameVerifier.INSTANCE;
       SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(sslContext, hostnameVerifier);

       HttpClientConfig.Builder httpClientBuilder = new HttpClientConfig.Builder(url.toString())
                                                   .sslSocketFactory(sslSocketFactory)
                                                   .readTimeout(30000) // Milliseconds
                                                   .multiThreaded(false);


    System.out.println(url);

    if(username.trim() != "" && password.trim() != ""){
        httpClientBuilder.defaultCredentials(username, password);
    }

    JestClientFactory jcf = new JestClientFactory();

    jcf.setHttpClientConfig(httpClientBuilder.build());

    this.client = jcf.getObject();


    this.outputType = outputType;
    this.outputFile = outputFile;
    this.runTika = runTika;
    this.htmlField = htmlField;
}