Java 类org.elasticsearch.client.RestClientBuilder 实例源码

public ElasticBulkSender(String user, String password, HttpHost... hosts) {
    this.user = user;
    this.password = password;
    this.hosts = hosts;
    this.restClient = RestClient.builder(hosts)
            .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
                @Override
                public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
                    if (!Strings.isBlank(user)) {
                        CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
                        credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(user, password));
                        return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
                    } else {
                        return httpClientBuilder;
                    }
                }
            })
            .build();
}

@Before
public void setUp() throws Exception {
    /**
     * 如果es集群安装了x-pack插件则以此种方式连接集群
     * 1. java客户端的方式是以tcp协议在9300端口上进行通信
     * 2. http客户端的方式是以http协议在9200端口上进行通信
     */
    Settings settings = Settings.builder(). put("xpack.security.user", "elastic:changeme").build();
    client = new PreBuiltXPackTransportClient(settings)
            .addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("localhost"), 9300));
    elasticsearchTemplate = new ElasticsearchTemplate(client);
    final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
    credentialsProvider.setCredentials(AuthScope.ANY,
            new UsernamePasswordCredentials("elastic", "changeme"));
    restClient = RestClient.builder(new HttpHost("localhost",9200,"http"))
            .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
                @Override
                public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
                    return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
                }
            }).build();
}

public static RestClient createClient(ElasticsearchDatastoreProperties datastore) throws MalformedURLException {
    String urlStr = datastore.nodes.getValue();
    String[] urls = urlStr.split(",");
    HttpHost[] hosts = new HttpHost[urls.length];
    int i = 0;
    for (String address : urls) {
        URL url = new URL("http://" + address);
        hosts[i] = new HttpHost(url.getHost(), url.getPort(), url.getProtocol());
        i++;
    }
    RestClientBuilder restClientBuilder = RestClient.builder(hosts);
    if (datastore.auth.useAuth.getValue()) {
        final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        credentialsProvider.setCredentials(AuthScope.ANY,
                new UsernamePasswordCredentials(datastore.auth.userId.getValue(), datastore.auth.password.getValue()));
        restClientBuilder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {

            public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
                return httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
            }
        });
    }
    return restClientBuilder.build();
}

private void startRestClient() {
    try {
        HttpHost[] hosts = new HttpHost[] {
                // new HttpHost("10.132.161.173", 30002, "http")
                new HttpHost("127.0.0.1", 9200, "http") };
        client = RestClient.builder(hosts).setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
            @Override
            public RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
                return requestConfigBuilder.setConnectTimeout(2000).setSocketTimeout(10000);
            }
        }).setMaxRetryTimeoutMillis(10000).setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
            @Override
            public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
                return httpClientBuilder.setMaxConnPerRoute(100).setMaxConnTotal(200);
                // return httpClientBuilder
                // .setDefaultIOReactorConfig(IOReactorConfig.custom().setIoThreadCount(1).build());
            }
        }).build();
    } catch (Exception e) {
        LOG.error(e.getMessage(), e.fillInStackTrace());
    }
}

@Before
public void init() {
    HttpHost host = new HttpHost("127.0.0.1", 30002, "http");
    // HttpHost host = new HttpHost("127.0.0.1", 9002, "http");
    client = RestClient.builder(host).setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
        @Override
        public RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
            return requestConfigBuilder.setConnectTimeout(5000).setSocketTimeout(30000);
        }
    })
            // .setMaxRetryTimeoutMillis(30000)
            // .setHttpClientConfigCallback(new
            // RestClientBuilder.HttpClientConfigCallback() {
            // @Override
            // public HttpAsyncClientBuilder
            // customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder)
            // {
            // return httpClientBuilder
            // .setDefaultIOReactorConfig(IOReactorConfig.custom().setIoThreadCount(1).build());
            // }
            // })
            .build();
}

@Before
public void init() {
    client = RestClient.builder(new HttpHost("127.0.0.1", 30002, "http"), new HttpHost("127.0.0.1", 30002, "http"))
            .setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
                @Override
                public RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
                    return requestConfigBuilder.setConnectTimeout(5000).setSocketTimeout(30000);
                }
            }).setMaxRetryTimeoutMillis(30000)
            // .setHttpClientConfigCallback(new
            // RestClientBuilder.HttpClientConfigCallback() {
            // @Override
            // public HttpAsyncClientBuilder
            // customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder)
            // {
            // return httpClientBuilder
            // .setDefaultIOReactorConfig(IOReactorConfig.custom().setIoThreadCount(1).build());
            // }
            // })
            .build();
}

@JsonCreator
public ElasticSearchFilter(@JsonProperty("hostname") String hostname, @JsonProperty("port") int port,
                           @JsonProperty("username") Optional<String> username,
                           @JsonProperty("password") Optional<String> password) {
  Header[] headers = {
    new BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json"),
    new BasicHeader("Role", "Read")};
  final RestClientBuilder restClientBuilder = RestClient.builder(new HttpHost(hostname, port))
    .setDefaultHeaders(headers);
  if (username.isPresent() && !username.get().isEmpty() && password.isPresent() && !password.get().isEmpty()) {
    final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();

    credentialsProvider.setCredentials(
      AuthScope.ANY,
      new UsernamePasswordCredentials(username.get(), password.get())
    );

    restClientBuilder.setHttpClientConfigCallback(b -> b.setDefaultCredentialsProvider(credentialsProvider));
  }
  restClient = restClientBuilder.build();
  mapper = new ObjectMapper();
}

protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException {
    RestClientBuilder builder = RestClient.builder(hosts);
    String keystorePath = settings.get(TRUSTSTORE_PATH);
    if (keystorePath != null) {
        final String keystorePass = settings.get(TRUSTSTORE_PASSWORD);
        if (keystorePass == null) {
            throw new IllegalStateException(TRUSTSTORE_PATH + " is provided but not " + TRUSTSTORE_PASSWORD);
        }
        Path path = PathUtils.get(keystorePath);
        if (!Files.exists(path)) {
            throw new IllegalStateException(TRUSTSTORE_PATH + " is set but points to a non-existing file");
        }
        try {
            KeyStore keyStore = KeyStore.getInstance("jks");
            try (InputStream is = Files.newInputStream(path)) {
                keyStore.load(is, keystorePass.toCharArray());
            }
            SSLContext sslcontext = SSLContexts.custom().loadTrustMaterial(keyStore, null).build();
            SSLIOSessionStrategy sessionStrategy = new SSLIOSessionStrategy(sslcontext);
            builder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setSSLStrategy(sessionStrategy));
        } catch (KeyStoreException|NoSuchAlgorithmException|KeyManagementException|CertificateException e) {
            throw new RuntimeException("Error setting up ssl", e);
        }
    }

    try (ThreadContext threadContext = new ThreadContext(settings)) {
        Header[] defaultHeaders = new Header[threadContext.getHeaders().size()];
        int i = 0;
        for (Map.Entry<String, String> entry : threadContext.getHeaders().entrySet()) {
            defaultHeaders[i++] = new BasicHeader(entry.getKey(), entry.getValue());
        }
        builder.setDefaultHeaders(defaultHeaders);
    }
    return builder.build();
}

public void run(Configuration configuration, Environment environment) throws Exception {
  final CollectorRegistry collectorRegistry = new CollectorRegistry();
  collectorRegistry.register(new DropwizardExports(environment.metrics()));
  environment.admin()
      .addServlet("metrics", new MetricsServlet(collectorRegistry))
      .addMapping("/metrics");

  final PrometheusMetricsReporter reporter = PrometheusMetricsReporter.newMetricsReporter()
      .withCollectorRegistry(collectorRegistry)
      .withConstLabel("service", getName())
      .build();

  final Tracer tracer = getTracer();
  final Tracer metricsTracer = io.opentracing.contrib.metrics.Metrics.decorate(tracer, reporter);
  GlobalTracer.register(metricsTracer);

  final DynamicFeature tracing = new ServerTracingDynamicFeature.Builder(metricsTracer).build();
  environment.jersey().register(tracing);

  final HttpHost httpHost = new HttpHost("tweets-elasticsearch", 9200);
  final RestClientBuilder restClientBuilder =
      RestClient.builder(httpHost).setHttpClientConfigCallback(new TracingHttpClientConfigCallback(metricsTracer));
  final RestClient restClient = restClientBuilder.build();
  final RestHighLevelClient restHighLevelClient = new RestHighLevelClient(restClient);
  final TweetRepository tweetRepository = new ElasticsearchTweetRepository(restHighLevelClient);
  final TweetsResource tweetsResource = new TweetsResource(tweetRepository);
  environment.jersey().register(tweetsResource);
}

public void run(Configuration configuration, Environment environment) throws Exception {
  // INSTRUMENTATION
  // Metrics Instrumentation
  final CollectorRegistry collectorRegistry = new CollectorRegistry();
  collectorRegistry.register(new DropwizardExports(environment.metrics()));
  environment.admin()
      .addServlet("metrics", new MetricsServlet(collectorRegistry))
      .addMapping("/metrics");

  final PrometheusMetricsReporter reporter = PrometheusMetricsReporter.newMetricsReporter()
      .withCollectorRegistry(collectorRegistry)
      .withConstLabel("service", getName())
      .build();

  // Tracing Instrumentation
  final Tracer tracer = getTracer();
  final Tracer metricsTracer = io.opentracing.contrib.metrics.Metrics.decorate(tracer, reporter);
  GlobalTracer.register(metricsTracer);

  final HttpHost httpHost = new HttpHost("tweets-elasticsearch", 9200);
  final RestClientBuilder restClientBuilder =
      RestClient.builder(httpHost).setHttpClientConfigCallback(new TracingHttpClientConfigCallback(metricsTracer));
  final RestClient restClient = restClientBuilder.build();
  final ElasticsearchTweetRepository elasticsearchRepository = new ElasticsearchTweetRepository(restClient);

  final Properties consumerConfigs = new Properties();
  consumerConfigs.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "tweets-kafka:9092");
  consumerConfigs.put(ConsumerConfig.GROUP_ID_CONFIG, getName());
  consumerConfigs.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
  consumerConfigs.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
  final KafkaConsumer<Long, String> kafkaConsumer = new KafkaConsumer<>(consumerConfigs, new LongDeserializer(), new StringDeserializer());
  final TracingKafkaConsumer<Long, String> tracingKafkaConsumer = new TracingKafkaConsumer<>(kafkaConsumer, metricsTracer);
  final Runnable kafkaTweetEventConsumer = new KafkaTweetEventConsumer(tracingKafkaConsumer, elasticsearchRepository);
  final ExecutorService executorService = environment.lifecycle().executorService("kafka-consumer").build();
  executorService.submit(kafkaTweetEventConsumer);
}

private void initESClient() throws NumberFormatException,
        UnknownHostException {

    List<HttpHost> httpHostList = hosts.stream().map(hostString -> {
        String[] parsedHost = hostString.split(":");
        String host = parsedHost[0];
        int port = parsedHost.length == 2 ? Integer.valueOf(parsedHost[1]) : 9200;
        return new HttpHost(host, port);
    }).collect(toList());
    List<HttpHost> clusterHosts = unmodifiableList(httpHostList);

    if (config.containsKey("user") && config.containsKey("password")) {
        String user = config.get("user").toString();
        String password = config.get("password").toString();
        final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        credentialsProvider.setCredentials(AuthScope.ANY,
                new UsernamePasswordCredentials(user, password));

        restClient = RestClient.builder(clusterHosts.toArray(new HttpHost[clusterHosts.size()]))
                .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
                    @Override
                    public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
                        return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
                    }
                }).build();
    } else {
        restClient = RestClient.builder(clusterHosts.toArray(new HttpHost[clusterHosts.size()]))
                .build();
    }
    if (this.isSniff) {
        sniffer = Sniffer.builder(restClient).build();
    }

}

private static void startRestClient() throws IOException {
    if (client == null) {
        client = RestClient.builder(new HttpHost(testClusterHost, testClusterRestPort, testClusterScheme)).build();

        securityInstalled = testClusterRunning(false);
        if (securityInstalled) {
            // We have a secured cluster. So we need to create a secured client
            // But first we need to close the previous client we built
            if (client != null) {
                client.close();
            }

            final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
            credentialsProvider.setCredentials(AuthScope.ANY,
                    new UsernamePasswordCredentials(testClusterUser, testClusterPass));

            client = RestClient.builder(new HttpHost(testClusterHost, testClusterRestPort, testClusterScheme))
                    .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
                        @Override
                        public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
                            return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
                        }
                    })
                    .build();
            securityInstalled = testClusterRunning(true);
        }
    }
}

public RestClient createRestClient(ElasticSearchConfig config) {

        List<String> esHosts = config.getRestApiHosts();
        List<HttpHost> hosts = new ArrayList<>();
        for (String host : esHosts) {
            try {
                URL url = new URL(host);
                hosts.add(new HttpHost(url.getHost(), url.getPort()));
            } catch (MalformedURLException e) {
                throw new RuntimeException("Failed to initialize Elasticsearch REST client. "
                        + "Invalid host: " + host, e);
            }
        }

        HttpHost[] httpHostsArray = (HttpHost[]) hosts.toArray(new HttpHost[hosts.size()]);

        client = RestClient.builder(httpHostsArray)
            .setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
                @Override
                public RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
                    return requestConfigBuilder
                            .setConnectTimeout(config.getRestConnectTimeout())
                            .setSocketTimeout(config.getRestSocketTimeout());
                }
            })
            .setMaxRetryTimeoutMillis(config.getRestMaxRetryTimeoutMillis())
            .build();

        logger.info("Initialized Elasticsearch REST client for hosts: "+Arrays.toString(httpHostsArray));
        return client;
    }

public static RestClientBuilder getESHosts() {
    if (isBlank(ES_HOSTS))
        throw new RuntimeException("Cannot get elasticSearch hosts from config! "
                + "Please check es.hosts config option.");
    String[] hosts = ES_HOSTS.split(",");
    int len;
    if ((len = hosts.length) <= 0)
        throw new RuntimeException("Cannot get elasticSearch hosts from config! "
                + "Please check es.hosts config option.");
    String host;
    String[] hostAndPort;
    LinkedList<HttpHost> httpHosts = new LinkedList<>();
    for (int i = 0; i < len; i++) {
        host = hosts[i];
        hostAndPort = host.split(":");
        if (hostAndPort.length != 2) {
            LOG.warn("Invalid es host: {}!", host);
            continue;
        }
        httpHosts.add(new HttpHost(hostAndPort[0], Integer.valueOf(hostAndPort[1]), "http"));
    }
    int size;
    HttpHost[] httpHostsArray = new HttpHost[size = httpHosts.size()];
    for (int i = 0; i < size; i++) {
        httpHostsArray[i] = httpHosts.get(i);
    }
    return RestClient.builder(httpHostsArray);
}

@Before
public void init() {
    // final CredentialsProvider credentialsProvider = new
    // BasicCredentialsProvider();
    // credentialsProvider.setCredentials(AuthScope.ANY, new
    // UsernamePasswordCredentials("user", "password"));

    client = RestClient.builder(new HttpHost("127.0.0.1", 9200, "http")).setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
        @Override
        public RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
            return requestConfigBuilder.setConnectTimeout(5000).setSocketTimeout(30000);
        }
    }).setMaxRetryTimeoutMillis(30000)
            // .setHttpClientConfigCallback(new
            // RestClientBuilder.HttpClientConfigCallback() {
            // @Override
            // public HttpAsyncClientBuilder
            // customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder)
            // {
            // return httpClientBuilder
            // .setDefaultIOReactorConfig(IOReactorConfig.custom().setIoThreadCount(1).build());
            // // .setDefaultCredentialsProvider(credentialsProvider);
            // }
            // })
            .build();

    // Credentials credentials = new UsernamePasswordCredentials("root",
    // "dzh");
    // client.getState().setCredentials(AuthScope.ANY, credentials);
}

private ElasticSearchRestClient createClient(final DataContextProperties properties) throws MalformedURLException {
    final URL url = new URL(properties.getUrl());
    final RestClientBuilder builder = RestClient.builder(new HttpHost(url.getHost(), url.getPort()));

    if (properties.getUsername() != null) {
        final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(properties.getUsername(),
                properties.getPassword()));

        builder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(
                credentialsProvider));
    }

    return new ElasticSearchRestClient(builder.build());
}

protected static RestClient createRestClient(RestClientBuilder.HttpClientConfigCallback httpClientConfigCallback) {
    return createRestClient(httpClientConfigCallback, "http");
}

@VisibleForTesting
RestClient createClient() throws IOException {
  HttpHost[] hosts = new HttpHost[getAddresses().size()];
  int i = 0;
  for (String address : getAddresses()) {
    URL url = new URL(address);
    hosts[i] = new HttpHost(url.getHost(), url.getPort(), url.getProtocol());
    i++;
  }
  RestClientBuilder restClientBuilder = RestClient.builder(hosts);
  if (getUsername() != null) {
    final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
    credentialsProvider.setCredentials(
        AuthScope.ANY, new UsernamePasswordCredentials(getUsername(), getPassword()));
    restClientBuilder.setHttpClientConfigCallback(
        new RestClientBuilder.HttpClientConfigCallback() {
          public HttpAsyncClientBuilder customizeHttpClient(
              HttpAsyncClientBuilder httpAsyncClientBuilder) {
            return httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
          }
        });
  }
  if (getKeystorePath() != null && !getKeystorePath().isEmpty()) {
    try {
      KeyStore keyStore = KeyStore.getInstance("jks");
      try (InputStream is = new FileInputStream(new File(getKeystorePath()))) {
        String keystorePassword = getKeystorePassword();
        keyStore.load(is, (keystorePassword == null) ? null : keystorePassword.toCharArray());
      }
      final SSLContext sslContext = SSLContexts.custom()
          .loadTrustMaterial(keyStore, new TrustSelfSignedStrategy()).build();
      final SSLIOSessionStrategy sessionStrategy = new SSLIOSessionStrategy(sslContext);
      restClientBuilder.setHttpClientConfigCallback(
          new RestClientBuilder.HttpClientConfigCallback() {
        @Override
        public HttpAsyncClientBuilder customizeHttpClient(
            HttpAsyncClientBuilder httpClientBuilder) {
          return httpClientBuilder.setSSLContext(sslContext).setSSLStrategy(sessionStrategy);
        }
      });
    } catch (Exception e) {
      throw new IOException("Can't load the client certificate from the keystore", e);
    }
  }
  return restClientBuilder.build();
}

@Override
public DataStore createDataStore(Map<String, Serializable> params) throws IOException {
    final String searchHost = (String) getValue(HOSTNAME, params);
    final Integer hostPort = (Integer) getValue(HOSTPORT, params);
    final String indexName = (String) INDEX_NAME.lookUp(params);
    final String arrayEncoding = (String) getValue(ARRAY_ENCODING, params);
    final Boolean sslEnabled = (Boolean) getValue(SSL_ENABLED, params);
    final Boolean sslRejectUnauthorized = (Boolean) getValue(SSL_REJECT_UNAUTHORIZED, params);

    final String scheme = sslEnabled ? "https" : "http";
    final RestClientBuilder builder = RestClient.builder(new HttpHost(searchHost, hostPort, scheme));

    if (sslEnabled) {
        builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
            @Override
            public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
                httpClientBuilder.useSystemProperties();
                if (!sslRejectUnauthorized) {
                    httpClientBuilder.setSSLHostnameVerifier((host,session) -> true);
                    try {
                        httpClientBuilder.setSSLContext(SSLContextBuilder.create().loadTrustMaterial((chain,authType) -> true).build());
                    } catch (KeyManagementException | NoSuchAlgorithmException | KeyStoreException e) {
                        throw new UncheckedIOException(new IOException("Unable to create SSLContext", e));
                    }
                }
                return httpClientBuilder;
            }
        });
    }

    final ElasticDataStore dataStore = new ElasticDataStore(builder.build(), indexName);
    dataStore.setDefaultMaxFeatures((Integer) getValue(DEFAULT_MAX_FEATURES, params));
    dataStore.setSourceFilteringEnabled((Boolean) getValue(SOURCE_FILTERING_ENABLED, params));
    dataStore.setScrollEnabled((Boolean)getValue(SCROLL_ENABLED, params));
    dataStore.setScrollSize(((Number)getValue(SCROLL_SIZE, params)).longValue());
    dataStore.setScrollTime((Integer)getValue(SCROLL_TIME_SECONDS, params));
    dataStore.setArrayEncoding(ArrayEncoding.valueOf(arrayEncoding.toUpperCase()));
    dataStore.setGridSize((Long) GRID_SIZE.lookUp(params));
    dataStore.setGridThreshold((Double) GRID_THRESHOLD.lookUp(params));
    return dataStore;
}