@Test public void upsertProduct() throws Exception { Product product = Product.newBuilder() .setProductId(faker.number().randomNumber()) .setProductName(faker.company().name()) .setProductPrice(faker.number().randomDouble(2, 10, 100)) .setProductStatus(ProductStatus.InStock) .build(); productDao.upsertProduct(product); esClient.admin().indices().flush(Requests.flushRequest(INDEX)).actionGet(); GetResponse getResponse = esClient.prepareGet(INDEX, TYPE, String.valueOf(product.getProductId())).get(); JsonFormat.Parser jsonParser = injector.getInstance(JsonFormat.Parser.class); Product.Builder builder = Product.newBuilder(); jsonParser.merge(getResponse.getSourceAsString(), builder); assertThat(builder.build()).isEqualTo(product); }
@Override public void process(DeviceEvent element, RuntimeContext ctx, RequestIndexer indexer) { Map<String, Object> json = new HashMap<>(); json.put("phoneNumber", element.getPhoneNumber()); json.put("bin", element.getBin()); json.put("bout", element.getBout()); json.put("timestamp", element.getTimestamp()); System.out.println(json); IndexRequest source = Requests.indexRequest() .index("flink-test") .type("flink-log") .source(json); indexer.add(source); }
@Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { BulkRequest bulkRequest = Requests.bulkRequest(); String defaultIndex = request.param("index"); String defaultType = request.param("type"); String defaultRouting = request.param("routing"); String fieldsParam = request.param("fields"); String defaultPipeline = request.param("pipeline"); String[] defaultFields = fieldsParam != null ? Strings.commaDelimitedListToStringArray(fieldsParam) : null; String waitForActiveShards = request.param("wait_for_active_shards"); if (waitForActiveShards != null) { bulkRequest.waitForActiveShards(ActiveShardCount.parseString(waitForActiveShards)); } bulkRequest.timeout(request.paramAsTime("timeout", BulkShardRequest.DEFAULT_TIMEOUT)); bulkRequest.setRefreshPolicy(request.param("refresh")); bulkRequest.add(request.content(), defaultIndex, defaultType, defaultRouting, defaultFields, null, defaultPipeline, null, true, request.getXContentType()); // short circuit the call to the transport layer return channel -> { BulkRestBuilderListener listener = new BulkRestBuilderListener(channel, request); listener.onResponse(bulkRequest); }; }
/** * Waits for all relocating shards to become active and the cluster has reached the given health status * using the cluster health API. */ public ClusterHealthStatus waitForRelocation(ClusterHealthStatus status) { ClusterHealthRequest request = Requests.clusterHealthRequest().waitForNoRelocatingShards(true); if (status != null) { request.waitForStatus(status); } ClusterHealthResponse actionGet = client().admin().cluster() .health(request).actionGet(); if (actionGet.isTimedOut()) { logger.info("waitForRelocation timed out (status={}), cluster state:\n{}\n{}", status, client().admin().cluster().prepareState().get().getState(), client().admin().cluster().preparePendingClusterTasks().get()); assertThat("timed out waiting for relocation", actionGet.isTimedOut(), equalTo(false)); } if (status != null) { assertThat(actionGet.getStatus(), equalTo(status)); } return actionGet.getStatus(); }
@Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { BulkRequest bulkRequest = Requests.bulkRequest(); String defaultIndex = request.param("index"); String defaultType = request.param("type"); String defaultRouting = request.param("routing"); FetchSourceContext defaultFetchSourceContext = FetchSourceContext.parseFromRestRequest(request); String fieldsParam = request.param("fields"); if (fieldsParam != null) { DEPRECATION_LOGGER.deprecated("Deprecated field [fields] used, expected [_source] instead"); } String[] defaultFields = fieldsParam != null ? Strings.commaDelimitedListToStringArray(fieldsParam) : null; String defaultPipeline = request.param("pipeline"); String waitForActiveShards = request.param("wait_for_active_shards"); if (waitForActiveShards != null) { bulkRequest.waitForActiveShards(ActiveShardCount.parseString(waitForActiveShards)); } bulkRequest.timeout(request.paramAsTime("timeout", BulkShardRequest.DEFAULT_TIMEOUT)); bulkRequest.setRefreshPolicy(request.param("refresh")); bulkRequest.add(request.content(), defaultIndex, defaultType, defaultRouting, defaultFields, defaultFetchSourceContext, defaultPipeline, null, allowExplicitIndex, request.getXContentType()); return channel -> client.bulk(bulkRequest, new RestStatusToXContentListener<>(channel)); }
void assertRealtimeGetWorks(String indexName) { assertAcked(client().admin().indices().prepareUpdateSettings(indexName).setSettings(Settings.builder() .put("refresh_interval", -1) .build())); SearchRequestBuilder searchReq = client().prepareSearch(indexName).setQuery(QueryBuilders.matchAllQuery()); SearchHit hit = searchReq.get().getHits().getAt(0); String docId = hit.getId(); // foo is new, it is not a field in the generated index client().prepareUpdate(indexName, "doc", docId).setDoc(Requests.INDEX_CONTENT_TYPE, "foo", "bar").get(); GetResponse getRsp = client().prepareGet(indexName, "doc", docId).get(); Map<String, Object> source = getRsp.getSourceAsMap(); assertThat(source, Matchers.hasKey("foo")); assertAcked(client().admin().indices().prepareUpdateSettings(indexName).setSettings(Settings.builder() .put("refresh_interval", IndexSettings.DEFAULT_REFRESH_INTERVAL) .build())); }
public void testBulkRequestExecution() throws Exception { BulkRequest bulkRequest = new BulkRequest(); String pipelineId = "_id"; int numRequest = scaledRandomIntBetween(8, 64); for (int i = 0; i < numRequest; i++) { IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").setPipeline(pipelineId); indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field1", "value1"); bulkRequest.add(indexRequest); } when(store.get(pipelineId)).thenReturn(new Pipeline(pipelineId, null, version, new CompoundProcessor())); @SuppressWarnings("unchecked") BiConsumer<IndexRequest, Exception> requestItemErrorHandler = mock(BiConsumer.class); @SuppressWarnings("unchecked") Consumer<Exception> completionHandler = mock(Consumer.class); executionService.executeBulkRequest(bulkRequest.requests(), requestItemErrorHandler, completionHandler); verify(requestItemErrorHandler, never()).accept(any(), any()); verify(completionHandler, times(1)).accept(null); }
public void testBulk() { // Index by bulk with RefreshPolicy.WAIT_UNTIL BulkRequestBuilder bulk = client().prepareBulk().setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); bulk.add(client().prepareIndex("test", "test", "1").setSource("foo", "bar")); assertBulkSuccess(bulk.get()); assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")).get(), "1"); // Update by bulk with RefreshPolicy.WAIT_UNTIL bulk = client().prepareBulk().setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); bulk.add(client().prepareUpdate("test", "test", "1").setDoc(Requests.INDEX_CONTENT_TYPE, "foo", "baz")); assertBulkSuccess(bulk.get()); assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "baz")).get(), "1"); // Delete by bulk with RefreshPolicy.WAIT_UNTIL bulk = client().prepareBulk().setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); bulk.add(client().prepareDelete("test", "test", "1")); assertBulkSuccess(bulk.get()); assertNoSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")).get()); // Update makes a noop bulk = client().prepareBulk().setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); bulk.add(client().prepareDelete("test", "test", "1")); assertBulkSuccess(bulk.get()); }
public void testDynamicDisabled() { IndexRequest request = new IndexRequest("index", "type", "1"); request.source(Requests.INDEX_CONTENT_TYPE, "foo", 3); BulkRequest bulkRequest = new BulkRequest(); bulkRequest.add(request); final AtomicBoolean onFailureCalled = new AtomicBoolean(); transportBulkAction.execute(bulkRequest, new ActionListener<BulkResponse>() { @Override public void onResponse(BulkResponse bulkResponse) { fail("onResponse shouldn't be called"); } @Override public void onFailure(Exception e) { onFailureCalled.set(true); assertThat(e, instanceOf(IndexNotFoundException.class)); assertEquals("no such index and [index.mapper.dynamic] is [false]", e.getMessage()); } }); assertTrue(onFailureCalled.get()); }
public void testThreadedListeners() throws Throwable { final CountDownLatch latch = new CountDownLatch(1); final AtomicReference<Throwable> failure = new AtomicReference<>(); final AtomicReference<String> threadName = new AtomicReference<>(); Client client = client(); IndexRequest request = new IndexRequest("test", "type", "1"); if (randomBoolean()) { // set the source, without it, we will have a verification failure request.source(Requests.INDEX_CONTENT_TYPE, "field1", "value1"); } client.index(request, new ActionListener<IndexResponse>() { @Override public void onResponse(IndexResponse indexResponse) { threadName.set(Thread.currentThread().getName()); latch.countDown(); } @Override public void onFailure(Exception e) { threadName.set(Thread.currentThread().getName()); failure.set(e); latch.countDown(); } }); latch.await(); boolean shouldBeThreaded = TransportClient.CLIENT_TYPE.equals(Client.CLIENT_TYPE_SETTING_S.get(client.settings())); if (shouldBeThreaded) { assertTrue(threadName.get().contains("listener")); } else { assertFalse(threadName.get().contains("listener")); } }
public void testExecuteBulkIndexRequestWithRejection() throws Exception { IndexMetaData metaData = indexMetaData(); IndexShard shard = newStartedShard(true); BulkItemRequest[] items = new BulkItemRequest[1]; DocWriteRequest writeRequest = new IndexRequest("index", "type", "id").source(Requests.INDEX_CONTENT_TYPE, "foo", "bar"); items[0] = new BulkItemRequest(0, writeRequest); BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, RefreshPolicy.NONE, items); Translog.Location location = new Translog.Location(0, 0, 0); UpdateHelper updateHelper = null; // Pretend the mappings haven't made it to the node yet, and throw a rejection Exception err = new ReplicationOperation.RetryOnPrimaryException(shardId, "rejection"); try { TransportShardBulkAction.executeBulkItemRequest(metaData, shard, bulkShardRequest, location, 0, updateHelper, threadPool::absoluteTimeInMillis, new ThrowingMappingUpdatePerformer(err)); fail("should have thrown a retry exception"); } catch (ReplicationOperation.RetryOnPrimaryException e) { assertThat(e, equalTo(err)); } closeShards(shard); }
public void testNoopUpdateReplicaRequest() throws Exception { DocWriteRequest writeRequest = new IndexRequest("index", "type", "id").source(Requests.INDEX_CONTENT_TYPE, "field", "value"); BulkItemRequest replicaRequest = new BulkItemRequest(0, writeRequest); DocWriteResponse noopUpdateResponse = new UpdateResponse(shardId, "index", "id", 0, DocWriteResponse.Result.NOOP); BulkItemResultHolder noopResults = new BulkItemResultHolder(noopUpdateResponse, null, replicaRequest); Translog.Location location = new Translog.Location(0, 0, 0); BulkItemRequest[] items = new BulkItemRequest[0]; BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, RefreshPolicy.NONE, items); Translog.Location newLocation = TransportShardBulkAction.updateReplicaRequest(noopResults, DocWriteRequest.OpType.UPDATE, location, bulkShardRequest); BulkItemResponse primaryResponse = replicaRequest.getPrimaryResponse(); // Basically nothing changes in the request since it's a noop assertThat(newLocation, equalTo(location)); assertThat(primaryResponse.getItemId(), equalTo(0)); assertThat(primaryResponse.getId(), equalTo("id")); assertThat(primaryResponse.getOpType(), equalTo(DocWriteRequest.OpType.UPDATE)); assertThat(primaryResponse.getResponse(), equalTo(noopUpdateResponse)); assertThat(primaryResponse.getResponse().getResult(), equalTo(DocWriteResponse.Result.NOOP)); }
public void testThatMissingIndexDoesNotAbortFullBulkRequest() throws Exception{ createIndex("bulkindex1", "bulkindex2"); BulkRequest bulkRequest = new BulkRequest(); bulkRequest.add(new IndexRequest("bulkindex1", "index1_type", "1").source(Requests.INDEX_CONTENT_TYPE, "text", "hallo1")) .add(new IndexRequest("bulkindex2", "index2_type", "1").source(Requests.INDEX_CONTENT_TYPE, "text", "hallo2")) .add(new IndexRequest("bulkindex2", "index2_type").source(Requests.INDEX_CONTENT_TYPE, "text", "hallo2")) .add(new UpdateRequest("bulkindex2", "index2_type", "2").doc(Requests.INDEX_CONTENT_TYPE, "foo", "bar")) .add(new DeleteRequest("bulkindex2", "index2_type", "3")) .setRefreshPolicy(RefreshPolicy.IMMEDIATE); client().bulk(bulkRequest).get(); SearchResponse searchResponse = client().prepareSearch("bulkindex*").get(); assertHitCount(searchResponse, 3); assertAcked(client().admin().indices().prepareClose("bulkindex2")); BulkResponse bulkResponse = client().bulk(bulkRequest).get(); assertThat(bulkResponse.hasFailures(), is(true)); assertThat(bulkResponse.getItems().length, is(5)); }
public void testFailedRequestsOnClosedIndex() throws Exception { createIndex("bulkindex1"); client().prepareIndex("bulkindex1", "index1_type", "1").setSource("text", "test").get(); assertAcked(client().admin().indices().prepareClose("bulkindex1")); BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(RefreshPolicy.IMMEDIATE); bulkRequest.add(new IndexRequest("bulkindex1", "index1_type", "1").source(Requests.INDEX_CONTENT_TYPE, "text", "hallo1")) .add(new UpdateRequest("bulkindex1", "index1_type", "1").doc(Requests.INDEX_CONTENT_TYPE, "foo", "bar")) .add(new DeleteRequest("bulkindex1", "index1_type", "1")); BulkResponse bulkResponse = client().bulk(bulkRequest).get(); assertThat(bulkResponse.hasFailures(), is(true)); BulkItemResponse[] responseItems = bulkResponse.getItems(); assertThat(responseItems.length, is(3)); assertThat(responseItems[0].getOpType(), is(OpType.INDEX)); assertThat(responseItems[1].getOpType(), is(OpType.UPDATE)); assertThat(responseItems[2].getOpType(), is(OpType.DELETE)); }
@Override public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) { ClusterStateRequest clusterStateRequest = Requests.clusterStateRequest() .routingTable(false) .nodes(false); clusterStateRequest.local(request.paramAsBoolean("local", clusterStateRequest.local())); client.admin().cluster().state(clusterStateRequest, new RestBuilderListener<ClusterStateResponse>(channel) { @Override public RestResponse buildResponse(ClusterStateResponse response, XContentBuilder builder) throws Exception { builder.startObject(); builder.startObject("persistent"); response.getState().metaData().persistentSettings().toXContent(builder, request); builder.endObject(); builder.startObject("transient"); response.getState().metaData().transientSettings().toXContent(builder, request); builder.endObject(); builder.endObject(); return new BytesRestResponse(RestStatus.OK, builder); } }); }
/** * Creates a new Elasticsearch request from the given element. * @param value the element to process. * @return the Elasticsearch request. */ private IndexRequest createWindowWordRanking(PlayerRunningStatistics value) { String json = "{\"tsStart\":" + value.getTsStart() + ",\"tsStop\":" + value.getTsStop() + ",\"pid\":" + value.getPid() + ",\"totalDistance\":" + value.getTotalDistance() + ",\"averageSpeed\":" + value.getAverageSpeed() + "}"; //LOG.debug("JSON: {}", json); return Requests.indexRequest() .index(this.indexName) .type(this.typeName) .source(json); }
/** * Creates a new Elasticsearch request from the given element. * @param value the element to process. * @return the Elasticsearch request. */ private IndexRequest createWindowWordRanking(PlayersSpeedRanking value) { String rankJson = value.getRank().stream() .map(e -> "{" + "\"pid\":" + e.getPid() + ",\"averageSpeed\":" + e.getAverageSpeed() + "}") .collect(Collectors.joining(",")); String json = "{\"tsStart\":" + value.getTsStart() + ",\"tsStop\":" + value.getTsStop() + ",\"rank\":[" + rankJson + "]}"; //LOG.debug("JSON: {}", json); return Requests.indexRequest() .index(this.indexName) .type(this.typeName) .source(json); }
/** * Creates a new Elasticsearch request from the given element. * @param value the element to process. * @return the Elasticsearch request. */ private IndexRequest createWindowWordRanking(PlayerGridStatistics value) { String cellsJson = value.getStats().entrySet().stream() .map(e -> "{" + "\"cid\":\"" + e.getKey() + "\",\"presence\":" + e.getValue() + "}") .collect(Collectors.joining(",")); String json = "{\"ts\":" + value.getTsStart() + ",\"pid\":" + value.getPid() + ",\"cells\":[" + cellsJson + "]}"; //LOG.debug("JSON: {}", json); return Requests.indexRequest() .index(this.indexName) .type(this.typeName) .source(json); }
@Override public void process( Tuple5<Float, Float, Long, Boolean, Integer> record, RuntimeContext ctx, RequestIndexer indexer) { // construct JSON document to index Map<String, String> json = new HashMap<>(); json.put("time", record.f2.toString()); // timestamp json.put("location", record.f1+","+record.f0); // lat,lon pair json.put("isStart", record.f3.toString()); // isStart json.put("cnt", record.f4.toString()); // count IndexRequest rqst = Requests.indexRequest() .index("nyc-places") // index name .type("popular-locations") // mapping name .source(json); indexer.add(rqst); }
private IndexMetaData getIndexMetaData(String indexName) { AliasOrIndex aliasOrIndex = elasticsearchTemplate.getClient() .admin() .cluster() .state(Requests.clusterStateRequest()) .actionGet() .getState() .getMetaData() .getAliasAndIndexLookup() .get(indexName); if (aliasOrIndex == null) { return null; } return aliasOrIndex.getIndices().get(0); }
private static SearchResponse zkfc(String indexName, String zkType, TransportClient client) throws IOException { //返回一个可以执行管理性操作的客户端 //1) cluster(),产生一个允许从集群中执行action或操作的client; //2) indices(),产生一个允许从索引中执行action或操作的client。 //创建zk分词 PutMappingRequest mapping = Requests.putMappingRequest(indexName).type(zkType).source(createIKMapping(zkType).string()); client.admin().indices().putMapping(mapping).actionGet(); Goods goodsOne= new Goods( 1,"iphone7 iphone7plus 钢化膜 玻璃膜 苹果 苹果7/7plus 贴膜 买就送清水","http://m.ule.com/item/detail/1771161"); Goods goodsTwo=new Goods( 2,"苹果 (Apple) iPhone 7 移动联通电信4G手机 土豪金 32G 标配","http://m.ule.com/item/detail/1799356"); Goods goodsThree=new Goods( 3,"苹果 Apple iPhone 7 (A1660) 128G 金色 移动联通电信 全网通 4G手机","http://m.ule.com/item/detail/1781429"); client.prepareIndex(indexName,zkType).setId(1+"").setSource(JSONObject.toJSONString(goodsOne)).execute().actionGet(); client.prepareIndex(indexName,zkType).setId(2+"").setSource(JSONObject.toJSONString(goodsTwo)).execute().actionGet(); client.prepareIndex(indexName,zkType).setId(3+"").setSource(JSONObject.toJSONString(goodsThree)).execute().actionGet(); SearchResponse response = client.prepareSearch(indexName) .setTypes(zkType) .setSearchType(SearchType.DFS_QUERY_THEN_FETCH) .setQuery( QueryBuilders.matchQuery("title", "苹果") ).execute().actionGet(); return response; }
public void testBadValues() throws IOException { Map<String, Object> map = new HashMap<>(); XContentBuilder builder = XContentBuilder.builder(Requests.INDEX_CONTENT_TYPE.xContent()); BytesReference bytes = builder.map(map).bytes(); assertThat(expectThrows(IllegalArgumentException.class, () -> IndexFeatureStore.parse(StoredFeature.class, StoredFeature.TYPE, bytes)) .getMessage(), equalTo("No StorableElement found.")); builder = XContentBuilder.builder(Requests.INDEX_CONTENT_TYPE.xContent()); map.put("featureset", LtrTestUtils.randomFeatureSet()); BytesReference bytes2 = builder.map(map).bytes(); assertThat(expectThrows(IllegalArgumentException.class, () -> IndexFeatureStore.parse(StoredFeature.class, StoredFeature.TYPE, bytes2)) .getMessage(), equalTo("Expected an element of type [" + StoredFeature.TYPE + "] but" + " got [" + StoredFeatureSet.TYPE + "].")); }
@Override public void run() { while(true) { try { logger.debug("getting es cluster health."); ActionFuture<ClusterHealthResponse> healthFuture = transportClient.admin().cluster().health(Requests.clusterHealthRequest()); ClusterHealthResponse healthResponse = healthFuture.get(5, TimeUnit.SECONDS); logger.debug("Get num of node:{}", healthResponse.getNumberOfNodes()); logger.debug("Get cluster health:{} ", healthResponse.getStatus()); isClusterOn.getAndSet(true); } catch(Throwable t) { if(t instanceof NoNodeAvailableException){//集群不可用 logger.error("the cluster no node avaliable."); isClusterOn.getAndSet(false); }else{ isClusterOn.getAndSet(true); } } try { Thread.sleep(1000);//FIXME } catch (InterruptedException ie) { ie.printStackTrace(); } } }
public ElasticsearchSink<Document> getElasticSink() { Map<String, String> config = Maps.newHashMap(); config.put("bulk.flush.max.actions", "1"); config.put("cluster.name", clusterName); config.put("discovery.zen.ping.multicast.enabled", "false"); config.put("discovery.zen.ping.unicast.hosts", "localhost"); return new ElasticsearchSink<>(config, new IndexRequestBuilder<Document>() { private static final long serialVersionUID = 5670092038059852584L; @Override public IndexRequest createIndexRequest(Document element, RuntimeContext ctx) { Map<String, Object> json = new HashMap<>(); json.put("data", element); return Requests.indexRequest().index("grades").type("student").source(json); } }); }
@Test public void shouldIndexOneEvent() throws Exception { Configurables.configure(fixture, new Context(parameters)); Channel channel = bindAndStartChannel(fixture); Transaction tx = channel.getTransaction(); tx.begin(); Event event = EventBuilder.withBody("event #1 or 1".getBytes()); channel.put(event); tx.commit(); tx.close(); fixture.process(); fixture.stop(); client.admin().indices() .refresh(Requests.refreshRequest(timestampedIndexName)).actionGet(); assertMatchAllQuery(1, event); assertBodyQuery(1, event); }
/** * Ensures the cluster has a green state via the cluster health API. This method will also wait for relocations. * It is useful to ensure that all action on the cluster have finished and all shards that were currently relocating * are now allocated and started. * * @param timeout * time out value to set on {@link org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest} */ public ClusterHealthStatus ensureGreen(final TimeValue timeout, final String... indices) { final ClusterHealthResponse actionGet = client() .admin() .cluster() .health(Requests.clusterHealthRequest(indices).timeout(timeout).waitForGreenStatus().waitForEvents(Priority.LANGUID) .waitForRelocatingShards(0)).actionGet(); if (actionGet.isTimedOut()) { logger.info("ensureGreen timed out, cluster state:\n{}\n{}", client().admin().cluster().prepareState().get().getState() .prettyPrint(), client().admin().cluster().preparePendingClusterTasks().get().prettyPrint()); assertThat("timed out waiting for green state", actionGet.isTimedOut(), equalTo(false)); } assertThat(actionGet.getStatus(), equalTo(ClusterHealthStatus.GREEN)); logger.debug("indices {} are green", indices.length == 0 ? "[_all]" : indices); return actionGet.getStatus(); }
/** * Waits for all relocating shards to become active and the cluster has reached the given health status * using the cluster health API. */ public ClusterHealthStatus waitForRelocation(final ClusterHealthStatus status) { final ClusterHealthRequest request = Requests.clusterHealthRequest().waitForRelocatingShards(0); if (status != null) { request.waitForStatus(status); } final ClusterHealthResponse actionGet = client().admin().cluster().health(request).actionGet(); if (actionGet.isTimedOut()) { logger.info("waitForRelocation timed out (status={}), cluster state:\n{}\n{}", status, client().admin().cluster() .prepareState().get().getState().prettyPrint(), client().admin().cluster().preparePendingClusterTasks().get() .prettyPrint()); assertThat("timed out waiting for relocation", actionGet.isTimedOut(), equalTo(false)); } if (status != null) { assertThat(actionGet.getStatus(), equalTo(status)); } return actionGet.getStatus(); }
@BeforeClass public void prepareTest() throws Exception { Config reference = ConfigFactory.load(); File conf_file = new File("target/test-classes/TwitterUserstreamElasticsearchIT.conf"); assert(conf_file.exists()); Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false)); Config typesafe = testResourceConfig.withFallback(reference).resolve(); testConfiguration = new ComponentConfigurator<>(TwitterUserstreamElasticsearchConfiguration.class).detectConfiguration(typesafe); testClient = ElasticsearchClientManager.getInstance(testConfiguration.getElasticsearch()).client(); ClusterHealthRequest clusterHealthRequest = Requests.clusterHealthRequest(); ClusterHealthResponse clusterHealthResponse = testClient.admin().cluster().health(clusterHealthRequest).actionGet(); assertNotEquals(clusterHealthResponse.getStatus(), ClusterHealthStatus.RED); IndicesExistsRequest indicesExistsRequest = Requests.indicesExistsRequest(testConfiguration.getElasticsearch().getIndex()); IndicesExistsResponse indicesExistsResponse = testClient.admin().indices().exists(indicesExistsRequest).actionGet(); if(indicesExistsResponse.isExists()) { DeleteIndexRequest deleteIndexRequest = Requests.deleteIndexRequest(testConfiguration.getElasticsearch().getIndex()); DeleteIndexResponse deleteIndexResponse = testClient.admin().indices().delete(deleteIndexRequest).actionGet(); assertTrue(deleteIndexResponse.isAcknowledged()); }; }
@BeforeClass public void prepareTest() throws Exception { Config reference = ConfigFactory.load(); File conf_file = new File("target/test-classes/MongoElasticsearchSyncIT.conf"); assert(conf_file.exists()); Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false)); Config typesafe = testResourceConfig.withFallback(reference).resolve(); testConfiguration = new ComponentConfigurator<>(MongoElasticsearchSyncConfiguration.class).detectConfiguration(typesafe); testClient = ElasticsearchClientManager.getInstance(testConfiguration.getDestination()).client(); ClusterHealthRequest clusterHealthRequest = Requests.clusterHealthRequest(); ClusterHealthResponse clusterHealthResponse = testClient.admin().cluster().health(clusterHealthRequest).actionGet(); assertNotEquals(clusterHealthResponse.getStatus(), ClusterHealthStatus.RED); IndicesExistsRequest indicesExistsRequest = Requests.indicesExistsRequest(testConfiguration.getDestination().getIndex()); IndicesExistsResponse indicesExistsResponse = testClient.admin().indices().exists(indicesExistsRequest).actionGet(); assertFalse(indicesExistsResponse.isExists()); }
@Test public void testSync() throws Exception { MongoElasticsearchSync sync = new MongoElasticsearchSync(testConfiguration); sync.run(); IndicesExistsRequest indicesExistsRequest = Requests.indicesExistsRequest(testConfiguration.getDestination().getIndex()); IndicesExistsResponse indicesExistsResponse = testClient.admin().indices().exists(indicesExistsRequest).actionGet(); assertTrue(indicesExistsResponse.isExists()); // assert lines in file SearchRequestBuilder countRequest = testClient .prepareSearch(testConfiguration.getDestination().getIndex()) .setTypes(testConfiguration.getDestination().getType()); SearchResponse countResponse = countRequest.execute().actionGet(); assertEquals(89, (int)countResponse.getHits().getTotalHits()); }
@BeforeClass public void prepareTest() throws Exception { Config reference = ConfigFactory.load(); File conf_file = new File("target/test-classes/HdfsElasticsearchIT.conf"); assert(conf_file.exists()); Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false)); Config typesafe = testResourceConfig.withFallback(reference).resolve(); testConfiguration = new ComponentConfigurator<>(HdfsElasticsearchConfiguration.class).detectConfiguration(typesafe); testClient = ElasticsearchClientManager.getInstance(testConfiguration.getDestination()).client(); ClusterHealthRequest clusterHealthRequest = Requests.clusterHealthRequest(); ClusterHealthResponse clusterHealthResponse = testClient.admin().cluster().health(clusterHealthRequest).actionGet(); assertNotEquals(clusterHealthResponse.getStatus(), ClusterHealthStatus.RED); IndicesExistsRequest indicesExistsRequest = Requests.indicesExistsRequest(testConfiguration.getDestination().getIndex()); IndicesExistsResponse indicesExistsResponse = testClient.admin().indices().exists(indicesExistsRequest).actionGet(); if(indicesExistsResponse.isExists()) { DeleteIndexRequest deleteIndexRequest = Requests.deleteIndexRequest(testConfiguration.getDestination().getIndex()); DeleteIndexResponse deleteIndexResponse = testClient.admin().indices().delete(deleteIndexRequest).actionGet(); assertTrue(deleteIndexResponse.isAcknowledged()); }; }
@BeforeClass public void prepareTest() throws Exception { Config reference = ConfigFactory.load(); File conf_file = new File("target/test-classes/TwitterHistoryElasticsearchIT.conf"); assert(conf_file.exists()); Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false)); Config typesafe = testResourceConfig.withFallback(reference).resolve(); testConfiguration = new ComponentConfigurator<>(TwitterHistoryElasticsearchConfiguration.class).detectConfiguration(typesafe); testClient = ElasticsearchClientManager.getInstance(testConfiguration.getElasticsearch()).client(); ClusterHealthRequest clusterHealthRequest = Requests.clusterHealthRequest(); ClusterHealthResponse clusterHealthResponse = testClient.admin().cluster().health(clusterHealthRequest).actionGet(); assertNotEquals(clusterHealthResponse.getStatus(), ClusterHealthStatus.RED); IndicesExistsRequest indicesExistsRequest = Requests.indicesExistsRequest(testConfiguration.getElasticsearch().getIndex()); IndicesExistsResponse indicesExistsResponse = testClient.admin().indices().exists(indicesExistsRequest).actionGet(); if(indicesExistsResponse.isExists()) { DeleteIndexRequest deleteIndexRequest = Requests.deleteIndexRequest(testConfiguration.getElasticsearch().getIndex()); DeleteIndexResponse deleteIndexResponse = testClient.admin().indices().delete(deleteIndexRequest).actionGet(); assertTrue(deleteIndexResponse.isAcknowledged()); }; }
@Override public String persist(final Event event) { String evStr = null; try { evStr = jsonMapper.writeValueAsString(event); final IndexRequest indexRequest = Requests .indexRequest(indexNamingStrategy.buildActiveName(event.getSnifferId())) .type(getType(event.getSnifferId())).source(evStr); final String eventId = clientTpl.executeWithClient(new ClientCallback<IndexResponse>() { @Override public IndexResponse execute(final Client client) { return client.index(indexRequest).actionGet(); } }).getId(); logger.debug("Persisted event with id: {}", eventId); return eventId; } catch (final Exception e) { throw new DataAccessException("Failed to persiste event: " + evStr, e); } }
public Client createClient() { final AtomicReference<Client> elasticsearchClient = new AtomicReference<>(); await().atMost(30, TimeUnit.SECONDS).pollDelay(1, TimeUnit.SECONDS).until(() -> { Client c = new TransportClient(ImmutableSettings.settingsBuilder().put("cluster.name", elasticsearchClusterName).build()).addTransportAddress(new InetSocketTransportAddress(getIpAddress(), 9300)); try { c.admin().cluster().health(Requests.clusterHealthRequest("_all")).actionGet(); } catch (ElasticsearchException e) { c.close(); return false; } elasticsearchClient.set(c); return true; }); assertEquals(elasticsearchClusterName, elasticsearchClient.get().admin().cluster().health(Requests.clusterHealthRequest("_all")).actionGet().getClusterName()); return elasticsearchClient.get(); }
/** * Deletes an index and block until deletion is complete. * * @param index The index to delete * @param client The client which points to the Elasticsearch instance * @throws InterruptedException if blocking thread is interrupted or index existence check failed * @throws java.util.concurrent.ExecutionException if index existence check failed * @throws IOException if deletion failed */ static void deleteIndex(String index, Client client) throws InterruptedException, java.util.concurrent.ExecutionException, IOException { IndicesAdminClient indices = client.admin().indices(); IndicesExistsResponse indicesExistsResponse = indices.exists(new IndicesExistsRequest(index)).get(); if (indicesExistsResponse.isExists()) { indices.prepareClose(index).get(); // delete index is an asynchronous request, neither refresh or upgrade // delete all docs before starting tests. WaitForYellow() and delete directory are too slow, // so block thread until it is done (make it synchronous!!!) AtomicBoolean indexDeleted = new AtomicBoolean(false); AtomicBoolean waitForIndexDeletion = new AtomicBoolean(true); indices.delete( Requests.deleteIndexRequest(index), new DeleteActionListener(indexDeleted, waitForIndexDeletion)); while (waitForIndexDeletion.get()) { Thread.sleep(100); } if (!indexDeleted.get()) { throw new IOException("Failed to delete index " + index); } } }
@Test public void should_create_index_with_settings() throws ExecutionException, InterruptedException { String settings = TestFilesUtils.readFromClasspath("com/github/obourgain/elasticsearch/http/handler/admin/indices/create_index_with_settings.json"); CreateIndexResponse response = httpClient.admin().indices() .createIndex(Requests.createIndexRequest(THE_INDEX) .settings(settings)) .get(); Assertions.assertThat(response.isAcknowledged()).isTrue(); GetSettingsResponse getSettingsResponse = transportClient.admin().indices().getSettings(new GetSettingsRequest().indices(THE_INDEX)).actionGet(); ImmutableOpenMap<String, Settings> indexToSettings = getSettingsResponse.getIndexToSettings(); Assertions.assertThat(indexToSettings.iterator().hasNext()).isTrue(); Assertions.assertThat(indexToSettings.iterator().next().key).isEqualTo(THE_INDEX); Settings expectedSettings = ImmutableSettings.builder().loadFromSource(settings).build(); Settings actualSettings = indexToSettings.get(THE_INDEX); assertSettingsEquals(expectedSettings, actualSettings); }
@Test public void should_create_index_with_mapping() throws ExecutionException, InterruptedException { String mapping = TestFilesUtils.readFromClasspath("com/github/obourgain/elasticsearch/http/handler/admin/indices/create_index_with_mapping.json"); CreateIndexResponse response = httpClient.admin().indices() .createIndex(Requests.createIndexRequest(THE_INDEX) .mapping(THE_TYPE, mapping)) .get(); Assertions.assertThat(response.isAcknowledged()).isTrue(); refresh(); GetMappingsResponse getMappingsResponse = transportClient.admin().indices().getMappings(new GetMappingsRequest().indices(THE_INDEX)).actionGet(); ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> indexToMappings = getMappingsResponse.getMappings(); Assertions.assertThat(indexToMappings.iterator().hasNext()).isTrue(); Assertions.assertThat(indexToMappings.iterator().next().key).isEqualTo(THE_INDEX); MappingMetaData actualMapping = indexToMappings.get(THE_INDEX).get(THE_TYPE); assertMappingsEquals(mappingAsJsonToMap(mapping), actualMapping); }
@Before public void setUpClient() throws IOException, InterruptedException, NoSuchFieldException, IllegalAccessException { NodeInfo[] nodes = admin().cluster().nodesInfo(Requests.nodesInfoRequest()).actionGet().getNodes(); Assert.assertThat(nodes.length, Matchers.greaterThanOrEqualTo(1)); TransportAddress transportAddress = nodes[0].getHttp().getAddress().publishAddress(); Assert.assertEquals(InetSocketTransportAddress.class, transportAddress.getClass()); InetSocketTransportAddress inetSocketTransportAddress = (InetSocketTransportAddress) transportAddress; InetSocketAddress socketAddress = inetSocketTransportAddress.address(); String url = String.format("http://%s:%d", socketAddress.getHostName(), socketAddress.getPort()); httpClient = new HttpClient(Collections.singleton(url)); createIndex("the_index"); ensureSearchable("the_index"); }