public void testPassQueryOrFilterAsJSONString() throws Exception { createIndex("test"); client().prepareIndex("test", "type1", "1").setSource("field1", "value1_1", "field2", "value2_1").setRefreshPolicy(IMMEDIATE).get(); WrapperQueryBuilder wrapper = new WrapperQueryBuilder("{ \"term\" : { \"field1\" : \"value1_1\" } }"); assertHitCount(client().prepareSearch().setQuery(wrapper).get(), 1L); BoolQueryBuilder bool = boolQuery().must(wrapper).must(new TermQueryBuilder("field2", "value2_1")); assertHitCount(client().prepareSearch().setQuery(bool).get(), 1L); WrapperQueryBuilder wrapperFilter = wrapperQuery("{ \"term\" : { \"field1\" : \"value1_1\" } }"); assertHitCount(client().prepareSearch().setPostFilter(wrapperFilter).get(), 1L); }
public void testRewrite() throws IOException { FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(new WrapperQueryBuilder(new TermQueryBuilder("foo", "bar").toString())) .boostMode(CombineFunction.REPLACE) .scoreMode(FiltersFunctionScoreQuery.ScoreMode.SUM) .setMinScore(1) .maxBoost(100); FunctionScoreQueryBuilder rewrite = (FunctionScoreQueryBuilder) functionScoreQueryBuilder.rewrite(createShardContext()); assertNotSame(functionScoreQueryBuilder, rewrite); assertEquals(rewrite.query(), new TermQueryBuilder("foo", "bar")); assertEquals(rewrite.boostMode(), CombineFunction.REPLACE); assertEquals(rewrite.scoreMode(), FiltersFunctionScoreQuery.ScoreMode.SUM); assertEquals(rewrite.getMinScore(), 1f, 0.0001); assertEquals(rewrite.maxBoost(), 100f, 0.0001); }
public void testRewriteWithFunction() throws IOException { QueryBuilder firstFunction = new WrapperQueryBuilder(new TermQueryBuilder("tq", "1").toString()); TermQueryBuilder secondFunction = new TermQueryBuilder("tq", "2"); QueryBuilder queryBuilder = randomBoolean() ? new WrapperQueryBuilder(new TermQueryBuilder("foo", "bar").toString()) : new TermQueryBuilder("foo", "bar"); FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(queryBuilder, new FunctionScoreQueryBuilder.FilterFunctionBuilder[] { new FunctionScoreQueryBuilder.FilterFunctionBuilder(firstFunction, new RandomScoreFunctionBuilder()), new FunctionScoreQueryBuilder.FilterFunctionBuilder(secondFunction, new RandomScoreFunctionBuilder()) }); FunctionScoreQueryBuilder rewrite = (FunctionScoreQueryBuilder) functionScoreQueryBuilder.rewrite(createShardContext()); assertNotSame(functionScoreQueryBuilder, rewrite); assertEquals(rewrite.query(), new TermQueryBuilder("foo", "bar")); assertEquals(rewrite.filterFunctionBuilders()[0].getFilter(), new TermQueryBuilder("tq", "1")); assertSame(rewrite.filterFunctionBuilders()[1].getFilter(), secondFunction); }
public void CopyIndexByQueryDsl(Client sourceclient, String sourceindex, Client targetclient, String targetindex, String DSL) { count = 0; // 单线程scroll,bulk写入 WrapperQueryBuilder wrapper = new WrapperQueryBuilder(DSL); try { SearchResponse scrollResp = sourceclient.prepareSearch(sourceindex) .addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC).setScroll(new TimeValue(60000)) .setQuery(wrapper).setSize(500).execute().actionGet(); while (true) { final BulkRequestBuilder bulkRequest = targetclient.prepareBulk(); for (SearchHit hit : scrollResp.getHits().getHits()) { bulkRequest.add(targetclient.prepareIndex(targetindex, hit.getType().toString(), hit.getId()) .setSource(hit.getSourceAsString())); count++; } bulkRequest.execute(); scrollResp = sourceclient.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(60000)) .execute().actionGet(); logger.info(count + " documents putted!!"); if (scrollResp.getHits().getHits().length == 0) { break; } } logger.info("copy index " + sourceindex + " " + count + " documents" + " to " + targetindex + " complete"); } catch (Exception e) { e.printStackTrace(); } }
public void CopyIndexByQueryDsl(Client sourceclient, String sourceindex, Client targetclient, String targetindex, String DSL) { count = 0; // 单线程scroll,bulk写入 WrapperQueryBuilder wrapper = new WrapperQueryBuilder(DSL); try { SearchResponse scrollResp = sourceclient.prepareSearch(sourceindex) .addSort(SortParseElement.DOC_FIELD_NAME, SortOrder.ASC).setScroll(new TimeValue(60000)) .setQuery(wrapper).setSize(500).execute().actionGet(); while (true) { final BulkRequestBuilder bulkRequest = targetclient.prepareBulk(); for (SearchHit hit : scrollResp.getHits().getHits()) { bulkRequest.add(targetclient.prepareIndex(targetindex, hit.getType().toString(), hit.getId()) .setSource(hit.getSourceAsString())); count++; } bulkRequest.execute(); scrollResp = sourceclient.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(60000)) .execute().actionGet(); logger.info(count + " documents putted!!"); if (scrollResp.getHits().getHits().length == 0) { break; } } logger.info("copy index " + sourceindex + " " + count + " documents" + " to " + targetindex + " complete"); } catch (Exception e) { e.printStackTrace(); } }
public static FeatureStoreLoader nullLoader() { return (storeName, client) -> {throw new IllegalStateException("Invalid state, this query cannot be " + "built without a valid store loader. Your are seeing this exception because you attempt to call " + "doToQuery on a " + StoredLtrQueryBuilder.class.getSimpleName() + " instance that was built with " + "an invalid FeatureStoreLoader. If you are trying to run integration tests with this query consider " + "wrapping it inside a " + WrapperQueryBuilder.class.getSimpleName() + ":\n" + "\tnew WrapperQueryBuilder(sltrBuilder.toString())\n" + "This will force elastic to initialize the feature loader properly");}; }
private void registerQueryParsers(List<SearchPlugin> plugins) { registerQuery(new QuerySpec<>(MatchQueryBuilder.NAME, MatchQueryBuilder::new, MatchQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(MatchPhraseQueryBuilder.NAME, MatchPhraseQueryBuilder::new, MatchPhraseQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(MatchPhrasePrefixQueryBuilder.NAME, MatchPhrasePrefixQueryBuilder::new, MatchPhrasePrefixQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(MultiMatchQueryBuilder.NAME, MultiMatchQueryBuilder::new, MultiMatchQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(NestedQueryBuilder.NAME, NestedQueryBuilder::new, NestedQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(HasChildQueryBuilder.NAME, HasChildQueryBuilder::new, HasChildQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(HasParentQueryBuilder.NAME, HasParentQueryBuilder::new, HasParentQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(DisMaxQueryBuilder.NAME, DisMaxQueryBuilder::new, DisMaxQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(IdsQueryBuilder.NAME, IdsQueryBuilder::new, IdsQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new, MatchAllQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(QueryStringQueryBuilder.NAME, QueryStringQueryBuilder::new, QueryStringQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(BoostingQueryBuilder.NAME, BoostingQueryBuilder::new, BoostingQueryBuilder::fromXContent)); BooleanQuery.setMaxClauseCount(INDICES_MAX_CLAUSE_COUNT_SETTING.get(settings)); registerQuery(new QuerySpec<>(BoolQueryBuilder.NAME, BoolQueryBuilder::new, BoolQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(TermQueryBuilder.NAME, TermQueryBuilder::new, TermQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(TermsQueryBuilder.NAME, TermsQueryBuilder::new, TermsQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(FuzzyQueryBuilder.NAME, FuzzyQueryBuilder::new, FuzzyQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(RegexpQueryBuilder.NAME, RegexpQueryBuilder::new, RegexpQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(RangeQueryBuilder.NAME, RangeQueryBuilder::new, RangeQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(PrefixQueryBuilder.NAME, PrefixQueryBuilder::new, PrefixQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(WildcardQueryBuilder.NAME, WildcardQueryBuilder::new, WildcardQueryBuilder::fromXContent)); registerQuery( new QuerySpec<>(ConstantScoreQueryBuilder.NAME, ConstantScoreQueryBuilder::new, ConstantScoreQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(SpanTermQueryBuilder.NAME, SpanTermQueryBuilder::new, SpanTermQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(SpanNotQueryBuilder.NAME, SpanNotQueryBuilder::new, SpanNotQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(SpanWithinQueryBuilder.NAME, SpanWithinQueryBuilder::new, SpanWithinQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(SpanContainingQueryBuilder.NAME, SpanContainingQueryBuilder::new, SpanContainingQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(FieldMaskingSpanQueryBuilder.NAME, FieldMaskingSpanQueryBuilder::new, FieldMaskingSpanQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(SpanFirstQueryBuilder.NAME, SpanFirstQueryBuilder::new, SpanFirstQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(SpanNearQueryBuilder.NAME, SpanNearQueryBuilder::new, SpanNearQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(SpanOrQueryBuilder.NAME, SpanOrQueryBuilder::new, SpanOrQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(MoreLikeThisQueryBuilder.NAME, MoreLikeThisQueryBuilder::new, MoreLikeThisQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(WrapperQueryBuilder.NAME, WrapperQueryBuilder::new, WrapperQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(CommonTermsQueryBuilder.NAME, CommonTermsQueryBuilder::new, CommonTermsQueryBuilder::fromXContent)); registerQuery( new QuerySpec<>(SpanMultiTermQueryBuilder.NAME, SpanMultiTermQueryBuilder::new, SpanMultiTermQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(FunctionScoreQueryBuilder.NAME, FunctionScoreQueryBuilder::new, FunctionScoreQueryBuilder::fromXContent)); registerQuery( new QuerySpec<>(SimpleQueryStringBuilder.NAME, SimpleQueryStringBuilder::new, SimpleQueryStringBuilder::fromXContent)); registerQuery(new QuerySpec<>(TypeQueryBuilder.NAME, TypeQueryBuilder::new, TypeQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(ScriptQueryBuilder.NAME, ScriptQueryBuilder::new, ScriptQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(GeoDistanceQueryBuilder.NAME, GeoDistanceQueryBuilder::new, GeoDistanceQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(GeoBoundingBoxQueryBuilder.NAME, GeoBoundingBoxQueryBuilder::new, GeoBoundingBoxQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(GeoPolygonQueryBuilder.NAME, GeoPolygonQueryBuilder::new, GeoPolygonQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(ExistsQueryBuilder.NAME, ExistsQueryBuilder::new, ExistsQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(MatchNoneQueryBuilder.NAME, MatchNoneQueryBuilder::new, MatchNoneQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(ParentIdQueryBuilder.NAME, ParentIdQueryBuilder::new, ParentIdQueryBuilder::fromXContent)); if (ShapesAvailability.JTS_AVAILABLE && ShapesAvailability.SPATIAL4J_AVAILABLE) { registerQuery(new QuerySpec<>(GeoShapeQueryBuilder.NAME, GeoShapeQueryBuilder::new, GeoShapeQueryBuilder::fromXContent)); } registerFromPlugin(plugins, SearchPlugin::getQueries, this::registerQuery); }
public void testFullUsecase() throws Exception { addElement(new StoredFeature("feature1", Collections.singletonList("query"), "mustache", QueryBuilders.matchQuery("field1", "{{query}}").toString())); addElement(new StoredFeature("feature2", Collections.singletonList("query"), "mustache", QueryBuilders.matchQuery("field2", "{{query}}").toString())); addElement(new StoredFeature("feature3", Collections.singletonList("query"), "derived_expression", "(feature1 - feature2) > 0 ? 1 : -1")); AddFeaturesToSetRequestBuilder builder = AddFeaturesToSetAction.INSTANCE.newRequestBuilder(client()); builder.request().setFeatureSet("my_set"); builder.request().setFeatureNameQuery("feature1"); builder.request().setStore(IndexFeatureStore.DEFAULT_STORE); builder.execute().get(); builder.request().setFeatureNameQuery("feature2"); builder.execute().get(); builder.request().setFeatureNameQuery("feature3"); long version = builder.get().getResponse().getVersion(); CreateModelFromSetRequestBuilder createModelFromSetRequestBuilder = CreateModelFromSetAction.INSTANCE.newRequestBuilder(client()); createModelFromSetRequestBuilder.withVersion(IndexFeatureStore.DEFAULT_STORE, "my_set", version, "my_model", new StoredLtrModel.LtrModelDefinition("model/linear", SIMPLE_MODEL, true)); createModelFromSetRequestBuilder.get(); buildIndex(); Map<String, Object> params = new HashMap<>(); boolean negativeScore = random().nextBoolean(); params.put("query", negativeScore ? "bonjour" : "hello"); SearchRequestBuilder sb = client().prepareSearch("test_index") .setQuery(QueryBuilders.matchQuery("field1", "world")) .setRescorer(RescoreBuilder .queryRescorer(new WrapperQueryBuilder(new StoredLtrQueryBuilder(LtrTestUtils.nullLoader()) .modelName("my_model").params(params).toString())) .setScoreMode(QueryRescoreMode.Total) .setQueryWeight(0) .setRescoreQueryWeight(1)); SearchResponse sr = sb.get(); assertEquals(1, sr.getHits().getTotalHits()); if(negativeScore) { assertThat(sr.getHits().getAt(0).getScore(), Matchers.lessThan(-10.0f)); } else { assertThat(sr.getHits().getAt(0).getScore(), Matchers.greaterThan(10.0f)); } StoredLtrModel model = getElement(StoredLtrModel.class, StoredLtrModel.TYPE, "my_model"); CachesStatsNodesResponse stats = CachesStatsAction.INSTANCE.newRequestBuilder(client()).execute().get(); assertEquals(1, stats.getAll().getTotal().getCount()); assertEquals(model.compile(parserFactory()).ramBytesUsed(), stats.getAll().getTotal().getRam()); assertEquals(1, stats.getAll().getModels().getCount()); assertEquals(model.compile(parserFactory()).ramBytesUsed(), stats.getAll().getModels().getRam()); assertEquals(0, stats.getAll().getFeatures().getCount()); assertEquals(0, stats.getAll().getFeatures().getRam()); assertEquals(0, stats.getAll().getFeaturesets().getCount()); assertEquals(0, stats.getAll().getFeaturesets().getRam()); ClearCachesAction.RequestBuilder clearCache = ClearCachesAction.INSTANCE.newRequestBuilder(client()); clearCache.request().clearModel(IndexFeatureStore.DEFAULT_STORE, "my_model"); clearCache.get(); stats = CachesStatsAction.INSTANCE.newRequestBuilder(client()).execute().get(); assertEquals(0, stats.getAll().getTotal().getCount()); assertEquals(0, stats.getAll().getTotal().getRam()); }
public void testLog() throws Exception { prepareModels(); Map<String, Doc> docs = buildIndex(); Map<String, Object> params = new HashMap<>(); params.put("query", "found"); List<String> idsColl = new ArrayList<>(docs.keySet()); Collections.shuffle(idsColl, random()); String[] ids = idsColl.subList(0, TestUtil.nextInt(random(), 5, 15)).toArray(new String[0]); StoredLtrQueryBuilder sbuilder = new StoredLtrQueryBuilder(LtrTestUtils.nullLoader()) .featureSetName("my_set") .params(params) .queryName("test") .boost(random().nextInt(3)); StoredLtrQueryBuilder sbuilder_rescore = new StoredLtrQueryBuilder(LtrTestUtils.nullLoader()) .featureSetName("my_set") .params(params) .queryName("test_rescore") .boost(random().nextInt(3)); QueryBuilder query = QueryBuilders.boolQuery().must(new WrapperQueryBuilder(sbuilder.toString())) .filter(QueryBuilders.idsQuery("test").addIds(ids)); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(query) .fetchSource(false) .size(10) .addRescorer(new QueryRescorerBuilder(new WrapperQueryBuilder(sbuilder_rescore.toString()))) .ext(Collections.singletonList( new LoggingSearchExtBuilder() .addQueryLogging("first_log", "test", false) .addRescoreLogging("second_log", 0, true))); SearchResponse resp = client().prepareSearch("test_index").setTypes("test").setSource(sourceBuilder).get(); assertSearchHits(docs, resp); sbuilder.featureSetName(null); sbuilder.modelName("my_model"); sbuilder.boost(random().nextInt(3)); sbuilder_rescore.featureSetName(null); sbuilder_rescore.modelName("my_model"); sbuilder_rescore.boost(random().nextInt(3)); query = QueryBuilders.boolQuery().must(new WrapperQueryBuilder(sbuilder.toString())) .filter(QueryBuilders.idsQuery("test").addIds(ids)); sourceBuilder = new SearchSourceBuilder().query(query) .fetchSource(false) .size(10) .addRescorer(new QueryRescorerBuilder(new WrapperQueryBuilder(sbuilder_rescore.toString()))) .ext(Collections.singletonList( new LoggingSearchExtBuilder() .addQueryLogging("first_log", "test", false) .addRescoreLogging("second_log", 0, true))); SearchResponse resp2 = client().prepareSearch("test_index").setTypes("test").setSource(sourceBuilder).get(); assertSearchHits(docs, resp2); }