private static RangeQueryBuilder createRangeQuery(String name, Map<String, Object> rangeOperation, Float boost) { RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery(name+RAW_APPEND); for (Map.Entry<String, Object> it : rangeOperation.entrySet()) { if (it.getKey().equalsIgnoreCase(LTE)) { rangeQueryBuilder.lte(it.getValue()); } else if (it.getKey().equalsIgnoreCase(LT)) { rangeQueryBuilder.lt(it.getValue()); } else if (it.getKey().equalsIgnoreCase(GTE)) { rangeQueryBuilder.gte(it.getValue()); } else if (it.getKey().equalsIgnoreCase(GT)) { rangeQueryBuilder.gt(it.getValue()); } } if (isNotNull(boost)) { return rangeQueryBuilder.boost(boost); } return rangeQueryBuilder; }
public void testUnsupportedQueries() { RangeQueryBuilder rangeQuery1 = new RangeQueryBuilder("field").from("2016-01-01||/D").to("2017-01-01||/D"); RangeQueryBuilder rangeQuery2 = new RangeQueryBuilder("field").from("2016-01-01||/D").to("now"); PercolatorFieldMapper.verifyQuery(rangeQuery1); expectThrows(IllegalArgumentException.class, () -> PercolatorFieldMapper.verifyQuery(rangeQuery2)); PercolatorFieldMapper.verifyQuery(new BoolQueryBuilder().must(rangeQuery1)); expectThrows(IllegalArgumentException.class, () -> PercolatorFieldMapper.verifyQuery(new BoolQueryBuilder().must(rangeQuery2))); PercolatorFieldMapper.verifyQuery(new ConstantScoreQueryBuilder((rangeQuery1))); expectThrows(IllegalArgumentException.class, () -> PercolatorFieldMapper.verifyQuery(new ConstantScoreQueryBuilder(rangeQuery2))); PercolatorFieldMapper.verifyQuery(new BoostingQueryBuilder(rangeQuery1, new MatchAllQueryBuilder())); expectThrows(IllegalArgumentException.class, () -> PercolatorFieldMapper.verifyQuery(new BoostingQueryBuilder(rangeQuery2, new MatchAllQueryBuilder()))); PercolatorFieldMapper.verifyQuery(new FunctionScoreQueryBuilder(rangeQuery1, new RandomScoreFunctionBuilder())); expectThrows(IllegalArgumentException.class, () -> PercolatorFieldMapper.verifyQuery(new FunctionScoreQueryBuilder(rangeQuery2, new RandomScoreFunctionBuilder()))); HasChildQueryBuilder hasChildQuery = new HasChildQueryBuilder("_type", new MatchAllQueryBuilder(), ScoreMode.None); expectThrows(IllegalArgumentException.class, () -> PercolatorFieldMapper.verifyQuery(hasChildQuery)); expectThrows(IllegalArgumentException.class, () -> PercolatorFieldMapper.verifyQuery(new BoolQueryBuilder().must(hasChildQuery))); HasParentQueryBuilder hasParentQuery = new HasParentQueryBuilder("_type", new MatchAllQueryBuilder(), false); expectThrows(IllegalArgumentException.class, () -> PercolatorFieldMapper.verifyQuery(hasParentQuery)); expectThrows(IllegalArgumentException.class, () -> PercolatorFieldMapper.verifyQuery(new BoolQueryBuilder().must(hasParentQuery))); }
/** * @param operator operator <,>,<=,>= * @param field field name * @param stringValue field value * @return a range query */ static QueryBuilder range(String operator, String field, String stringValue) { String key = StringUtils.replaceAll(field, "[<>=\\s]+$", ""); boolean nestedMode = nestedMode() && field.startsWith(PROPS_PREFIX); RangeQueryBuilder rfb = rangeQuery(nestedMode ? getValueFieldName(stringValue) : key); if (">".equals(operator)) { rfb.gt(getNumericValue(stringValue)); } else if ("<".equals(operator)) { rfb.lt(getNumericValue(stringValue)); } else if (">=".equals(operator)) { rfb.gte(getNumericValue(stringValue)); } else if ("<=".equals(operator)) { rfb.lte(getNumericValue(stringValue)); } if (nestedMode) { return nestedPropsQuery(keyValueBoolQuery(key, stringValue, rfb)); } else { return rfb; } }
private static QueryBuilder termRange(Query q) { QueryBuilder qb = null; TermRangeQuery trq = (TermRangeQuery) q; if (!StringUtils.isBlank(trq.getField())) { String from = trq.getLowerTerm() != null ? Term.toString(trq.getLowerTerm()) : "*"; String to = trq.getUpperTerm() != null ? Term.toString(trq.getUpperTerm()) : "*"; boolean nestedMode = nestedMode() && trq.getField().matches(PROPS_REGEX); qb = rangeQuery(nestedMode ? getValueFieldNameFromRange(from, to) : trq.getField()); if ("*".equals(from) && "*".equals(to)) { qb = matchAllQuery(); } if (!"*".equals(from)) { ((RangeQueryBuilder) qb).from(getNumericValue(from)).includeLower(trq.includesLower()); } if (!"*".equals(to)) { ((RangeQueryBuilder) qb).to(getNumericValue(to)).includeUpper(trq.includesUpper()); } if (nestedMode) { qb = nestedPropsQuery(keyValueBoolQuery(trq.getField(), qb)); } } return qb; }
@SuppressWarnings("unchecked") @Before public void setUp() throws Exception { doReturn(classValue).when(visitor).doGetPrimitiveFieldClass(any(PrimitiveStatement.class)); doReturn(wildcardQueryBuilder).when(visitor).createWildcardQuery(anyString(), anyString()); doReturn(boolQueryBuilder).when(visitor).createBoolQueryBuilder(); doReturn(termQueryBuilder).when(visitor).createTermQuery(anyString(), any()); doReturn(rangeQueryBuilder).when(visitor).createRangeQuery(anyString()); doReturn(boolQueryBuilder).when(boolQueryBuilder).mustNot(any(QueryBuilder.class)); doReturn(rangeQueryBuilder).when(rangeQueryBuilder).lt(any(RangeQueryBuilder.class)); doReturn(rangeQueryBuilder).when(rangeQueryBuilder).lte(any(RangeQueryBuilder.class)); doReturn(rangeQueryBuilder).when(rangeQueryBuilder).gt(any(RangeQueryBuilder.class)); doReturn(rangeQueryBuilder).when(rangeQueryBuilder).gte(any(RangeQueryBuilder.class)); // We'll test our enum-safe getter separately. doAnswer(answer -> answer.getArgumentAt(0, TestSearchConditionVisitor.ClassValue.class).getValue()) .when(visitor).getEnumSafeValue(classValue); }
/** * Tests {@link ElasticsearchQueryBuilderVisitor#buildSimpleExpression(PrimitiveStatement)} for the case where * we're building a {@link ConditionType#LESS_THAN} expression. */ @SuppressWarnings("unchecked") @Test public void testBuildSimpleLessThanExpression() throws Exception { final Object value = "hello"; final String property = "property"; doReturn(value).when(classValue).getValue(); doReturn(LESS_THAN).when(statement).getCondition(); doReturn(property).when(statement).getProperty(); assertThat(visitor.buildSimpleExpression(statement), instanceOf(RangeQueryBuilder.class)); verify(visitor).buildSimpleExpression(statement); verify(visitor).doGetPrimitiveFieldClass(statement); verify(visitor).validateNotCollectionCheck(statement, classValue); verify(visitor).createRangeQuery(property); verify(visitor).getEnumSafeValue(classValue); verify(statement).getProperty(); verify(statement).getCondition(); verify(rangeQueryBuilder).lt(value); verifyNoMoreCollaboration(); }
/** * Tests {@link ElasticsearchQueryBuilderVisitor#buildSimpleExpression(PrimitiveStatement)} for the case where * we're building a {@link ConditionType#LESS_OR_EQUALS} expression. */ @SuppressWarnings("unchecked") @Test public void testBuildSimpleLessThanOrEqualsExpression() throws Exception { final Object value = "hello"; final String property = "property"; doReturn(value).when(classValue).getValue(); doReturn(LESS_OR_EQUALS).when(statement).getCondition(); doReturn(property).when(statement).getProperty(); assertThat(visitor.buildSimpleExpression(statement), instanceOf(RangeQueryBuilder.class)); verify(visitor).buildSimpleExpression(statement); verify(visitor).doGetPrimitiveFieldClass(statement); verify(visitor).validateNotCollectionCheck(statement, classValue); verify(visitor).createRangeQuery(property); verify(visitor).getEnumSafeValue(classValue); verify(statement).getProperty(); verify(statement).getCondition(); verify(rangeQueryBuilder).lte(value); verifyNoMoreCollaboration(); }
/** * Tests {@link ElasticsearchQueryBuilderVisitor#buildSimpleExpression(PrimitiveStatement)} for the case where * we're building a {@link ConditionType#GREATER_THAN} expression. */ @SuppressWarnings("unchecked") @Test public void testBuildSimpleGreaterThanExpression() throws Exception { final Object value = "hello"; final String property = "property"; doReturn(value).when(classValue).getValue(); doReturn(GREATER_THAN).when(statement).getCondition(); doReturn(property).when(statement).getProperty(); assertThat(visitor.buildSimpleExpression(statement), instanceOf(RangeQueryBuilder.class)); verify(visitor).buildSimpleExpression(statement); verify(visitor).doGetPrimitiveFieldClass(statement); verify(visitor).validateNotCollectionCheck(statement, classValue); verify(visitor).createRangeQuery(property); verify(visitor).getEnumSafeValue(classValue); verify(statement).getProperty(); verify(statement).getCondition(); verify(rangeQueryBuilder).gt(value); verifyNoMoreCollaboration(); }
/** * Tests {@link ElasticsearchQueryBuilderVisitor#buildSimpleExpression(PrimitiveStatement)} for the case where * we're building a {@link ConditionType#GREATER_OR_EQUALS} expression. */ @SuppressWarnings("unchecked") @Test public void testBuildSimpleGreaterThanOrEqualsExpression() throws Exception { final Object value = "hello"; final String property = "property"; doReturn(value).when(classValue).getValue(); doReturn(GREATER_OR_EQUALS).when(statement).getCondition(); doReturn(property).when(statement).getProperty(); assertThat(visitor.buildSimpleExpression(statement), instanceOf(RangeQueryBuilder.class)); verify(visitor).buildSimpleExpression(statement); verify(visitor).doGetPrimitiveFieldClass(statement); verify(visitor).validateNotCollectionCheck(statement, classValue); verify(visitor).createRangeQuery(property); verify(visitor).getEnumSafeValue(classValue); verify(statement).getProperty(); verify(statement).getCondition(); verify(rangeQueryBuilder).gte(value); verifyNoMoreCollaboration(); }
/** * Tests {@link ElasticsearchQueryBuilderVisitor#mergeQueryParts(QueryBuilder, QueryBuilder)} for the case where our * current query part is a less than/less than or equal to. */ @Test public void testMergeQueryPartsForLessThan() throws Exception { final RangeQueryBuilder previousPart = mock(RangeQueryBuilder.class); final RangeQueryBuilder currentPart = mock(RangeQueryBuilder.class); final String toValue = "4000"; final boolean includeUpper = true; doReturn(toValue).when(currentPart).to(); doReturn(includeUpper).when(currentPart).includeUpper(); visitor.mergeQueryParts(previousPart, currentPart); verify(visitor).mergeQueryParts(previousPart, currentPart); verify(previousPart).to(toValue); verify(previousPart).includeUpper(includeUpper); verify(currentPart, times(2)).to(); verify(currentPart).includeUpper(); verifyNoMoreCollaboration(previousPart, currentPart); }
/** * Tests {@link ElasticsearchQueryBuilderVisitor#mergeQueryParts(QueryBuilder, QueryBuilder)} for the case where our * current query part is a greater than/greater than or equal to. */ @Test public void testMergeQueryPartsForGreaterThan() throws Exception { final RangeQueryBuilder previousPart = mock(RangeQueryBuilder.class); final RangeQueryBuilder currentPart = mock(RangeQueryBuilder.class); final String fromValue = "4000"; final boolean includeLower = true; doReturn(fromValue).when(currentPart).from(); doReturn(includeLower).when(currentPart).includeLower(); visitor.mergeQueryParts(previousPart, currentPart); verify(visitor).mergeQueryParts(previousPart, currentPart); verify(previousPart).from(fromValue); verify(previousPart).includeLower(includeLower); verify(currentPart).to(); verify(currentPart).from(); verify(currentPart).includeLower(); verifyNoMoreCollaboration(previousPart, currentPart); }
@Test public void assertMultipleMetricAggregationWithScriptError() { String indexName = "test1"; int size = 1; Map<String, Integer> termsFactor = new HashMap<String, Integer>(); termsFactor.put("foo", 1); buildTestDataset(1, indexName, "type1", size, termsFactor); SearchResponse searchResponse = client().prepareSearch(indexName) .setQuery(matchAllQuery()) .addAggregation(new MultipleMetricBuilder("metrics") .script(new ScriptBuilder("ratio").script(new Script("value1 / value2"))) .field(new SumBuilder("value1").field("value1")) .field(new CountBuilder("value2").field("value2").filter(new RangeQueryBuilder("value1").gt(1000)))) .execute().actionGet(); MultipleMetric metrics = searchResponse.getAggregations().get("metrics"); assertEquals(metrics.getValue("value1"), 45.0 * size, 0.0); assertEquals(metrics.getValue("value2"), 0.0 * size, 0.0); assertEquals(metrics.getValue("ratio"), Double.POSITIVE_INFINITY, 0.0); assertEquals(metrics.getDocCount("value1"), 10); assertEquals(metrics.getDocCount("value2"), 0); }
@Test public void assertMultipleMetricAggregationWithFilter() { String indexName = "test2"; int size = 1; Map<String, Integer> termsFactor = new HashMap<String, Integer>(); termsFactor.put("foo", 1); buildTestDataset(1, indexName, "type1", size, termsFactor); SearchResponse searchResponse = client().prepareSearch(indexName) .setQuery(matchAllQuery()) .addAggregation(new MultipleMetricBuilder("metrics") .script(new ScriptBuilder("ratio").script(new Script("value1 / value2"))) .field(new SumBuilder("value1").field("value1").filter(new RangeQueryBuilder("value1").gt(5))) .field(new CountBuilder("value2").field("value2"))) .execute().actionGet(); MultipleMetric metrics = searchResponse.getAggregations().get("metrics"); assertEquals(metrics.getValue("value1"), 30.0 * size, 0.0); assertEquals(metrics.getValue("value2"), 10.0 * size, 0.0); assertEquals(metrics.getValue("ratio"), metrics.getValue("value1") / metrics.getValue("value2"), 0.0); assertEquals(metrics.getDocCount("value1"), 4); assertEquals(metrics.getDocCount("value2"), 10); }
@Test public void assertMultipleMetricAggregationWithUnmappedField() { String indexName = "test3"; int size = 1; Map<String, Integer> termsFactor = new HashMap<String, Integer>(); termsFactor.put("foo", 1); buildTestDataset(1, indexName, "type1", size, termsFactor); SearchResponse searchResponse = client().prepareSearch(indexName) .setQuery(matchAllQuery()) .addAggregation(new MultipleMetricBuilder("metrics") .script(new ScriptBuilder("ratio").script(new Script("value1 + value2"))) .field(new SumBuilder("value1").field("value4").filter(new RangeQueryBuilder("value1").gt(5))) .field(new CountBuilder("value2").field("value5"))) .execute().actionGet(); MultipleMetric metrics = searchResponse.getAggregations().get("metrics"); assertEquals(metrics.getValue("value1"), 0.0, 0.0); assertEquals(metrics.getValue("value2"), 0.0, 0.0); assertEquals(metrics.getValue("ratio"), metrics.getValue("value1") + metrics.getValue("value2"), 0.0); assertEquals(metrics.getDocCount("value1"), 0); assertEquals(metrics.getDocCount("value2"), 0); }
@Override protected SearchRequestBuilder adaptRequestBuilder(final Client esClient, final SearchRequestBuilder requestBuilder) { QueryBuilder filter = null; if (from != null || to != null) { final RangeQueryBuilder occRange = QueryBuilders.rangeQuery(Event.FIELD_TIMESTAMP); if (from != null) { occRange.gte(from.getTime()); } if (to != null) { occRange.lte(to.getTime()); } filter = occRange; } if (filter != null) { requestBuilder.setQuery(filter); } return requestBuilder; }
public Map<String, List<EborpSample>> getAll(String timeRange) { //TODO datumrange toevoegen SearchRequestBuilder search = client.prepareSearch(INDEX_NAME); RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery("epoch"); rangeQuery.from("now-" + timeRange); SearchRequestBuilder srb = search.setQuery(rangeQuery); srb.setSize(1000000); final SearchResponse response = srb.execute().actionGet(); // final Map<String, List<Triplet<Integer, Instant, String>>> results = new HashMap<>(); // System.out.println("response.getHits().hits() = " + response.getHits().hits().length); return Arrays.asList(response.getHits().hits()).stream() .map(SearchHit::getSource) .map(this::mapToEborpSample) .collect(Collectors.groupingBy(EborpSample::getMac)); }
private static QueryBuilder randomRangeQuery(List<String> fields, int numDocs) { QueryBuilder q = QueryBuilders.rangeQuery(randomField(fields)); if (randomBoolean()) { ((RangeQueryBuilder)q).from(randomIntBetween(0, numDocs / 2 - 1)); } if (randomBoolean()) { ((RangeQueryBuilder)q).to(randomIntBetween(numDocs / 2, numDocs)); } return q; }
/** * Provides a convenience method to merge two query parts together. Assuming that the value of * {@link #canMergeQueryParts(QueryBuilder, QueryBuilder, ConditionType)} is {@code true}, this method will attempt * to take the values out of the current query part and put them into the appropriate fields in the previous query part. * * @param previousPart The previous query part to merge into. * @param part The current query part to merge into the previous. * @see #canMergeQueryParts(QueryBuilder, QueryBuilder, ConditionType) */ void mergeQueryParts(final QueryBuilder previousPart, final QueryBuilder part) { final RangeQueryBuilder previousRangeQuery = (RangeQueryBuilder) previousPart; final RangeQueryBuilder currentRangeQuery = (RangeQueryBuilder) part; // If the "from" field is filled in, we know we're greater than/greater than or equal to. if (null == currentRangeQuery.to()) { previousRangeQuery.from(currentRangeQuery.from()); previousRangeQuery.includeLower(currentRangeQuery.includeLower()); } else { // Otherwise we're less than/less than or equal to. previousRangeQuery.to(currentRangeQuery.to()); previousRangeQuery.includeUpper(currentRangeQuery.includeUpper()); } }
/** * Tests {@link ElasticsearchQueryBuilderVisitor#canMergeQueryParts(QueryBuilder, QueryBuilder, ConditionType)} for * the happy path. */ @Test public void testCanMergeQueryParts() throws Exception { final RangeQueryBuilder previousPart = mock(RangeQueryBuilder.class); final RangeQueryBuilder currentPart = mock(RangeQueryBuilder.class); doReturn("taters").when(previousPart).fieldName(); doReturn("taters").when(currentPart).fieldName(); assertThat(visitor.canMergeQueryParts(previousPart, currentPart, ConditionType.AND), is(true)); }
/** * Tests {@link ElasticsearchQueryBuilderVisitor#canMergeQueryParts(QueryBuilder, QueryBuilder, ConditionType)} for * the case where something other than {@link ConditionType#AND} was passed in. This will usually be something like * {@link ConditionType#OR}. */ @Test public void testCanMergeQueryPartsForNonAndCondition() throws Exception { Stream.of(ConditionType.values()) .filter(type -> !type.equals(AND)) .forEach(type -> { final RangeQueryBuilder previousPart = mock(RangeQueryBuilder.class); final RangeQueryBuilder currentPart = mock(RangeQueryBuilder.class); doReturn("taters").when(previousPart).fieldName(); doReturn("taters").when(currentPart).fieldName(); assertThat(visitor.canMergeQueryParts(previousPart, currentPart, type), is(false)); }); }
/** * Tests {@link ElasticsearchQueryBuilderVisitor#canMergeQueryParts(QueryBuilder, QueryBuilder, ConditionType)} for * the case where the previous query part is not a {@link RangeQueryBuilder}. You might see this with something like * an expression like {@code foo==3;foo=le=400}. */ @Test public void testCanMergeQueryPartsForNonRangePreviousQueryPart() throws Exception { final TermQueryBuilder previousPart = mock(TermQueryBuilder.class); final RangeQueryBuilder currentPart = mock(RangeQueryBuilder.class); doReturn("taters").when(previousPart).fieldName(); doReturn("taters").when(currentPart).fieldName(); assertThat(visitor.canMergeQueryParts(previousPart, currentPart, ConditionType.AND), is(false)); }
/** * Tests {@link ElasticsearchQueryBuilderVisitor#canMergeQueryParts(QueryBuilder, QueryBuilder, ConditionType)} for * the case where the current query part is not a {@link RangeQueryBuilder}. You might see this with something like * an expression like {@code foo=le=300;foo==4}. */ @Test public void testCanMergeQueryPartsForNonRangeCurrentQueryPart() throws Exception { final RangeQueryBuilder previousPart = mock(RangeQueryBuilder.class); final WildcardQueryBuilder currentPart = mock(WildcardQueryBuilder.class); doReturn("taters").when(previousPart).fieldName(); doReturn("taters").when(currentPart).fieldName(); assertThat(visitor.canMergeQueryParts(previousPart, currentPart, ConditionType.AND), is(false)); }
/** * Tests {@link ElasticsearchQueryBuilderVisitor#canMergeQueryParts(QueryBuilder, QueryBuilder, ConditionType)} for * the case where both query parts are lining up, except they are pointing at different fields. This would be * something like {@code foo=le=300;bar=ge=400}. */ @Test public void testCanMergeQueryPartsForDifferentFields() throws Exception { final RangeQueryBuilder previousPart = mock(RangeQueryBuilder.class); final RangeQueryBuilder currentPart = mock(RangeQueryBuilder.class); doReturn("taters").when(previousPart).fieldName(); doReturn("definitely_not_taters").when(currentPart).fieldName(); assertThat(visitor.canMergeQueryParts(previousPart, currentPart, ConditionType.AND), is(false)); }
@Override public Object query(Person person) { List<Map<String, Object>> result = new ArrayList<>(); try { BoolQueryBuilder boolBuilder = QueryBuilders.boolQuery(); if (person.getName() != null) { // boolBuilder.must(QueryBuilders.matchQuery("name", person.getName())); boolBuilder.should(QueryBuilders.matchQuery("name", person.getName())); } if (person.getIntroduce() != null) { // boolBuilder.must(QueryBuilders.matchQuery("introduce", person.getIntroduce())); boolBuilder.should(QueryBuilders.matchQuery("introduce", person.getIntroduce())); } //大于age,小于age+10 if (person.getAge() > 0) { RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery("age"); rangeQuery.from(person.getAge()); rangeQuery.to(person.getAge() + 10); boolBuilder.filter(rangeQuery); } SearchRequestBuilder builder = transportClient.prepareSearch(index) .setTypes(type) .setSearchType(SearchType.QUERY_THEN_FETCH) .setQuery(boolBuilder) .setFrom(0) .setSize(10); log.info(String.valueOf(builder)); SearchResponse response = builder.get(); response.getHits().forEach((s) -> result.add(s.getSource())); } catch (Exception e) { e.printStackTrace(); log.error(e.getMessage(), e); } return result; }
@Override public Observable<PersistentMasterKey> call(Void aVoid) { final Elasticsearch elasticSearch = vertxContext.verticle().elasticsearch(); RangeQueryBuilder thresholdFilter = rangeQuery("re_encrypt_ts") .lte(toDateTimeString(threshold)); BoolQueryBuilder query = boolQuery().must(thresholdFilter); SearchRequestBuilder request = elasticSearch.get() .prepareSearch( elasticSearch.masterKeyTypeIndex()) .setVersion(true) .setTypes(elasticSearch.defaultType()) .setQuery(query) .setSize(5000) .setTimeout(timeValueMillis(elasticSearch.getDefaultSearchTimeout() - 10)); if (LOGGER.isDebugEnabled()) { LOGGER.debug(format("Search Request {%s,%s} = %s", elasticSearch.defaultType(), elasticSearch.masterKeyTypeIndex(), Jsonify.toString(request))); } return elasticSearch.execute(vertxContext, request, elasticSearch.getDefaultSearchTimeout()) .flatMap(oSearchResponse -> { SearchResponse searchResponse = oSearchResponse.get(); if (LOGGER.isDebugEnabled()) { LOGGER.debug(format("Search Response {%s,%s} = %s", elasticSearch.defaultType(), elasticSearch.masterKeyTypeIndex(), Jsonify.toString(searchResponse))); } if (oSearchResponse.isPresent()) { return from(oSearchResponse.get().getHits()); } else { return from(emptyList()); } }) .map(PersistentMasterKey::fromSearchHit); }
protected Observable<Void> execute0(VertxContext<Server> vertxContext, MultiMap parameters) { final Elasticsearch elasticSearch = vertxContext.verticle().elasticsearch(); return Defer.aVoid() .flatMap(new ListSfsObjectIndexes(vertxContext)) .flatMap(index -> { String unparsedForceRemoveVolumes = JobParams.getFirstOptionalParam(parameters, Jobs.Parameters.FORCE_REMOVE_VOLUMES); Set<String> forceRemoveVolumes = Strings.isNullOrEmpty(unparsedForceRemoveVolumes) ? Collections.emptySet() : Sets.newHashSet(Splitter.on(',').omitEmptyStrings().trimResults().split(unparsedForceRemoveVolumes)); long now = System.currentTimeMillis() - CONSISTENCY_THRESHOLD; Calendar consistencyThreshold = Calendar.getInstance(); consistencyThreshold.setTimeInMillis(now); RangeQueryBuilder query = rangeQuery("update_ts").lte(toDateTimeString(consistencyThreshold)); ScanAndScrollStreamProducer producer = new ScanAndScrollStreamProducer(vertxContext, query) .setIndeces(index) .setTypes(elasticSearch.defaultType()) .setReturnVersion(true); producerList.add(producer); if (aborted) { producer.abort(); } SearchHitMaintainObjectEndableWrite consumer = new SearchHitMaintainObjectEndableWrite(vertxContext, forceRemoveVolumes); LOGGER.info("Starting maintain on index " + index); return AsyncIO.pump(producer, consumer) .doOnNext(aVoid1 -> LOGGER.info("Finished maintain on index " + index)); }) .count() .map(new ToVoid<>()); }
/** * 范围查询 * * @param boolQ * @param conditions * @param conditionType */ private void doRange(BoolQueryBuilder boolQ, List<Condition> conditions, ConditionType conditionType) { String[] ranges; RangeQueryBuilder range; for (Condition condition : conditions) { ranges = EsRange.rangeAdapter(condition.getValue()); range = QueryBuilders.rangeQuery(condition.getFiled()).from(ranges[0]).to(ranges[1]); mergeBuilder(boolQ, range, conditionType); } }
public QueryBuilder rangeQuery(String name) { QueryBuilder query = null; RangeQueryBuilder rquery = QueryBuilders.rangeQuery(name); min.ifPresent(m -> rquery.from(m)); max.ifPresent(m -> rquery.to(m)); query = rquery; return query; }
public SimpleBooleanQueryBuilder<PB> rangeBuilder(String name, Function<RangeQueryBuilder, RangeQueryBuilder> func){ RangeQueryBuilder range = new RangeQueryBuilder(name); range = func.apply(range); if(range!=null){ must(range); } return this; }
@Test public void shouldCreateDoubleBoundedFilter() throws Exception { //given BoundedFilterFactory factory = new BoundedFilterFactory(); RangeSegment anyRangeSegment = new RangeSegment(1.0, 2.0); //when QueryBuilder filter = factory.createBoundedFilter("fieldName", anyRangeSegment); //then assertThat(filter).isInstanceOf(RangeQueryBuilder.class); }
private static BoolQueryBuilder createFulltextSearchQueryBetween(final List<String> splitToList, final LocalDateTime from, final LocalDateTime to) { BoolQueryBuilder searchQuery = null; if (splitToList.isEmpty()) { searchQuery = QueryBuilders.boolQuery().must(QueryBuilders.matchAllQuery()); } else { searchQuery = createFulltextSearchQueryBuilder(splitToList); } final RangeQueryBuilder rqb = QueryBuilders.rangeQuery("datePublished").from(convertToDate(from)).to(convertToDate(to)); searchQuery.must(rqb); return searchQuery; }
@Override public QueryBuilder convert(Filter filter) { String field = filter.getString("field"); if (field == null) { field = filter.getName(); } RangeQueryBuilder result = rangeQuery(field); String gte = filter.getString("gte"); String gt = filter.getString("gt"); String lt = filter.getString("lt"); String lte = filter.getString("lte"); String time_zone = filter.getString("time_zone"); if (gte != null) { result.gte(gte); } if (gt != null) { result.gt(gt); } if (lt != null) { result.lt(lt); } if (lte != null) { result.lte(lte); } if (time_zone != null) { result.timeZone(time_zone); } return result; }
@Override public void addRangeTerm(String field, String startValue, String endValue) { RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery(field); if (!startValue.equals("*")) { rangeQueryBuilder = rangeQueryBuilder.from(startValue); } if (!endValue.equals("*")) { rangeQueryBuilder = rangeQueryBuilder.to(endValue); } _boolQueryBuilder = _boolQueryBuilder.should(rangeQueryBuilder); }
private QueryBuilder buildSingleRangeQuery(String key, String value) { String[] values = value.split(" - "); if (value.length() == 0) { return null; } RangeQueryBuilder queryBuilder = QueryBuilders.rangeQuery(key); if (value.length() == 2) { queryBuilder.from(Double.valueOf(values[0]).doubleValue()).to(Double.valueOf(values[1])); } else if (value.startsWith(values[0])) { queryBuilder.gte(Double.valueOf(values[0]).doubleValue()); } else { queryBuilder.lt(Double.valueOf(values[0]).doubleValue()); } return queryBuilder; }
private void registerQueryParsers(List<SearchPlugin> plugins) { registerQuery(new QuerySpec<>(MatchQueryBuilder.NAME, MatchQueryBuilder::new, MatchQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(MatchPhraseQueryBuilder.NAME, MatchPhraseQueryBuilder::new, MatchPhraseQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(MatchPhrasePrefixQueryBuilder.NAME, MatchPhrasePrefixQueryBuilder::new, MatchPhrasePrefixQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(MultiMatchQueryBuilder.NAME, MultiMatchQueryBuilder::new, MultiMatchQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(NestedQueryBuilder.NAME, NestedQueryBuilder::new, NestedQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(HasChildQueryBuilder.NAME, HasChildQueryBuilder::new, HasChildQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(HasParentQueryBuilder.NAME, HasParentQueryBuilder::new, HasParentQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(DisMaxQueryBuilder.NAME, DisMaxQueryBuilder::new, DisMaxQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(IdsQueryBuilder.NAME, IdsQueryBuilder::new, IdsQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new, MatchAllQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(QueryStringQueryBuilder.NAME, QueryStringQueryBuilder::new, QueryStringQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(BoostingQueryBuilder.NAME, BoostingQueryBuilder::new, BoostingQueryBuilder::fromXContent)); BooleanQuery.setMaxClauseCount(INDICES_MAX_CLAUSE_COUNT_SETTING.get(settings)); registerQuery(new QuerySpec<>(BoolQueryBuilder.NAME, BoolQueryBuilder::new, BoolQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(TermQueryBuilder.NAME, TermQueryBuilder::new, TermQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(TermsQueryBuilder.NAME, TermsQueryBuilder::new, TermsQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(FuzzyQueryBuilder.NAME, FuzzyQueryBuilder::new, FuzzyQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(RegexpQueryBuilder.NAME, RegexpQueryBuilder::new, RegexpQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(RangeQueryBuilder.NAME, RangeQueryBuilder::new, RangeQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(PrefixQueryBuilder.NAME, PrefixQueryBuilder::new, PrefixQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(WildcardQueryBuilder.NAME, WildcardQueryBuilder::new, WildcardQueryBuilder::fromXContent)); registerQuery( new QuerySpec<>(ConstantScoreQueryBuilder.NAME, ConstantScoreQueryBuilder::new, ConstantScoreQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(SpanTermQueryBuilder.NAME, SpanTermQueryBuilder::new, SpanTermQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(SpanNotQueryBuilder.NAME, SpanNotQueryBuilder::new, SpanNotQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(SpanWithinQueryBuilder.NAME, SpanWithinQueryBuilder::new, SpanWithinQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(SpanContainingQueryBuilder.NAME, SpanContainingQueryBuilder::new, SpanContainingQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(FieldMaskingSpanQueryBuilder.NAME, FieldMaskingSpanQueryBuilder::new, FieldMaskingSpanQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(SpanFirstQueryBuilder.NAME, SpanFirstQueryBuilder::new, SpanFirstQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(SpanNearQueryBuilder.NAME, SpanNearQueryBuilder::new, SpanNearQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(SpanOrQueryBuilder.NAME, SpanOrQueryBuilder::new, SpanOrQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(MoreLikeThisQueryBuilder.NAME, MoreLikeThisQueryBuilder::new, MoreLikeThisQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(WrapperQueryBuilder.NAME, WrapperQueryBuilder::new, WrapperQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(CommonTermsQueryBuilder.NAME, CommonTermsQueryBuilder::new, CommonTermsQueryBuilder::fromXContent)); registerQuery( new QuerySpec<>(SpanMultiTermQueryBuilder.NAME, SpanMultiTermQueryBuilder::new, SpanMultiTermQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(FunctionScoreQueryBuilder.NAME, FunctionScoreQueryBuilder::new, FunctionScoreQueryBuilder::fromXContent)); registerQuery( new QuerySpec<>(SimpleQueryStringBuilder.NAME, SimpleQueryStringBuilder::new, SimpleQueryStringBuilder::fromXContent)); registerQuery(new QuerySpec<>(TypeQueryBuilder.NAME, TypeQueryBuilder::new, TypeQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(ScriptQueryBuilder.NAME, ScriptQueryBuilder::new, ScriptQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(GeoDistanceQueryBuilder.NAME, GeoDistanceQueryBuilder::new, GeoDistanceQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(GeoBoundingBoxQueryBuilder.NAME, GeoBoundingBoxQueryBuilder::new, GeoBoundingBoxQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(GeoPolygonQueryBuilder.NAME, GeoPolygonQueryBuilder::new, GeoPolygonQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(ExistsQueryBuilder.NAME, ExistsQueryBuilder::new, ExistsQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(MatchNoneQueryBuilder.NAME, MatchNoneQueryBuilder::new, MatchNoneQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(ParentIdQueryBuilder.NAME, ParentIdQueryBuilder::new, ParentIdQueryBuilder::fromXContent)); if (ShapesAvailability.JTS_AVAILABLE && ShapesAvailability.SPATIAL4J_AVAILABLE) { registerQuery(new QuerySpec<>(GeoShapeQueryBuilder.NAME, GeoShapeQueryBuilder::new, GeoShapeQueryBuilder::fromXContent)); } registerFromPlugin(plugins, SearchPlugin::getQueries, this::registerQuery); }
public void testCreateAndDeleteIndexConcurrently() throws InterruptedException { createIndex("test"); final AtomicInteger indexVersion = new AtomicInteger(0); final Object indexVersionLock = new Object(); final CountDownLatch latch = new CountDownLatch(1); int numDocs = randomIntBetween(1, 10); for (int i = 0; i < numDocs; i++) { client().prepareIndex("test", "test").setSource("index_version", indexVersion.get()).get(); } synchronized (indexVersionLock) { // not necessarily needed here but for completeness we lock here too indexVersion.incrementAndGet(); } client().admin().indices().prepareDelete("test").execute(new ActionListener<DeleteIndexResponse>() { // this happens async!!! @Override public void onResponse(DeleteIndexResponse deleteIndexResponse) { Thread thread = new Thread() { @Override public void run() { try { // recreate that index client().prepareIndex("test", "test").setSource("index_version", indexVersion.get()).get(); synchronized (indexVersionLock) { // we sync here since we have to ensure that all indexing operations below for a given ID are done before // we increment the index version otherwise a doc that is in-flight could make it into an index that it // was supposed to be deleted for and our assertion fail... indexVersion.incrementAndGet(); } // from here on all docs with index_version == 0|1 must be gone!!!! only 2 are ok; assertAcked(client().admin().indices().prepareDelete("test").get()); } finally { latch.countDown(); } } }; thread.start(); } @Override public void onFailure(Exception e) { throw new RuntimeException(e); } } ); numDocs = randomIntBetween(100, 200); for (int i = 0; i < numDocs; i++) { try { synchronized (indexVersionLock) { client().prepareIndex("test", "test").setSource("index_version", indexVersion.get()) .setTimeout(TimeValue.timeValueSeconds(10)).get(); } } catch (IndexNotFoundException inf) { // fine } catch (UnavailableShardsException ex) { assertEquals(ex.getCause().getClass(), IndexNotFoundException.class); // fine we run into a delete index while retrying } } latch.await(); refresh(); // we only really assert that we never reuse segments of old indices or anything like this here and that nothing fails with // crazy exceptions SearchResponse expected = client().prepareSearch("test").setIndicesOptions(IndicesOptions.lenientExpandOpen()) .setQuery(new RangeQueryBuilder("index_version").from(indexVersion.get(), true)).get(); SearchResponse all = client().prepareSearch("test").setIndicesOptions(IndicesOptions.lenientExpandOpen()).get(); assertEquals(expected + " vs. " + all, expected.getHits().getTotalHits(), all.getHits().getTotalHits()); logger.info("total: {}", expected.getHits().getTotalHits()); }
/** * Provides a convenience method to determine whether or not two query parts can be merged into a single part. This * is possible IFF: * <p> * <pre> * * Neither part is null * * The {@link ConditionType} being applied to both query parts is {@link ConditionType#AND} * * Both query parts are {@link RangeQueryBuilder} instances (EG: no term or wildcard queries) * * Both query parts refer to the same field in Elasticsearch * * Both query parts are within the same expression (IE: within the same level of parenthesis). * </pre> * * @param previousPart The previous query part to check the merging ability of. * @param currentPart The current query part to check the merging ability of. * @param conditionType The {@link ConditionType} being applied to both query parts. * @return {@code True} if the current query part can be merged into the previous query part, else {@code false}. * @see #mergeQueryParts(QueryBuilder, QueryBuilder) */ boolean canMergeQueryParts(final QueryBuilder previousPart, final QueryBuilder currentPart, final ConditionType conditionType) { // We can only merge together AND'ed range queries. if (ConditionType.AND.equals(conditionType) && RangeQueryBuilder.class.isAssignableFrom(previousPart.getClass()) && RangeQueryBuilder.class.isAssignableFrom(currentPart.getClass())) { final RangeQueryBuilder previousRangeQuery = (RangeQueryBuilder) previousPart; final RangeQueryBuilder currentRangeQuery = (RangeQueryBuilder) currentPart; // It also helps if both queries apply to the same field. if (previousRangeQuery.fieldName().equals(currentRangeQuery.fieldName())) { return true; } } return false; }
@Override public ListenableFuture<List<List<Span>>> getTraces(QueryRequest request) { long endMillis = request.endTs; long beginMillis = endMillis - request.lookback; BoolQueryBuilder filter = boolQuery() .must(rangeQuery("timestamp_millis") .gte(beginMillis) .lte(endMillis)); if (request.serviceName != null) { filter.must(boolQuery() .should(nestedQuery( "annotations", termQuery("annotations.endpoint.serviceName", request.serviceName))) .should(nestedQuery( "binaryAnnotations", termQuery("binaryAnnotations.endpoint.serviceName", request.serviceName)))); } if (request.spanName != null) { filter.must(termQuery("name", request.spanName)); } for (String annotation : request.annotations) { BoolQueryBuilder annotationQuery = boolQuery() .must(termQuery("annotations.value", annotation)); if (request.serviceName != null) { annotationQuery.must(termQuery("annotations.endpoint.serviceName", request.serviceName)); } filter.must(nestedQuery("annotations", annotationQuery)); } for (Map.Entry<String, String> kv : request.binaryAnnotations.entrySet()) { // In our index template, we make sure the binaryAnnotation value is indexed as string, // meaning non-string values won't even be indexed at all. This means that we can only // match string values here, which happens to be exactly what we want. BoolQueryBuilder binaryAnnotationQuery = boolQuery() .must(termQuery("binaryAnnotations.key", kv.getKey())) .must(termQuery("binaryAnnotations.value", kv.getValue())); if (request.serviceName != null) { binaryAnnotationQuery.must( termQuery("binaryAnnotations.endpoint.serviceName", request.serviceName)); } filter.must(nestedQuery("binaryAnnotations", binaryAnnotationQuery)); } if (request.minDuration != null) { RangeQueryBuilder durationQuery = rangeQuery("duration").gte(request.minDuration); if (request.maxDuration != null) { durationQuery.lte(request.maxDuration); } filter.must(durationQuery); } Set<String> strings = indexNameFormatter.indexNamePatternsForRange(beginMillis, endMillis); final String[] indices = strings.toArray(new String[0]); // We need to filter to traces that contain at least one span that matches the request, // but the zipkin API is supposed to order traces by first span, regardless of if it was // filtered or not. This is not possible without either multiple, heavyweight queries // or complex multiple indexing, defeating much of the elegance of using elasticsearch for this. // So we fudge and order on the first span among the filtered spans - in practice, there should // be no significant difference in user experience since span start times are usually very // close to each other in human time. ListenableFuture<List<String>> traceIds = client.collectBucketKeys(indices, boolQuery().must(matchAllQuery()).filter(filter), AggregationBuilders.terms("traceId_agg") .field("traceId") .subAggregation(AggregationBuilders.min("timestamps_agg") .field("timestamp_millis")) .order(Order.aggregation("timestamps_agg", false)) .size(request.limit)); return transform(traceIds, new AsyncFunction<List<String>, List<List<Span>>>() { @Override public ListenableFuture<List<List<Span>>> apply(List<String> input) { return getTracesByIds(input, indices); } } ); }
@Override public Observable<PersistentContainerKey> call(Void aVoid) { final Elasticsearch elasticSearch = vertxContext.verticle().elasticsearch(); RangeQueryBuilder reEncryptThresholdExceededFilter = rangeQuery("re_encrypt_ts") .lte(toDateTimeString(threshold)); BoolQueryBuilder query = boolQuery().must(reEncryptThresholdExceededFilter); if (LOGGER.isDebugEnabled()) { LOGGER.debug(format("Search Request {%s,%s} = %s", elasticSearch.defaultType(), elasticSearch.containerKeyIndex(), Jsonify.toString(query))); } SearchRequestBuilder request = elasticSearch.get() .prepareSearch( elasticSearch.containerKeyIndex()) .setVersion(true) .setTypes(elasticSearch.defaultType()) .setQuery(query) .setVersion(true) .setSize(5000) .setTimeout(timeValueMillis(elasticSearch.getDefaultSearchTimeout() - 10)); Cache<String, PersistentAccount> accountCache = newBuilder().maximumSize(1000).build(); Cache<String, PersistentContainer> containerCache = newBuilder().maximumSize(1000).build(); return elasticSearch.execute(vertxContext, request, elasticSearch.getDefaultSearchTimeout()) .flatMap(oSearchResponse -> { SearchResponse searchResponse = oSearchResponse.get(); if (LOGGER.isDebugEnabled()) { LOGGER.debug(format("Search Response {%s,%s} = %s", elasticSearch.defaultType(), elasticSearch.containerKeyIndex(), Jsonify.toString(searchResponse))); } if (oSearchResponse.isPresent()) { return from(oSearchResponse.get().getHits()); } else { return from(emptyList()); } }) .flatMap(searchHit -> { JsonObject jsonObject = new JsonObject(searchHit.getSourceAsString()); String accountId = jsonObject.getString("account_id"); String containerId = jsonObject.getString("container_id"); return loadAccount(vertxContext, accountId, accountCache) .flatMap(persistentAccount -> loadContainer(vertxContext, persistentAccount, containerId, containerCache)) .map(persistentContainer -> fromSearchHit(persistentContainer, searchHit)); }); }
public static String getRangeQuery(String fieldName, double from, double to) throws JSONException { RangeQueryBuilder range = QueryBuilders.rangeQuery(fieldName).from(from).to(to); return "{ \"query\": " + range.toString() + "}"; }