Java 类org.elasticsearch.index.query.InnerHitBuilder 实例源码

public void testNestedDefinedAsObject() throws Exception {
    assertAcked(prepareCreate("articles").addMapping("article", "comments", "type=nested", "title", "type=text"));

    List<IndexRequestBuilder> requests = new ArrayList<>();
    requests.add(client().prepareIndex("articles", "article", "1").setSource(jsonBuilder().startObject()
            .field("title", "quick brown fox")
            .startObject("comments").field("message", "fox eat quick").endObject()
            .endObject()));
    indexRandom(true, requests);

    SearchResponse response = client().prepareSearch("articles")
            .setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.Avg)
                    .innerHit(new InnerHitBuilder(), false))
            .get();
    assertNoFailures(response);
    assertHitCount(response, 1);
    assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
    assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getTotalHits(), equalTo(1L));
    assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getId(), equalTo("1"));
    assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getNestedIdentity().getField().string(),
            equalTo("comments"));
    assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getNestedIdentity().getOffset(), equalTo(0));
    assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getNestedIdentity().getChild(), nullValue());
}

public void testNestedInnerHitWrappedInParentChildInnerhit() throws Exception {
    assertAcked(prepareCreate("test").addMapping("child_type", "_parent", "type=parent_type", "nested_type", "type=nested"));
    client().prepareIndex("test", "parent_type", "1").setSource("key", "value").get();
    client().prepareIndex("test", "child_type", "2").setParent("1").setSource("nested_type", Collections.singletonMap("key", "value"))
        .get();
    refresh();
    SearchResponse response = client().prepareSearch("test")
        .setQuery(boolQuery().must(matchQuery("key", "value"))
            .should(hasChildQuery("child_type", nestedQuery("nested_type", matchAllQuery(), ScoreMode.None)
                .innerHit(new InnerHitBuilder(), false), ScoreMode.None).innerHit(new InnerHitBuilder(), false)))
        .get();
    assertHitCount(response, 1);
    SearchHit hit = response.getHits().getAt(0);
    assertThat(hit.getInnerHits().get("child_type").getAt(0).field("_parent").getValue(), equalTo("1"));
    assertThat(hit.getInnerHits().get("child_type").getAt(0).getInnerHits().get("nested_type").getAt(0).field("_parent"), nullValue());
}

public void testHasChildInnerHitsHighlighting() throws Exception {
    assertAcked(prepareCreate("test")
            .addMapping("parent")
            .addMapping("child", "_parent", "type=parent"));
    ensureGreen();

    client().prepareIndex("test", "parent", "1").setSource("p_field", 1).get();
    client().prepareIndex("test", "child", "2").setParent("1").setSource("c_field", "foo bar").get();
    client().admin().indices().prepareFlush("test").get();

    SearchResponse searchResponse = client().prepareSearch("test").setQuery(
            hasChildQuery("child", matchQuery("c_field", "foo"), ScoreMode.None)
                .innerHit(new InnerHitBuilder().setHighlightBuilder(
                    new HighlightBuilder().field(new Field("c_field")
                            .highlightQuery(QueryBuilders.matchQuery("c_field", "bar")))), false))
            .get();
    assertNoFailures(searchResponse);
    assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
    assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("1"));
    SearchHit[] searchHits = searchResponse.getHits().getHits()[0].getInnerHits().get("child").getHits();
    assertThat(searchHits.length, equalTo(1));
    assertThat(searchHits[0].getHighlightFields().get("c_field").getFragments().length, equalTo(1));
    assertThat(searchHits[0].getHighlightFields().get("c_field").getFragments()[0].string(), equalTo("foo <em>bar</em>"));
}

@Override
public SearchResponse getAllMetaAlertsForAlert(String guid) throws InvalidSearchException {
  if (guid == null || guid.trim().isEmpty()) {
    throw new InvalidSearchException("Guid cannot be empty");
  }
  // Searches for all alerts containing the meta alert guid in it's "metalerts" array
  QueryBuilder qb = boolQuery()
      .must(
          nestedQuery(
              ALERT_FIELD,
              boolQuery()
                  .must(termQuery(ALERT_FIELD + "." + GUID, guid)),
                  ScoreMode.None
          ).innerHit(new InnerHitBuilder())
      )
      .must(termQuery(STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString()));
  return queryAllResults(qb);
}

private SearchSourceBuilder buildExpandSearchSourceBuilder(InnerHitBuilder options) {
    SearchSourceBuilder groupSource = new SearchSourceBuilder();
    groupSource.from(options.getFrom());
    groupSource.size(options.getSize());
    if (options.getSorts() != null) {
        options.getSorts().forEach(groupSource::sort);
    }
    if (options.getFetchSourceContext() != null) {
        if (options.getFetchSourceContext().includes() == null && options.getFetchSourceContext().excludes() == null) {
            groupSource.fetchSource(options.getFetchSourceContext().fetchSource());
        } else {
            groupSource.fetchSource(options.getFetchSourceContext().includes(),
                options.getFetchSourceContext().excludes());
        }
    }
    if (options.getDocValueFields() != null) {
        options.getDocValueFields().forEach(groupSource::docValueField);
    }
    if (options.getStoredFieldsContext() != null && options.getStoredFieldsContext().fieldNames() != null) {
        options.getStoredFieldsContext().fieldNames().forEach(groupSource::storedField);
    }
    if (options.getScriptFields() != null) {
        for (SearchSourceBuilder.ScriptField field : options.getScriptFields()) {
            groupSource.scriptField(field.fieldName(), field.script());
        }
    }
    if (options.getHighlightBuilder() != null) {
        groupSource.highlighter(options.getHighlightBuilder());
    }
    groupSource.explain(options.isExplain());
    groupSource.trackScores(options.isTrackScores());
    return groupSource;
}

public void testMatchesQueriesParentChildInnerHits() throws Exception {
    assertAcked(prepareCreate("index").addMapping("child", "_parent", "type=parent"));
    List<IndexRequestBuilder> requests = new ArrayList<>();
    requests.add(client().prepareIndex("index", "parent", "1").setSource("{}", XContentType.JSON));
    requests.add(client().prepareIndex("index", "child", "1").setParent("1").setSource("field", "value1"));
    requests.add(client().prepareIndex("index", "child", "2").setParent("1").setSource("field", "value2"));
    requests.add(client().prepareIndex("index", "parent", "2").setSource("{}", XContentType.JSON));
    requests.add(client().prepareIndex("index", "child", "3").setParent("2").setSource("field", "value1"));
    indexRandom(true, requests);

    SearchResponse response = client().prepareSearch("index")
            .setQuery(hasChildQuery("child", matchQuery("field", "value1").queryName("_name1"), ScoreMode.None)
                    .innerHit(new InnerHitBuilder(), false))
            .addSort("_uid", SortOrder.ASC)
            .get();
    assertHitCount(response, 2);
    assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
    assertThat(response.getHits().getAt(0).getInnerHits().get("child").getTotalHits(), equalTo(1L));
    assertThat(response.getHits().getAt(0).getInnerHits().get("child").getAt(0).getMatchedQueries().length, equalTo(1));
    assertThat(response.getHits().getAt(0).getInnerHits().get("child").getAt(0).getMatchedQueries()[0], equalTo("_name1"));

    assertThat(response.getHits().getAt(1).getId(), equalTo("2"));
    assertThat(response.getHits().getAt(1).getInnerHits().get("child").getTotalHits(), equalTo(1L));
    assertThat(response.getHits().getAt(1).getInnerHits().get("child").getAt(0).getMatchedQueries().length, equalTo(1));
    assertThat(response.getHits().getAt(1).getInnerHits().get("child").getAt(0).getMatchedQueries()[0], equalTo("_name1"));

    QueryBuilder query = hasChildQuery("child", matchQuery("field", "value2").queryName("_name2"), ScoreMode.None)
            .innerHit(new InnerHitBuilder(), false);
    response = client().prepareSearch("index")
            .setQuery(query)
            .addSort("_uid", SortOrder.ASC)
            .get();
    assertHitCount(response, 1);
    assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
    assertThat(response.getHits().getAt(0).getInnerHits().get("child").getTotalHits(), equalTo(1L));
    assertThat(response.getHits().getAt(0).getInnerHits().get("child").getAt(0).getMatchedQueries().length, equalTo(1));
    assertThat(response.getHits().getAt(0).getInnerHits().get("child").getAt(0).getMatchedQueries()[0], equalTo("_name2"));
}

public void testDontExplode() throws Exception {
    assertAcked(prepareCreate("index1").addMapping("child", "_parent", "type=parent"));
    List<IndexRequestBuilder> requests = new ArrayList<>();
    requests.add(client().prepareIndex("index1", "parent", "1").setSource("{}", XContentType.JSON));
    requests.add(client().prepareIndex("index1", "child", "1").setParent("1").setSource("field", "value1"));
    indexRandom(true, requests);

    QueryBuilder query = hasChildQuery("child", matchQuery("field", "value1"), ScoreMode.None)
            .innerHit(new InnerHitBuilder().setSize(ArrayUtil.MAX_ARRAY_LENGTH - 1), false);
    SearchResponse response = client().prepareSearch("index1")
            .setQuery(query)
            .addSort("_uid", SortOrder.ASC)
            .get();
    assertNoFailures(response);
    assertHitCount(response, 1);

    assertAcked(prepareCreate("index2").addMapping("type", "nested", "type=nested"));
    client().prepareIndex("index2", "type", "1").setSource(jsonBuilder().startObject()
            .startArray("nested")
            .startObject()
            .field("field", "value1")
            .endObject()
            .endArray()
            .endObject())
    .setRefreshPolicy(IMMEDIATE)
    .get();

    query = nestedQuery("nested", matchQuery("nested.field", "value1"), ScoreMode.Avg)
            .innerHit(new InnerHitBuilder().setSize(ArrayUtil.MAX_ARRAY_LENGTH - 1), false);
    response = client().prepareSearch("index2")
            .setQuery(query)
            .addSort("_uid", SortOrder.ASC)
            .get();
    assertNoFailures(response);
    assertHitCount(response, 1);
}

public void testNestedSourceFiltering() throws Exception {
    assertAcked(prepareCreate("index1").addMapping("message", "comments", "type=nested"));
    client().prepareIndex("index1", "message", "1").setSource(jsonBuilder().startObject()
            .field("message", "quick brown fox")
            .startArray("comments")
            .startObject().field("message", "fox eat quick").endObject()
            .startObject().field("message", "fox ate rabbit x y z").endObject()
            .startObject().field("message", "rabbit got away").endObject()
            .endArray()
            .endObject()).get();
    refresh();

    // the field name (comments.message) used for source filtering should be the same as when using that field for
    // other features (like in the query dsl or aggs) in order for consistency:
    SearchResponse response = client().prepareSearch()
            .setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.None)
            .innerHit(new InnerHitBuilder().setFetchSourceContext(new FetchSourceContext(true,
                new String[]{"comments.message"}, null)), false))
            .get();
    assertNoFailures(response);
    assertHitCount(response, 1);

    assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getTotalHits(), equalTo(2L));
    assertThat(extractValue("comments.message", response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap()),
            equalTo("fox eat quick"));
    assertThat(extractValue("comments.message", response.getHits().getAt(0).getInnerHits().get("comments").getAt(1).getSourceAsMap()),
            equalTo("fox ate rabbit x y z"));
}

public void testInnerHitsWithIgnoreUnmapped() throws Exception {
    assertAcked(prepareCreate("index1")
        .addMapping("parent_type", "nested_type", "type=nested")
        .addMapping("child_type", "_parent", "type=parent_type")
    );
    createIndex("index2");
    client().prepareIndex("index1", "parent_type", "1").setSource("nested_type", Collections.singletonMap("key", "value")).get();
    client().prepareIndex("index1", "child_type", "2").setParent("1").setSource("{}", XContentType.JSON).get();
    client().prepareIndex("index2", "type", "3").setSource("key", "value").get();
    refresh();

    SearchResponse response = client().prepareSearch("index1", "index2")
        .setQuery(boolQuery()
            .should(nestedQuery("nested_type", matchAllQuery(), ScoreMode.None).ignoreUnmapped(true)
                    .innerHit(new InnerHitBuilder(), true))
            .should(termQuery("key", "value"))
        )
        .addSort("_uid", SortOrder.ASC)
        .get();
    assertNoFailures(response);
    assertHitCount(response, 2);
    assertThat(response.getHits().getAt(0).getId(), equalTo("1"));

    response = client().prepareSearch("index1", "index2")
        .setQuery(boolQuery()
            .should(hasChildQuery("child_type", matchAllQuery(), ScoreMode.None).ignoreUnmapped(true)
                    .innerHit(new InnerHitBuilder(), true))
            .should(termQuery("key", "value"))
        )
        .addSort("_uid", SortOrder.ASC)
        .get();
    assertNoFailures(response);
    assertHitCount(response, 2);
    assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
}

public static CollapseBuilder randomCollapseBuilder() {
    CollapseBuilder builder = new CollapseBuilder(randomAsciiOfLength(10));
    builder.setMaxConcurrentGroupRequests(randomIntBetween(1, 48));
    if (randomBoolean()) {
        InnerHitBuilder innerHit = InnerHitBuilderTests.randomInnerHits(false, false);
        builder.setInnerHits(innerHit);
    }
    return builder;
}

public void testBuild() throws IOException {
    Directory dir = new RAMDirectory();
    try (IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())))) {
        writer.commit();
    }
    SearchContext searchContext = mockSearchContext();
    try (IndexReader reader = DirectoryReader.open(dir)) {
        when(searchContext.getQueryShardContext().getIndexReader()).thenReturn(reader);
        MappedFieldType numberFieldType =
            new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
        MappedFieldType keywordFieldType =
            new KeywordFieldMapper.KeywordFieldType();
        for (MappedFieldType fieldType : new MappedFieldType[] {numberFieldType, keywordFieldType}) {
            fieldType.setName("field");
            fieldType.setHasDocValues(true);
            when(searchContext.getQueryShardContext().fieldMapper("field")).thenReturn(fieldType);
            CollapseBuilder builder = new CollapseBuilder("field");
            CollapseContext collapseContext = builder.build(searchContext);
            assertEquals(collapseContext.getFieldType(), fieldType);

            fieldType.setIndexOptions(IndexOptions.NONE);
            collapseContext = builder.build(searchContext);
            assertEquals(collapseContext.getFieldType(), fieldType);

            fieldType.setHasDocValues(false);
            SearchContextException exc = expectThrows(SearchContextException.class, () -> builder.build(searchContext));
            assertEquals(exc.getMessage(), "cannot collapse on field `field` without `doc_values`");

            fieldType.setHasDocValues(true);
            builder.setInnerHits(new InnerHitBuilder());
            exc = expectThrows(SearchContextException.class, () -> builder.build(searchContext));
            assertEquals(exc.getMessage(),
                "cannot expand `inner_hits` for collapse field `field`, " +
                    "only indexed field can retrieve `inner_hits`");
        }
    }
}

/**
 * Given an alert GUID, retrieve all associated meta alerts.
 * @param alertGuid The GUID of the child alert
 * @return The Elasticsearch response containing the meta alerts
 */
protected SearchResponse getMetaAlertsForAlert(String alertGuid) {
  QueryBuilder qb = boolQuery()
      .must(
          nestedQuery(
              ALERT_FIELD,
              boolQuery()
                  .must(termQuery(ALERT_FIELD + "." + Constants.GUID, alertGuid)),
              ScoreMode.None
          ).innerHit(new InnerHitBuilder())
      )
      .must(termQuery(STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString()));
  return queryAllResults(qb);
}

public CollapseContext(MappedFieldType fieldType, InnerHitBuilder innerHit) {
    this.fieldType = fieldType;
    this.innerHit = innerHit;
}

/** The inner hit options to expand the collapsed results **/
public InnerHitBuilder getInnerHit() {
    return innerHit;
}

public CollapseBuilder(StreamInput in) throws IOException {
    this.field = in.readString();
    this.maxConcurrentGroupRequests = in.readVInt();
    this.innerHit = in.readOptionalWriteable(InnerHitBuilder::new);
}

public CollapseBuilder setInnerHits(InnerHitBuilder innerHit) {
    this.innerHit = innerHit;
    return this;
}

/**
 * The inner hit options to expand the collapsed results
 */
public InnerHitBuilder getInnerHit() {
    return this.innerHit;
}

@Override
protected void extractInnerHitBuilders(Map<String, InnerHitBuilder> innerHits) {
    InnerHitBuilder.extractInnerHits(query(), innerHits);
}

public void testSimpleParentChild() throws Exception {
    assertAcked(prepareCreate("articles")
            .addMapping("article", "title", "type=text")
            .addMapping("comment", "_parent", "type=article", "message", "type=text,fielddata=true")
    );

    List<IndexRequestBuilder> requests = new ArrayList<>();
    requests.add(client().prepareIndex("articles", "article", "1").setSource("title", "quick brown fox"));
    requests.add(client().prepareIndex("articles", "comment", "1").setParent("1").setSource("message", "fox eat quick"));
    requests.add(client().prepareIndex("articles", "comment", "2").setParent("1").setSource("message", "fox ate rabbit x y z"));
    requests.add(client().prepareIndex("articles", "comment", "3").setParent("1").setSource("message", "rabbit got away"));
    requests.add(client().prepareIndex("articles", "article", "2").setSource("title", "big gray elephant"));
    requests.add(client().prepareIndex("articles", "comment", "4").setParent("2").setSource("message", "elephant captured"));
    requests.add(client().prepareIndex("articles", "comment", "5").setParent("2").setSource("message", "mice squashed by elephant x"));
    requests.add(client().prepareIndex("articles", "comment", "6").setParent("2").setSource("message", "elephant scared by mice x y"));
    indexRandom(true, requests);

    SearchResponse response = client().prepareSearch("articles")
            .setQuery(hasChildQuery("comment", matchQuery("message", "fox"), ScoreMode.None)
                    .innerHit(new InnerHitBuilder(), false))
            .get();
    assertNoFailures(response);
    assertHitCount(response, 1);
    assertSearchHit(response, 1, hasId("1"));
    assertThat(response.getHits().getAt(0).getShard(), notNullValue());

    assertThat(response.getHits().getAt(0).getInnerHits().size(), equalTo(1));
    SearchHits innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
    assertThat(innerHits.getTotalHits(), equalTo(2L));

    assertThat(innerHits.getAt(0).getId(), equalTo("1"));
    assertThat(innerHits.getAt(0).getType(), equalTo("comment"));
    assertThat(innerHits.getAt(1).getId(), equalTo("2"));
    assertThat(innerHits.getAt(1).getType(), equalTo("comment"));

    response = client().prepareSearch("articles")
            .setQuery(hasChildQuery("comment", matchQuery("message", "elephant"), ScoreMode.None)
                    .innerHit(new InnerHitBuilder(), false))
            .get();
    assertNoFailures(response);
    assertHitCount(response, 1);
    assertSearchHit(response, 1, hasId("2"));

    assertThat(response.getHits().getAt(0).getInnerHits().size(), equalTo(1));
    innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
    assertThat(innerHits.getTotalHits(), equalTo(3L));

    assertThat(innerHits.getAt(0).getId(), equalTo("4"));
    assertThat(innerHits.getAt(0).getType(), equalTo("comment"));
    assertThat(innerHits.getAt(1).getId(), equalTo("5"));
    assertThat(innerHits.getAt(1).getType(), equalTo("comment"));
    assertThat(innerHits.getAt(2).getId(), equalTo("6"));
    assertThat(innerHits.getAt(2).getType(), equalTo("comment"));

    response = client().prepareSearch("articles")
            .setQuery(
                    hasChildQuery("comment", matchQuery("message", "fox"), ScoreMode.None).innerHit(
                            new InnerHitBuilder()
                                    .addDocValueField("message")
                                    .setHighlightBuilder(new HighlightBuilder().field("message"))
                                    .setExplain(true).setSize(1)
                                    .addScriptField("script", new Script(ScriptType.INLINE, MockScriptEngine.NAME, "5",
                                        Collections.emptyMap())),
                            false)
            ).get();
    assertNoFailures(response);
    innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
    assertThat(innerHits.getHits().length, equalTo(1));
    assertThat(innerHits.getAt(0).getHighlightFields().get("message").getFragments()[0].string(), equalTo("<em>fox</em> eat quick"));
    assertThat(innerHits.getAt(0).getExplanation().toString(), containsString("weight(message:fox"));
    assertThat(innerHits.getAt(0).getFields().get("message").getValue().toString(), equalTo("eat"));
    assertThat(innerHits.getAt(0).getFields().get("script").getValue().toString(), equalTo("5"));
}

public void testInnerHitsOnHasParent() throws Exception {
    assertAcked(prepareCreate("stack")
                    .addMapping("question", "body", "type=text")
                    .addMapping("answer", "_parent", "type=question", "body", "type=text")
    );
    List<IndexRequestBuilder> requests = new ArrayList<>();
    requests.add(client().prepareIndex("stack", "question", "1").setSource("body", "I'm using HTTPS + Basic authentication "
            + "to protect a resource. How can I throttle authentication attempts to protect against brute force attacks?"));
    requests.add(client().prepareIndex("stack", "answer", "1").setParent("1").setSource("body",
            "install fail2ban and enable rules for apache"));
    requests.add(client().prepareIndex("stack", "question", "2").setSource("body",
            "I have firewall rules set up and also denyhosts installed.\\ndo I also need to install fail2ban?"));
    requests.add(client().prepareIndex("stack", "answer", "2").setParent("2").setSource("body",
            "Denyhosts protects only ssh; Fail2Ban protects all daemons."));
    indexRandom(true, requests);

    SearchResponse response = client().prepareSearch("stack")
            .setTypes("answer")
            .addSort("_uid", SortOrder.ASC)
            .setQuery(
                    boolQuery()
                            .must(matchQuery("body", "fail2ban"))
                            .must(hasParentQuery("question", matchAllQuery(), false).innerHit(new InnerHitBuilder(), false))
            ).get();
    assertNoFailures(response);
    assertHitCount(response, 2);

    SearchHit searchHit = response.getHits().getAt(0);
    assertThat(searchHit.getId(), equalTo("1"));
    assertThat(searchHit.getType(), equalTo("answer"));
    assertThat(searchHit.getInnerHits().get("question").getTotalHits(), equalTo(1L));
    assertThat(searchHit.getInnerHits().get("question").getAt(0).getType(), equalTo("question"));
    assertThat(searchHit.getInnerHits().get("question").getAt(0).getId(), equalTo("1"));

    searchHit = response.getHits().getAt(1);
    assertThat(searchHit.getId(), equalTo("2"));
    assertThat(searchHit.getType(), equalTo("answer"));
    assertThat(searchHit.getInnerHits().get("question").getTotalHits(), equalTo(1L));
    assertThat(searchHit.getInnerHits().get("question").getAt(0).getType(), equalTo("question"));
    assertThat(searchHit.getInnerHits().get("question").getAt(0).getId(), equalTo("2"));
}

public void testParentChildMultipleLayers() throws Exception {
    assertAcked(prepareCreate("articles")
                    .addMapping("article", "title", "type=text")
                    .addMapping("comment", "_parent", "type=article", "message", "type=text")
                    .addMapping("remark", "_parent", "type=comment", "message", "type=text")
    );

    List<IndexRequestBuilder> requests = new ArrayList<>();
    requests.add(client().prepareIndex("articles", "article", "1").setSource("title", "quick brown fox"));
    requests.add(client().prepareIndex("articles", "comment", "1").setParent("1").setSource("message", "fox eat quick"));
    requests.add(client().prepareIndex("articles", "remark", "1").setParent("1").setRouting("1").setSource("message", "good"));
    requests.add(client().prepareIndex("articles", "article", "2").setSource("title", "big gray elephant"));
    requests.add(client().prepareIndex("articles", "comment", "2").setParent("2").setSource("message", "elephant captured"));
    requests.add(client().prepareIndex("articles", "remark", "2").setParent("2").setRouting("2").setSource("message", "bad"));
    indexRandom(true, requests);

    SearchResponse response = client().prepareSearch("articles")
            .setQuery(hasChildQuery("comment",
                        hasChildQuery("remark", matchQuery("message", "good"), ScoreMode.None).innerHit(new InnerHitBuilder(), false),
                    ScoreMode.None).innerHit(new InnerHitBuilder(), false))
            .get();

    assertNoFailures(response);
    assertHitCount(response, 1);
    assertSearchHit(response, 1, hasId("1"));

    assertThat(response.getHits().getAt(0).getInnerHits().size(), equalTo(1));
    SearchHits innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
    assertThat(innerHits.getTotalHits(), equalTo(1L));
    assertThat(innerHits.getAt(0).getId(), equalTo("1"));
    assertThat(innerHits.getAt(0).getType(), equalTo("comment"));

    innerHits = innerHits.getAt(0).getInnerHits().get("remark");
    assertThat(innerHits.getTotalHits(), equalTo(1L));
    assertThat(innerHits.getAt(0).getId(), equalTo("1"));
    assertThat(innerHits.getAt(0).getType(), equalTo("remark"));

    response = client().prepareSearch("articles")
            .setQuery(hasChildQuery("comment",
                    hasChildQuery("remark", matchQuery("message", "bad"), ScoreMode.None).innerHit(new InnerHitBuilder(), false),
                    ScoreMode.None).innerHit(new InnerHitBuilder(), false))
            .get();

    assertNoFailures(response);
    assertHitCount(response, 1);
    assertSearchHit(response, 1, hasId("2"));

    assertThat(response.getHits().getAt(0).getInnerHits().size(), equalTo(1));
    innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
    assertThat(innerHits.getTotalHits(), equalTo(1L));
    assertThat(innerHits.getAt(0).getId(), equalTo("2"));
    assertThat(innerHits.getAt(0).getType(), equalTo("comment"));

    innerHits = innerHits.getAt(0).getInnerHits().get("remark");
    assertThat(innerHits.getTotalHits(), equalTo(1L));
    assertThat(innerHits.getAt(0).getId(), equalTo("2"));
    assertThat(innerHits.getAt(0).getType(), equalTo("remark"));
}

public void testInnerHitsWithObjectFieldThatHasANestedField() throws Exception {
    assertAcked(prepareCreate("articles")
                    .addMapping("article", jsonBuilder().startObject()
                                    .startObject("properties")
                                        .startObject("comments")
                                            .field("type", "object")
                                            .startObject("properties")
                                                .startObject("messages").field("type", "nested").endObject()
                                            .endObject()
                                            .endObject()
                                        .endObject()
                                    .endObject()
                    )
    );

    List<IndexRequestBuilder> requests = new ArrayList<>();
    requests.add(client().prepareIndex("articles", "article", "1").setSource(jsonBuilder().startObject()
            .field("title", "quick brown fox")
            .startObject("comments")
            .startArray("messages")
                .startObject().field("message", "fox eat quick").endObject()
                .startObject().field("message", "bear eat quick").endObject()
            .endArray()
            .endObject()
            .endObject()));
    indexRandom(true, requests);

    SearchResponse response = client().prepareSearch("articles")
            .setQuery(nestedQuery("comments.messages", matchQuery("comments.messages.message", "fox"), ScoreMode.Avg)
                    .innerHit(new InnerHitBuilder(), false)).get();
    assertNoFailures(response);
    assertHitCount(response, 1);
    SearchHit hit = response.getHits().getAt(0);
    assertThat(hit.getId(), equalTo("1"));
    SearchHits messages = hit.getInnerHits().get("comments.messages");
    assertThat(messages.getTotalHits(), equalTo(1L));
    assertThat(messages.getAt(0).getId(), equalTo("1"));
    assertThat(messages.getAt(0).getNestedIdentity().getField().string(), equalTo("comments.messages"));
    assertThat(messages.getAt(0).getNestedIdentity().getOffset(), equalTo(0));
    assertThat(messages.getAt(0).getNestedIdentity().getChild(), nullValue());

    response = client().prepareSearch("articles")
            .setQuery(nestedQuery("comments.messages", matchQuery("comments.messages.message", "bear"), ScoreMode.Avg)
                    .innerHit(new InnerHitBuilder(), false)).get();
    assertNoFailures(response);
    assertHitCount(response, 1);
    hit = response.getHits().getAt(0);
    assertThat(hit.getId(), equalTo("1"));
    messages = hit.getInnerHits().get("comments.messages");
    assertThat(messages.getTotalHits(), equalTo(1L));
    assertThat(messages.getAt(0).getId(), equalTo("1"));
    assertThat(messages.getAt(0).getNestedIdentity().getField().string(), equalTo("comments.messages"));
    assertThat(messages.getAt(0).getNestedIdentity().getOffset(), equalTo(1));
    assertThat(messages.getAt(0).getNestedIdentity().getChild(), nullValue());

    // index the message in an object form instead of an array
    requests = new ArrayList<>();
    requests.add(client().prepareIndex("articles", "article", "1").setSource(jsonBuilder().startObject()
            .field("title", "quick brown fox")
            .startObject("comments").startObject("messages").field("message", "fox eat quick").endObject().endObject()
            .endObject()));
    indexRandom(true, requests);
    response = client().prepareSearch("articles")
            .setQuery(nestedQuery("comments.messages", matchQuery("comments.messages.message", "fox"), ScoreMode.Avg)
                    .innerHit(new InnerHitBuilder(), false)).get();
    assertNoFailures(response);
    assertHitCount(response, 1);
    hit = response.getHits().getAt(0);;
    assertThat(hit.getId(), equalTo("1"));
    messages = hit.getInnerHits().get("comments.messages");
    assertThat(messages.getTotalHits(), equalTo(1L));
    assertThat(messages.getAt(0).getId(), equalTo("1"));
    assertThat(messages.getAt(0).getNestedIdentity().getField().string(), equalTo("comments.messages"));
    assertThat(messages.getAt(0).getNestedIdentity().getOffset(), equalTo(0));
    assertThat(messages.getAt(0).getNestedIdentity().getChild(), nullValue());
}

public void testCollapseSingleHit() throws IOException {
    final int iters = randomIntBetween(5, 10);
    for (int i = 0; i < iters; i++) {
        SearchHits collapsedHits = new SearchHits(new SearchHit[]{new SearchHit(2, "ID", new Text("type"),
            Collections.emptyMap()), new SearchHit(3, "ID", new Text("type"),
            Collections.emptyMap())}, 1, 1.0F);
        AtomicBoolean executedMultiSearch = new AtomicBoolean(false);
        QueryBuilder originalQuery = randomBoolean() ? null : QueryBuilders.termQuery("foo", "bar");

        MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1);
        String collapseValue = randomBoolean() ? null : "boom";
        mockSearchPhaseContext.getRequest().source(new SearchSourceBuilder()
            .collapse(new CollapseBuilder("someField").setInnerHits(new InnerHitBuilder().setName("foobarbaz"))));
        mockSearchPhaseContext.getRequest().source().query(originalQuery);
        mockSearchPhaseContext.searchTransport = new SearchTransportService(
            Settings.builder().put("search.remote.connect", false).build(), null, null) {

            @Override
            void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionListener<MultiSearchResponse> listener) {
                assertTrue(executedMultiSearch.compareAndSet(false, true));
                assertEquals(1, request.requests().size());
                SearchRequest searchRequest = request.requests().get(0);
                assertTrue(searchRequest.source().query() instanceof BoolQueryBuilder);
                BoolQueryBuilder groupBuilder = (BoolQueryBuilder) searchRequest.source().query();
                if (collapseValue == null) {
                    assertThat(groupBuilder.mustNot(), Matchers.contains(QueryBuilders.existsQuery("someField")));
                } else {
                    assertThat(groupBuilder.filter(), Matchers.contains(QueryBuilders.matchQuery("someField", "boom")));
                }
                if (originalQuery != null) {
                    assertThat(groupBuilder.must(), Matchers.contains(QueryBuilders.termQuery("foo", "bar")));
                }
                assertArrayEquals(mockSearchPhaseContext.getRequest().indices(), searchRequest.indices());
                assertArrayEquals(mockSearchPhaseContext.getRequest().types(), searchRequest.types());


                InternalSearchResponse internalSearchResponse = new InternalSearchResponse(collapsedHits,
                    null, null, null, false, null, 1);
                SearchResponse response = mockSearchPhaseContext.buildSearchResponse(internalSearchResponse, null);
                listener.onResponse(new MultiSearchResponse(new MultiSearchResponse.Item[]{
                    new MultiSearchResponse.Item(response, null)
                }));

            }
        };

        SearchHits hits = new SearchHits(new SearchHit[]{new SearchHit(1, "ID", new Text("type"),
            Collections.singletonMap("someField", new SearchHitField("someField", Collections.singletonList(collapseValue))))},
            1, 1.0F);
        InternalSearchResponse internalSearchResponse = new InternalSearchResponse(hits, null, null, null, false, null, 1);
        SearchResponse response = mockSearchPhaseContext.buildSearchResponse(internalSearchResponse, null);
        AtomicReference<SearchResponse> reference = new AtomicReference<>();
        ExpandSearchPhase phase = new ExpandSearchPhase(mockSearchPhaseContext, response, r ->
            new SearchPhase("test") {
                @Override
                public void run() throws IOException {
                    reference.set(r);
                }
            }
        );

        phase.run();
        mockSearchPhaseContext.assertNoFailure();
        assertNotNull(reference.get());
        SearchResponse theResponse = reference.get();
        assertSame(theResponse, response);
        assertEquals(1, theResponse.getHits().getHits()[0].getInnerHits().size());
        assertSame(theResponse.getHits().getHits()[0].getInnerHits().get("foobarbaz"), collapsedHits);
        assertTrue(executedMultiSearch.get());
        assertEquals(1, mockSearchPhaseContext.phasesExecuted.get());
    }
}

public void testFailOneItemFailsEntirePhase() throws IOException {
    AtomicBoolean executedMultiSearch = new AtomicBoolean(false);

    SearchHits collapsedHits = new SearchHits(new SearchHit[]{new SearchHit(2, "ID", new Text("type"),
        Collections.emptyMap()), new SearchHit(3, "ID", new Text("type"),
        Collections.emptyMap())}, 1, 1.0F);
    MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1);
    String collapseValue = randomBoolean() ? null : "boom";
    mockSearchPhaseContext.getRequest().source(new SearchSourceBuilder()
        .collapse(new CollapseBuilder("someField").setInnerHits(new InnerHitBuilder().setName("foobarbaz"))));
    mockSearchPhaseContext.searchTransport = new SearchTransportService(
        Settings.builder().put("search.remote.connect", false).build(), null, null) {

        @Override
        void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionListener<MultiSearchResponse> listener) {
            assertTrue(executedMultiSearch.compareAndSet(false, true));
            InternalSearchResponse internalSearchResponse = new InternalSearchResponse(collapsedHits,
                null, null, null, false, null, 1);
            SearchResponse response = mockSearchPhaseContext.buildSearchResponse(internalSearchResponse, null);
            listener.onResponse(new MultiSearchResponse(new MultiSearchResponse.Item[]{
                new MultiSearchResponse.Item(null, new RuntimeException("boom")),
                new MultiSearchResponse.Item(response, null)
            }));
        }
    };

    SearchHits hits = new SearchHits(new SearchHit[]{new SearchHit(1, "ID", new Text("type"),
        Collections.singletonMap("someField", new SearchHitField("someField", Collections.singletonList(collapseValue)))),
        new SearchHit(2, "ID2", new Text("type"),
            Collections.singletonMap("someField", new SearchHitField("someField", Collections.singletonList(collapseValue))))}, 1,
        1.0F);
    InternalSearchResponse internalSearchResponse = new InternalSearchResponse(hits, null, null, null, false, null, 1);
    SearchResponse response = mockSearchPhaseContext.buildSearchResponse(internalSearchResponse, null);
    AtomicReference<SearchResponse> reference = new AtomicReference<>();
    ExpandSearchPhase phase = new ExpandSearchPhase(mockSearchPhaseContext, response, r ->
        new SearchPhase("test") {
            @Override
            public void run() throws IOException {
                reference.set(r);
            }
        }
    );
    phase.run();
    assertThat(mockSearchPhaseContext.phaseFailure.get(), Matchers.instanceOf(RuntimeException.class));
    assertEquals("boom", mockSearchPhaseContext.phaseFailure.get().getMessage());
    assertNotNull(mockSearchPhaseContext.phaseFailure.get());
    assertNull(reference.get());
    assertEquals(0, mockSearchPhaseContext.phasesExecuted.get());
}

@Override
protected void populateBuffer() {
    // not used yet or returned empty results
    if (lastDate == null) {
        lastDate = new Date();
        lastStartOffset = 0;
    }
    // been running same query for too long and paging deep?
    else if (maxStartOffset != -1 && lastStartOffset > maxStartOffset) {
        LOG.info("Reached max start offset {}", lastStartOffset);
        lastStartOffset = 0;
    }

    String formattedLastDate = ISODateTimeFormat.dateTimeNoMillis().print(
            lastDate.getTime());

    LOG.info("{} Populating buffer with nextFetchDate <= {}", logIdprefix,
            formattedLastDate);

    QueryBuilder queryBuilder = QueryBuilders.rangeQuery("nextFetchDate")
            .lte(formattedLastDate);

    if (filterQuery != null) {
        queryBuilder = boolQuery().must(queryBuilder).filter(
                QueryBuilders.queryStringQuery(filterQuery));
    }

    SearchRequestBuilder srb = client.prepareSearch(indexName)
            .setTypes(docType).setSearchType(SearchType.QUERY_THEN_FETCH)
            .setQuery(queryBuilder).setFrom(lastStartOffset)
            .setSize(maxBucketNum).setExplain(false);

    // https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-preference.html
    // _shards:2,3
    if (shardID != -1) {
        srb.setPreference("_shards:" + shardID);
    }

    if (StringUtils.isNotBlank(totalSortField)) {
        FieldSortBuilder sorter = SortBuilders.fieldSort(totalSortField)
                .order(SortOrder.ASC);
        srb.addSort(sorter);
    }

    CollapseBuilder collapse = new CollapseBuilder(partitionField);
    srb.setCollapse(collapse);

    // group expansion -> sends sub queries for each bucket
    if (maxURLsPerBucket > 1) {
        InnerHitBuilder ihb = new InnerHitBuilder();
        ihb.setSize(maxURLsPerBucket);
        ihb.setName("urls_per_bucket");
        // sort within a bucket
        if (StringUtils.isNotBlank(bucketSortField)) {
            List<SortBuilder<?>> sorts = new LinkedList<>();
            FieldSortBuilder bucketsorter = SortBuilders.fieldSort(
                    bucketSortField).order(SortOrder.ASC);
            sorts.add(bucketsorter);
            ihb.setSorts(sorts);
        }
        collapse.setInnerHits(ihb);
    }

    // dump query to log
    LOG.debug("{} ES query {}", logIdprefix, srb.toString());

    timeStartESQuery = System.currentTimeMillis();
    isInESQuery.set(true);
    srb.execute(this);
}