Java 类org.apache.lucene.index.RandomIndexWriter 实例源码

public void testRandom() throws IOException {
    final int numDocs = randomIntBetween(10, 200);
    final Document doc = new Document();
    final Directory dir = newDirectory();
    final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
    for (int i = 0; i < numDocs; ++i) {
        w.addDocument(doc);
    }
    final IndexReader reader = w.getReader();
    final IndexSearcher searcher = newSearcher(reader);
    for (int i = 0; i <= numDocs; ++i) {
        assertEquals(numDocs - i, searcher.count(new MinDocQuery(i)));
    }
    w.close();
    reader.close();
    dir.close();
}

private IndexReader indexOneDoc(Directory dir, String field, String value, Analyzer analyzer) throws IOException {
    IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
    iwc.setMergePolicy(newLogMergePolicy());
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);

    FieldType ft = new FieldType(TextField.TYPE_STORED);
    ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
    Field textField = new Field(field, "", ft);
    Document doc = new Document();
    doc.add(textField);

    textField.setStringValue(value);
    iw.addDocument(doc);
    IndexReader ir = iw.getReader();
    iw.close();
    return ir;
}

@SuppressWarnings("unchecked")
public void testNoDocs() throws IOException {
    try (Directory directory = newDirectory()) {
        try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
            // intentionally not writing any docs
        }
        try (IndexReader indexReader = DirectoryReader.open(directory)) {
            ScriptedMetricAggregationBuilder aggregationBuilder = new ScriptedMetricAggregationBuilder(AGG_NAME);
            aggregationBuilder.mapScript(MAP_SCRIPT); // map script is mandatory, even if its not used in this case
            ScriptedMetric scriptedMetric = search(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder);
            assertEquals(AGG_NAME, scriptedMetric.getName());
            assertNotNull(scriptedMetric.aggregation());
            assertEquals(0, ((HashMap<Object, String>) scriptedMetric.aggregation()).size());
        }
    }
}

/**
 * without combine script, the "_aggs" map should contain a list of the size of the number of documents matched
 */
@SuppressWarnings("unchecked")
public void testScriptedMetricWithoutCombine() throws IOException {
    try (Directory directory = newDirectory()) {
        int numDocs = randomInt(100);
        try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
            for (int i = 0; i < numDocs; i++) {
                indexWriter.addDocument(singleton(new SortedNumericDocValuesField("number", i)));
            }
        }
        try (IndexReader indexReader = DirectoryReader.open(directory)) {
            ScriptedMetricAggregationBuilder aggregationBuilder = new ScriptedMetricAggregationBuilder(AGG_NAME);
            aggregationBuilder.initScript(INIT_SCRIPT).mapScript(MAP_SCRIPT);
            ScriptedMetric scriptedMetric = search(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder);
            assertEquals(AGG_NAME, scriptedMetric.getName());
            assertNotNull(scriptedMetric.aggregation());
            Map<String, Object> agg = (Map<String, Object>) scriptedMetric.aggregation();
            assertEquals(numDocs, ((List<Integer>) agg.get("collector")).size());
        }
    }
}

/**
 * test that combine script sums the list produced by the "mapScript"
 */
public void testScriptedMetricWithCombine() throws IOException {
    try (Directory directory = newDirectory()) {
        Integer numDocs = randomInt(100);
        try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
            for (int i = 0; i < numDocs; i++) {
                indexWriter.addDocument(singleton(new SortedNumericDocValuesField("number", i)));
            }
        }
        try (IndexReader indexReader = DirectoryReader.open(directory)) {
            ScriptedMetricAggregationBuilder aggregationBuilder = new ScriptedMetricAggregationBuilder(AGG_NAME);
            aggregationBuilder.initScript(INIT_SCRIPT).mapScript(MAP_SCRIPT).combineScript(COMBINE_SCRIPT);
            ScriptedMetric scriptedMetric = search(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder);
            assertEquals(AGG_NAME, scriptedMetric.getName());
            assertNotNull(scriptedMetric.aggregation());
            assertEquals(numDocs, scriptedMetric.aggregation());
        }
    }
}

public void testCase(MappedFieldType ft,
                     CheckedConsumer<RandomIndexWriter, IOException> buildIndex,
                     Consumer<InternalExtendedStats> verify) throws IOException {
    try (Directory directory = newDirectory();
         RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
        buildIndex.accept(indexWriter);
        try (IndexReader reader = indexWriter.getReader()) {
            IndexSearcher searcher = new IndexSearcher(reader);
            ExtendedStatsAggregationBuilder aggBuilder = new ExtendedStatsAggregationBuilder("my_agg")
                .field("field")
                .sigma(randomDoubleBetween(0, 10, true));
            InternalExtendedStats stats = search(searcher, new MatchAllDocsQuery(), aggBuilder, ft);
            verify.accept(stats);
        }
    }
}

public void testEmpty() throws Exception {
    try (Directory dir = newDirectory();
         RandomIndexWriter w = new RandomIndexWriter(random(), dir)) {
        GeoBoundsAggregationBuilder aggBuilder = new GeoBoundsAggregationBuilder("my_agg")
            .field("field")
            .wrapLongitude(false);

        MappedFieldType fieldType = new GeoPointFieldMapper.GeoPointFieldType();
        fieldType.setHasDocValues(true);
        fieldType.setName("field");
        try (IndexReader reader = w.getReader()) {
            IndexSearcher searcher = new IndexSearcher(reader);
            InternalGeoBounds bounds = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType);
            assertTrue(Double.isInfinite(bounds.top));
            assertTrue(Double.isInfinite(bounds.bottom));
            assertTrue(Double.isInfinite(bounds.posLeft));
            assertTrue(Double.isInfinite(bounds.posRight));
            assertTrue(Double.isInfinite(bounds.negLeft));
            assertTrue(Double.isInfinite(bounds.negRight));
        }
    }
}

private void testCase(Query query, CheckedConsumer<RandomIndexWriter, IOException> buildIndex, Consumer<InternalMax> verify)
        throws IOException {
    Directory directory = newDirectory();
    RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
    buildIndex.accept(indexWriter);
    indexWriter.close();

    IndexReader indexReader = DirectoryReader.open(directory);
    IndexSearcher indexSearcher = newSearcher(indexReader, true, true);

    MaxAggregationBuilder aggregationBuilder = new MaxAggregationBuilder("_name").field("number");
    MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
    fieldType.setName("number");
    try (MaxAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType)) {
        aggregator.preCollection();
        indexSearcher.search(query, aggregator);
        aggregator.postCollection();
        verify.accept((InternalMax) aggregator.buildAggregation(0L));
    }
    indexReader.close();
    directory.close();
}

private Aggregation testCase(Query query, AggregationBuilder builder) throws IOException {
    Directory directory = newDirectory();
    RandomIndexWriter iw = new RandomIndexWriter(random(), directory);
    iw.addDocument(document("1", "a", "b"));
    iw.addDocument(document("2", "c", "a"));
    iw.addDocument(document("3", "b", "d"));
    iw.close();

    IndexReader indexReader = DirectoryReader.open(directory);
    // We do not use LuceneTestCase.newSearcher because we need a DirectoryReader for "testInsideTerms"
    IndexSearcher indexSearcher = new IndexSearcher(indexReader);

    Aggregation result = searchAndReduce(indexSearcher, query, builder, STRING_FIELD_TYPE);
    indexReader.close();
    directory.close();
    return result;
}

private void testCase(Query query, CheckedConsumer<RandomIndexWriter, IOException> buildIndex, Consumer<InternalAvg> verify)
        throws IOException {
    Directory directory = newDirectory();
    RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
    buildIndex.accept(indexWriter);
    indexWriter.close();

    IndexReader indexReader = DirectoryReader.open(directory);
    IndexSearcher indexSearcher = newSearcher(indexReader, true, true);

    AvgAggregationBuilder aggregationBuilder = new AvgAggregationBuilder("_name").field("number");
    MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
    fieldType.setName("number");
    try (AvgAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType)) {
        aggregator.preCollection();
        indexSearcher.search(query, aggregator);
        aggregator.postCollection();
        verify.accept((InternalAvg) aggregator.buildAggregation(0L));
    }
    indexReader.close();
    directory.close();
}

public void testMinAggregator_noDocs() throws Exception {
    Directory directory = newDirectory();
    RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
    indexWriter.close();

    IndexReader indexReader = DirectoryReader.open(directory);
    IndexSearcher indexSearcher = newSearcher(indexReader, true, true);

    MinAggregationBuilder aggregationBuilder = new MinAggregationBuilder("_name").field("number");
    MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
    fieldType.setName("number");
    try (MinAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType)) {
        aggregator.preCollection();
        indexSearcher.search(new MatchAllDocsQuery(), aggregator);
        aggregator.postCollection();
        InternalMin result = (InternalMin) aggregator.buildAggregation(0L);
        assertEquals(Double.POSITIVE_INFINITY, result.getValue(), 0);
    }
    indexReader.close();
    directory.close();
}

public void testNoDocs() throws IOException {
    Directory directory = newDirectory();

    RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
    // intentionally not writing any docs
    indexWriter.close();
    IndexReader indexReader = DirectoryReader.open(directory);

    testCase(new MatchAllDocsQuery(), newSearcher(indexReader, false, true), parentToChild -> {
        assertEquals(0, parentToChild.getDocCount());
        assertEquals(Double.POSITIVE_INFINITY, ((InternalMin) parentToChild.getAggregations().get("in_child")).getValue(),
                Double.MIN_VALUE);
    });
    indexReader.close();
    directory.close();
}

private static Map<String, Tuple<Integer, Integer>> setupIndex(RandomIndexWriter iw) throws IOException {
    Map<String, Tuple<Integer, Integer>> expectedValues = new HashMap<>();
    int numParents = randomIntBetween(1, 10);
    for (int i = 0; i < numParents; i++) {
        String parent = "parent" + i;
        iw.addDocument(createParentDocument(parent));
        int numChildren = randomIntBetween(1, 10);
        int minValue = Integer.MAX_VALUE;
        for (int c = 0; c < numChildren; c++) {
            int randomValue = randomIntBetween(0, 100);
            minValue = Math.min(minValue, randomValue);
            iw.addDocument(createChildDocument("child" + c + "_" + parent, parent, randomValue));
        }
        expectedValues.put(parent, new Tuple<>(numChildren, minValue));
    }
    return expectedValues;
}

public void testDiversifiedSampler_noDocs() throws Exception {
    Directory directory = newDirectory();
    RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
    indexWriter.close();
    IndexReader indexReader = DirectoryReader.open(directory);
    IndexSearcher indexSearcher = new IndexSearcher(indexReader);

    MappedFieldType idFieldType = new KeywordFieldMapper.KeywordFieldType();
    idFieldType.setName("id");
    idFieldType.setHasDocValues(true);

    MappedFieldType genreFieldType = new KeywordFieldMapper.KeywordFieldType();
    genreFieldType.setName("genre");
    genreFieldType.setHasDocValues(true);

    DiversifiedAggregationBuilder builder = new DiversifiedAggregationBuilder("_name")
            .field(genreFieldType.name())
            .subAggregation(new TermsAggregationBuilder("terms", null).field("id"));

    InternalSampler result = search(indexSearcher, new MatchAllDocsQuery(), builder, genreFieldType, idFieldType);
    Terms terms = result.getAggregations().get("terms");
    assertEquals(0, terms.getBuckets().size());
    indexReader.close();
    directory.close();
}

private void testCase(Query query, String field, int precision, CheckedConsumer<RandomIndexWriter, IOException> buildIndex,
                      Consumer<InternalGeoHashGrid> verify) throws IOException {
    Directory directory = newDirectory();
    RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
    buildIndex.accept(indexWriter);
    indexWriter.close();

    IndexReader indexReader = DirectoryReader.open(directory);
    IndexSearcher indexSearcher = newSearcher(indexReader, true, true);

    GeoGridAggregationBuilder aggregationBuilder = new GeoGridAggregationBuilder("_name").field(field);
    aggregationBuilder.precision(precision);
    MappedFieldType fieldType = new GeoPointFieldMapper.GeoPointFieldType();
    fieldType.setHasDocValues(true);
    fieldType.setName(FIELD_NAME);
    try (Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType)) {
        aggregator.preCollection();
        indexSearcher.search(query, aggregator);
        aggregator.postCollection();
        verify.accept((InternalGeoHashGrid) aggregator.buildAggregation(0L));
    }
    indexReader.close();
    directory.close();
}

public void testEmpty() throws Exception {
    Directory directory = newDirectory();
    RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
    indexWriter.close();
    IndexReader indexReader = DirectoryReader.open(directory);
    IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
    int numFilters = randomIntBetween(1, 10);
    QueryBuilder[] filters = new QueryBuilder[numFilters];
    for (int i = 0; i < filters.length; i++) {
        filters[i] = QueryBuilders.termQuery("field", randomAsciiOfLength(5));
    }
    FiltersAggregationBuilder builder = new FiltersAggregationBuilder("test", filters);
    builder.otherBucketKey("other");
    InternalFilters response = search(indexSearcher, new MatchAllDocsQuery(), builder, fieldType);
    assertEquals(response.getBuckets().size(), numFilters);
    for (InternalFilters.InternalBucket filter : response.getBuckets()) {
        assertEquals(filter.getDocCount(), 0);
    }
    indexReader.close();
    directory.close();
}

public void testMinDocCount() throws Exception {
    try (Directory dir = newDirectory();
            RandomIndexWriter w = new RandomIndexWriter(random(), dir)) {
        for (long value : new long[] {7, 3, -10, -6, 5, 50}) {
            Document doc = new Document();
            doc.add(new SortedNumericDocValuesField("field", value));
            w.addDocument(doc);
        }

        HistogramAggregationBuilder aggBuilder = new HistogramAggregationBuilder("my_agg")
                .field("field")
                .interval(10)
                .minDocCount(2);
        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
        fieldType.setName("field");
        try (IndexReader reader = w.getReader()) {
            IndexSearcher searcher = new IndexSearcher(reader);
            Histogram histogram = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType);
            assertEquals(2, histogram.getBuckets().size());
            assertEquals(-10d, histogram.getBuckets().get(0).getKey());
            assertEquals(2, histogram.getBuckets().get(0).getDocCount());
            assertEquals(0d, histogram.getBuckets().get(1).getKey());
            assertEquals(3, histogram.getBuckets().get(1).getDocCount());
        }
    }
}

@BeforeClass
public static void setup() throws IOException {
    dir = newDirectory();
    RandomIndexWriter w = new RandomIndexWriter(random(), dir);
    final int numDocs = TestUtil.nextInt(random(), 1, 20);
    for (int i = 0; i < numDocs; ++i) {
        final int numHoles = random().nextInt(5);
        for (int j = 0; j < numHoles; ++j) {
            w.addDocument(new Document());
        }
        Document doc = new Document();
        doc.add(new StringField("foo", "bar", Store.NO));
        w.addDocument(doc);
    }
    reader = w.getReader();
    w.close();
    Engine.Searcher engineSearcher = new Engine.Searcher("test", new IndexSearcher(reader));
    searcher = new ContextIndexSearcher(engineSearcher, IndexSearcher.getDefaultQueryCache(), MAYBE_CACHE_POLICY);
}

public void testMissingShard() throws IOException {
    try (Directory dir = newDirectory();
            RandomIndexWriter w = new RandomIndexWriter(random(), dir)) {
        w.addDocument(new Document());
        try (IndexReader reader = w.getReader()) {
            ShardCoreKeyMap map = new ShardCoreKeyMap();
            for (LeafReaderContext ctx : reader.leaves()) {
                try {
                    map.add(ctx.reader());
                    fail();
                } catch (IllegalArgumentException expected) {
                    // ok
                }
            }
        }
    }
}

public void testAddingAClosedReader() throws Exception {
    LeafReader reader;
    try (Directory dir = newDirectory();
            RandomIndexWriter writer = new RandomIndexWriter(random(), dir)) {
        writer.addDocument(new Document());
        try (DirectoryReader dirReader = ElasticsearchDirectoryReader.wrap(writer.getReader(), new ShardId("index1", "_na_", 1))) {
            reader = dirReader.leaves().get(0).reader();
        }
    }
    ShardCoreKeyMap map = new ShardCoreKeyMap();
    try {
        map.add(reader);
        fail("Expected AlreadyClosedException");
    } catch (AlreadyClosedException e) {
        // What we wanted
    }
    assertEquals(0, map.size());
}

public void testTall() throws IOException {
  Directory indexStore = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), indexStore);
  add("blueberry chocolate pie", writer);
  add("blueberry chocolate tart", writer);
  IndexReader r = writer.getReader();
  writer.close();

  IndexSearcher searcher = newSearcher(r);
  MultiPhraseQuery q = new MultiPhraseQuery();
  q.add(new Term("body", "blueberry"));
  q.add(new Term("body", "chocolate"));
  q.add(new Term[] {new Term("body", "pie"), new Term("body", "tart")});
  assertEquals(2, searcher.search(q, 1).totalHits);
  r.close();
  indexStore.close();
}

@Ignore //LUCENE-3821 fixes sloppy phrase scoring, except for this known problem 
public void testMultiSloppyWithRepeats() throws IOException {
  Directory indexStore = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), indexStore);
  add("a b c d e f g h i k", writer);
  IndexReader r = writer.getReader();
  writer.close();

  IndexSearcher searcher = newSearcher(r);

  MultiPhraseQuery q = new MultiPhraseQuery();
  // this will fail, when the scorer would propagate [a] rather than [a,b],
  q.add(new Term[] {new Term("body", "a"), new Term("body", "b")});
  q.add(new Term[] {new Term("body", "a")});
  q.setSlop(6);
  assertEquals(1, searcher.search(q, 1).totalHits); // should match on "a b"

  r.close();
  indexStore.close();
}

@Override
public void setUp() throws Exception {
  super.setUp();
  directory = newDirectory();

  RandomIndexWriter writer = new RandomIndexWriter(random(), directory, 
      newIndexWriterConfig(new MockAnalyzer(random()))
      .setMergePolicy(newLogMergePolicy())
      .setSimilarity(new DefaultSimilarity()));
  for (int i = 0; i < values.length; i++) {
    Document doc = new Document();
    doc
        .add(newTextField(FIELD, values[i], Field.Store.YES));
    writer.addDocument(doc);
  }
  indexReader = SlowCompositeReaderWrapper.wrap(writer.getReader());
  writer.close();
  indexSearcher = newSearcher(indexReader);
  indexSearcher.setSimilarity(new DefaultSimilarity());
}

/** Make sure BooleanScorer can embed another
 *  BooleanScorer. */
public void testEmbeddedBooleanScorer() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter w = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  doc.add(newTextField("field", "doctors are people who prescribe medicines of which they know little, to cure diseases of which they know less, in human beings of whom they know nothing", Field.Store.NO));
  w.addDocument(doc);
  IndexReader r = w.getReader();
  w.close();

  IndexSearcher s = newSearcher(r);
  BooleanQuery q1 = new BooleanQuery();
  q1.add(new TermQuery(new Term("field", "little")), BooleanClause.Occur.SHOULD);
  q1.add(new TermQuery(new Term("field", "diseases")), BooleanClause.Occur.SHOULD);

  BooleanQuery q2 = new BooleanQuery();
  q2.add(q1, BooleanClause.Occur.SHOULD);
  q2.add(new CrazyMustUseBulkScorerQuery(), BooleanClause.Occur.SHOULD);

  assertEquals(1, s.search(q2, 10).totalHits);
  r.close();
  dir.close();
}

public void testPositionIncrementMultiFields() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  writer.addDocument(makeDocumentWithFields());
  IndexReader reader = writer.getReader();

  IndexSearcher searcher = newSearcher(reader);
  PhraseQuery query = new PhraseQuery();
  query.add(new Term("indexed_not_tokenized", "test1"));
  query.add(new Term("indexed_not_tokenized", "test2"));

  ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
  assertEquals(1, hits.length);

  doAssert(searcher.doc(hits[0].doc), true);
  writer.close();
  reader.close();
  dir.close();    
}

public void testReuseDocsEnumNoReuse() throws IOException {
  Directory dir = newDirectory();
  Codec cp = TestUtil.alwaysPostingsFormat(new Lucene40RWPostingsFormat());
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir,
      newIndexWriterConfig(new MockAnalyzer(random())).setCodec(cp));
  int numdocs = atLeast(20);
  createRandomIndex(numdocs, writer, random());
  writer.commit();

  DirectoryReader open = DirectoryReader.open(dir);
  for (AtomicReaderContext ctx : open.leaves()) {
    AtomicReader indexReader = ctx.reader();
    Terms terms = indexReader.terms("body");
    TermsEnum iterator = terms.iterator(null);
    IdentityHashMap<DocsEnum, Boolean> enums = new IdentityHashMap<>();
    MatchNoBits bits = new Bits.MatchNoBits(indexReader.maxDoc());
    while ((iterator.next()) != null) {
      DocsEnum docs = iterator.docs(random().nextBoolean() ? bits : new Bits.MatchNoBits(indexReader.maxDoc()), null, random().nextBoolean() ? DocsEnum.FLAG_FREQS : DocsEnum.FLAG_NONE);
      enums.put(docs, true);
    }

    assertEquals(terms.size(), enums.size());
  }
  IOUtils.close(writer, open, dir);
}

public void testSpanNot() throws Exception {
  SpanQuery[] clauses = new SpanQuery[2];
  clauses[0] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "one"));
  clauses[1] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "three"));
  SpanQuery spq = new SpanNearQuery(clauses, 5, true);
  SpanNotQuery snq = new SpanNotQuery(spq, new SpanTermQuery(new Term(PayloadHelper.FIELD, "two")));



  Directory directory = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), directory,
                                                   newIndexWriterConfig(new PayloadAnalyzer()).setSimilarity(similarity));

  Document doc = new Document();
  doc.add(newTextField(PayloadHelper.FIELD, "one two three one four three", Field.Store.YES));
  writer.addDocument(doc);
  IndexReader reader = writer.getReader();
  writer.close();


  checkSpans(MultiSpansWrapper.wrap(reader.getContext(), snq), 1,new int[]{2});
  reader.close();
  directory.close();
}

/** similar to the above, but ORs the query with a real field */
public void testEmptyField() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  doc.add(newTextField("foo", "bar", Field.Store.NO));
  iw.addDocument(doc);
  IndexReader ir = iw.getReader();
  iw.close();
  IndexSearcher is = newSearcher(ir);

  for (Similarity sim : sims) {
    is.setSimilarity(sim);
    BooleanQuery query = new BooleanQuery(true);
    query.add(new TermQuery(new Term("foo", "bar")), BooleanClause.Occur.SHOULD);
    query.add(new TermQuery(new Term("bar", "baz")), BooleanClause.Occur.SHOULD);
    assertEquals(1, is.search(query, 10).totalHits);
  }
  ir.close();
  dir.close();
}

@BeforeClass
public static void beforeClass() throws Exception {
  dir = newDirectory();
  IndexWriterConfig iwConfig = newIndexWriterConfig(new MockAnalyzer(random()));
  iwConfig.setMergePolicy(newLogMergePolicy());
  iwConfig.setSimilarity(sim);
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConfig);

  Document doc = new Document();
  doc.add(new TextField("text", "this is a test test test", Field.Store.NO));
  iw.addDocument(doc);

  doc = new Document();
  doc.add(new TextField("text", "second test", Field.Store.NO));
  iw.addDocument(doc);

  reader = iw.getReader();
  searcher = newSearcher(reader);
  iw.close();
}

/** make sure all sims work if TF and norms is omitted */
public void testOmitTFAndNorms() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
  ft.setIndexOptions(IndexOptions.DOCS_ONLY);
  ft.setOmitNorms(true);
  ft.freeze();
  Field f = newField("foo", "bar", ft);
  doc.add(f);
  iw.addDocument(doc);
  IndexReader ir = iw.getReader();
  iw.close();
  IndexSearcher is = newSearcher(ir);

  for (Similarity sim : sims) {
    is.setSimilarity(sim);
    BooleanQuery query = new BooleanQuery(true);
    query.add(new TermQuery(new Term("foo", "bar")), BooleanClause.Occur.SHOULD);
    assertEquals(1, is.search(query, 10).totalHits);
  }
  ir.close();
  dir.close();
}

@Override
public void setUp() throws Exception {
  super.setUp();
  directory = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), directory);

  // Add series of docs with specific information for MoreLikeThis
  addDoc(writer, "lucene");
  addDoc(writer, "lucene release");
  addDoc(writer, "apache");
  addDoc(writer, "apache lucene");

  reader = writer.getReader();
  writer.close();
  searcher = newSearcher(reader);
}

public void testDetectHierarchicalField() throws Exception {
  Directory dir = newDirectory();
  Directory taxoDir = newDirectory();
  TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode.CREATE);
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  FacetsConfig config = new FacetsConfig();

  Document doc = new Document();
  doc.add(newTextField("field", "text", Field.Store.NO));
  doc.add(new FacetField("a", "path", "other"));
  try {
    config.build(taxoWriter, doc);
    fail("did not hit expected exception");
  } catch (IllegalArgumentException iae) {
    // expected
  }
  IOUtils.close(writer, taxoWriter, dir, taxoDir);
}

@Override
public void setUp() throws Exception {
  super.setUp();
  dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
  int numDocs = TestUtil.nextInt(random(), 2049, 4000);
  for (int i = 0; i < numDocs; i++) {
    Document document = new Document();
    document.add(newTextField("english", English.intToEnglish(i), Field.Store.NO));
    document.add(newTextField("oddeven", (i % 2 == 0) ? "even" : "odd", Field.Store.NO));
    document.add(newStringField("byte", "" + ((byte) random().nextInt()), Field.Store.NO));
    document.add(newStringField("short", "" + ((short) random().nextInt()), Field.Store.NO));
    document.add(new IntField("int", random().nextInt(), Field.Store.NO));
    document.add(new LongField("long", random().nextLong(), Field.Store.NO));

    document.add(new FloatField("float", random().nextFloat(), Field.Store.NO));
    document.add(new DoubleField("double", random().nextDouble(), Field.Store.NO));

    document.add(new NumericDocValuesField("intdocvalues", random().nextInt()));
    document.add(new FloatDocValuesField("floatdocvalues", random().nextFloat()));
    iw.addDocument(document);
  }
  reader = iw.getReader();
  iw.close();
  searcher = newSearcher(reader);
}

public void testCustomIDF() throws Exception {
  Directory indexStore = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), indexStore);
  add("This is a test", "object", writer);
  add("a note", "note", writer);

  IndexReader reader = writer.getReader();
  IndexSearcher searcher = newSearcher(reader);
  searcher.setSimilarity(new DefaultSimilarity() { 
    @Override
    public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics termStats[]) {
      return new Explanation(10f, "just a test");
    } 
  });

  MultiPhraseQuery query = new MultiPhraseQuery();
  query.add(new Term[] { new Term("body", "this"), new Term("body", "that") });
  query.add(new Term("body", "is"));
  Weight weight = query.createWeight(searcher);
  assertEquals(10f * 10f, weight.getValueForNormalization(), 0.001f);

  writer.close();
  reader.close();
  indexStore.close();
}

/** Tests sorting on type string_val */
public void testStringVal() throws IOException {
  Directory dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  doc.add(newStringField("value", "foo", Field.Store.YES));
  writer.addDocument(doc);
  doc = new Document();
  doc.add(newStringField("value", "bar", Field.Store.YES));
  writer.addDocument(doc);
  IndexReader ir = writer.getReader();
  writer.close();

  IndexSearcher searcher = newSearcher(ir);
  Sort sort = new Sort(new SortField("value", SortField.Type.STRING_VAL));

  TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
  assertEquals(2, td.totalHits);
  // 'bar' comes before 'foo'
  assertEquals("bar", searcher.doc(td.scoreDocs[0].doc).get("value"));
  assertEquals("foo", searcher.doc(td.scoreDocs[1].doc).get("value"));

  ir.close();
  dir.close();
}

/** Tests sorting on type string */
public void testString() throws IOException {
  Directory dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  doc.add(newStringField("value", "foo", Field.Store.YES));
  writer.addDocument(doc);
  doc = new Document();
  doc.add(newStringField("value", "bar", Field.Store.YES));
  writer.addDocument(doc);
  IndexReader ir = writer.getReader();
  writer.close();

  IndexSearcher searcher = newSearcher(ir);
  Sort sort = new Sort(new SortField("value", SortField.Type.STRING));

  TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
  assertEquals(2, td.totalHits);
  // 'bar' comes before 'foo'
  assertEquals("bar", searcher.doc(td.scoreDocs[0].doc).get("value"));
  assertEquals("foo", searcher.doc(td.scoreDocs[1].doc).get("value"));

  ir.close();
  dir.close();
}

public void testTransposition2() throws Exception {
  DirectSpellChecker spellChecker = new DirectSpellChecker();
  Directory dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, 
      new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));

  for (int i = 0; i < 20; i++) {
    Document doc = new Document();
    doc.add(newTextField("numbers", English.intToEnglish(i), Field.Store.NO));
    writer.addDocument(doc);
  }

  IndexReader ir = writer.getReader();

  SuggestWord[] similar = spellChecker.suggestSimilar(new Term(
      "numbers", "seevntene"), 2, ir,
      SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX);
  assertEquals(1, similar.length);
  assertEquals("seventeen", similar[0].string);
  ir.close();
  writer.close();
  dir.close();
}

public void testMixedTypesInSameIndexField() throws Exception {
  Directory dir = newDirectory();
  Directory taxoDir = newDirectory();

  TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
  FacetsConfig config = new FacetsConfig();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);

  Document doc = new Document();
  doc.add(new IntAssociationFacetField(14, "a", "x"));
  doc.add(new FloatAssociationFacetField(55.0f, "b", "y"));
  try {
    writer.addDocument(config.build(taxoWriter, doc));
    fail("did not hit expected exception");
  } catch (IllegalArgumentException exc) {
    // expected
  }
  IOUtils.close(writer, taxoWriter, dir, taxoDir);
}

public void testInvalidPayload() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
  iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat()));
  RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
  Document doc = new Document();
  doc.add(new StringAndPayloadField("id", "id", new BytesRef("foo")));
  try {
    w.addDocument(doc);
    w.commit();
    fail("didn't hit expected exception");
  } catch (IllegalArgumentException iae) {
    // expected
  }

  w.close();
  dir.close();
}

public void testEvilSearcherFactory() throws Exception {
  final Directory dir = newDirectory();
  final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
  w.commit();

  final IndexReader other = DirectoryReader.open(dir);

  final SearcherFactory theEvilOne = new SearcherFactory() {
    @Override
    public IndexSearcher newSearcher(IndexReader ignored) {
      return LuceneTestCase.newSearcher(other);
    }
    };

  try {
    new SearcherManager(w.w, false, theEvilOne);
    fail("didn't hit expected exception");
  } catch (IllegalStateException ise) {
    // expected
  }
  w.close();
  other.close();
  dir.close();
}