Java 类org.apache.lucene.index.IndexWriter 实例源码

static void indexDoc(IndexWriter writer, Path file, long lastModified) throws IOException {
    try (InputStream stream = Files.newInputStream(file)) {
        Document doc = new Document();
        Field pathField = new StringField("path", file.toString(), Field.Store.YES);
        doc.add(pathField);
        doc.add(new LongPoint("modified", lastModified));
        doc.add(new TextField("contents", new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))));

        if (writer.getConfig().getOpenMode() == OpenMode.CREATE) {
            System.out.println("adding " + file);
            writer.addDocument(doc);
        } else {
            System.out.println("updating " + file);
            writer.updateDocument(new Term("path", file.toString()), doc);
        }
    }
}

public void testMMapDirectory() throws IOException {
    long start = System.currentTimeMillis();
    IndexWriterConfig indexWriterConfig = new IndexWriterConfig(new WhitespaceAnalyzer()).setOpenMode(IndexWriterConfig
            .OpenMode.CREATE);
    FSDirectory open = FSDirectory.open(Paths.get("E:/testlucene"));
    IndexWriter indexWriter = new IndexWriter(open, indexWriterConfig);
    for (int i = 0; i < 10000000; i++) {
        indexWriter.addDocument(addDocument(i));
    }
    indexWriter.commit();
    indexWriter.close();
    long end = System.currentTimeMillis();
    log.error("MMapDirectory consumes {}s!", (end - start) / 1000);
    start = System.currentTimeMillis();
    IndexSearcher indexSearcher = new IndexSearcher(DirectoryReader.open(open));
    int total = 0;
    for (int i = 0; i < 10000000; i++) {
        TermQuery key1 = new TermQuery(new Term("key1", "key" + i));
        TopDocs search = indexSearcher.search(key1, 10);
        total += search.totalHits;
    }
    System.out.println(total);
    end = System.currentTimeMillis();
    log.error("MMapDirectory search consumes {}ms!", (end - start));
}

public void update() throws CorruptIndexException, IOException, ParserConfigurationException, SAXException {
    try (IndexWriter indexWriter = openIndexWriter(indexDir)) {
        SAXParser parser = createParser();
        PubMedIndexDOMBuilderHandler handler = new PubMedIndexDOMBuilderHandler(XMLUtils.docBuilder, indexWriter, meshPaths, openLicenses);
        PubmedIndexProperties properties = new PubmedIndexProperties(indexWriter);
        fileFilter.properties = properties;
        SourceStream source = new CollectionSourceStream("UTF-8", sources);
        for (InputStream is : Iterators.loop(source.getInputStreams())) {
            String streamName = source.getStreamName(is);
            String filename = getFilename(streamName);
            PubMedIndexUtils.log("parsing and indexing: %s", filename);
            handler.resetCounts();
            handler.setSource(filename);
            parser.parse(is, handler);
            properties.addIndexedFile(filename);
            properties.update(indexWriter);
            indexWriter.commit();
            PubMedIndexUtils.log("citations updated: %d", handler.getUpdatedCitationsCount());
            PubMedIndexUtils.log("citations deleted: %d", handler.getDeletedCitationsCount());
        }
    }
}

/**
 * Acquires, then releases, all {@code write.lock} files in the given
 * shard paths. The "write.lock" file is assumed to be under the shard
 * path's "index" directory as used by Elasticsearch.
 *
 * @throws LockObtainFailedException if any of the locks could not be acquired
 */
public static void acquireFSLockForPaths(Settings indexSettings, Path... shardPaths) throws IOException {
    Lock[] locks = new Lock[shardPaths.length];
    Directory[] dirs = new Directory[shardPaths.length];
    try {
        for (int i = 0; i < shardPaths.length; i++) {
            // resolve the directory the shard actually lives in
            Path p = shardPaths[i].resolve("index");
            // open a directory (will be immediately closed) on the shard's location
            dirs[i] = new SimpleFSDirectory(p, FsDirectoryService.buildLockFactory(indexSettings));
            // create a lock for the "write.lock" file
            try {
                locks[i] = dirs[i].obtainLock(IndexWriter.WRITE_LOCK_NAME);
            } catch (IOException ex) {
                throw new LockObtainFailedException("unable to acquire " +
                        IndexWriter.WRITE_LOCK_NAME + " for " + p);
            }
        }
    } finally {
        IOUtils.closeWhileHandlingException(locks);
        IOUtils.closeWhileHandlingException(dirs);
    }
}

public void testRamDirectory() throws IOException {
    long start = System.currentTimeMillis();
    IndexWriterConfig indexWriterConfig = new IndexWriterConfig(new WhitespaceAnalyzer()).setOpenMode(IndexWriterConfig
            .OpenMode.CREATE);
    RAMDirectory ramDirectory = new RAMDirectory();
    IndexWriter indexWriter = new IndexWriter(ramDirectory, indexWriterConfig);
    for (int i = 0; i < 10000000; i++) {
        indexWriter.addDocument(addDocument(i));
    }
    indexWriter.commit();
    indexWriter.close();
    long end = System.currentTimeMillis();
    log.error("RamDirectory consumes {}s!", (end - start) / 1000);
    start = System.currentTimeMillis();
    IndexSearcher indexSearcher = new IndexSearcher(DirectoryReader.open(ramDirectory));
    int total = 0;
    for (int i = 0; i < 10000000; i++) {
        TermQuery key1 = new TermQuery(new Term("key1", "key" + i));
        TopDocs search = indexSearcher.search(key1, 10);
        total += search.totalHits;
    }
    System.out.println(total);
    end = System.currentTimeMillis();
    log.error("RamDirectory search consumes {}ms!", (end - start));
}

/**
 * gets a {@link Store.MetadataSnapshot} for the current directory. This method is safe to call in all lifecycle of the index shard,
 * without having to worry about the current state of the engine and concurrent flushes.
 *
 * @throws org.apache.lucene.index.IndexNotFoundException     if no index is found in the current directory
 * @throws CorruptIndexException      if the lucene index is corrupted. This can be caused by a checksum mismatch or an
 *                                    unexpected exception when opening the index reading the segments file.
 * @throws IndexFormatTooOldException if the lucene index is too old to be opened.
 * @throws IndexFormatTooNewException if the lucene index is too new to be opened.
 * @throws FileNotFoundException      if one or more files referenced by a commit are not present.
 * @throws NoSuchFileException        if one or more files referenced by a commit are not present.
 */
public Store.MetadataSnapshot snapshotStoreMetadata() throws IOException {
    IndexCommit indexCommit = null;
    store.incRef();
    try {
        synchronized (mutex) {
            // if the engine is not running, we can access the store directly, but we need to make sure no one starts
            // the engine on us. If the engine is running, we can get a snapshot via the deletion policy which is initialized.
            // That can be done out of mutex, since the engine can be closed half way.
            Engine engine = getEngineOrNull();
            if (engine == null) {
                try (Lock ignored = store.directory().obtainLock(IndexWriter.WRITE_LOCK_NAME)) {
                    return store.getMetadata(null);
                }
            }
        }
        indexCommit = deletionPolicy.snapshot();
        return store.getMetadata(indexCommit);
    } finally {
        store.decRef();
        if (indexCommit != null) {
            deletionPolicy.release(indexCommit);
        }
    }
}

static void indexDocs(final IndexWriter writer, Path path) throws IOException {
    if (Files.isDirectory(path)) {
        Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
            @Override
            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
                try {
                    indexDoc(writer, file, attrs.lastModifiedTime().toMillis());
                } catch (IOException ignore) {
                    // don't index files that can't be read.
                }
                return FileVisitResult.CONTINUE;
            }
        });
    } else {
        indexDoc(writer, path, Files.getLastModifiedTime(path).toMillis());
    }
}

/**
 * This method removes all lucene files from the given directory. It will first try to delete all commit points / segments
 * files to ensure broken commits or corrupted indices will not be opened in the future. If any of the segment files can't be deleted
 * this operation fails.
 */
public static void cleanLuceneIndex(Directory directory) throws IOException {
    try (Lock writeLock = directory.obtainLock(IndexWriter.WRITE_LOCK_NAME)) {
        for (final String file : directory.listAll()) {
            if (file.startsWith(IndexFileNames.SEGMENTS) || file.equals(IndexFileNames.OLD_SEGMENTS_GEN)) {
                directory.deleteFile(file); // remove all segment_N files
            }
        }
    }
    try (IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)
            .setMergePolicy(NoMergePolicy.INSTANCE) // no merges
            .setCommitOnClose(false) // no commits
            .setOpenMode(IndexWriterConfig.OpenMode.CREATE))) // force creation - don't append...
    {
        // do nothing and close this will kick of IndexFileDeleter which will remove all pending files
    }
}

private Object randomOfType(SortField.Type type) {
    switch (type) {
    case CUSTOM:
        throw new UnsupportedOperationException();
    case DOC:
        return between(0, IndexWriter.MAX_DOCS);
    case DOUBLE:
        return randomDouble();
    case FLOAT:
        return randomFloat();
    case INT:
        return randomInt();
    case LONG:
        return randomLong();
    case REWRITEABLE:
        throw new UnsupportedOperationException();
    case SCORE:
        return randomFloat();
    case STRING:
        return new BytesRef(randomAsciiOfLength(5));
    case STRING_VAL:
        return new BytesRef(randomAsciiOfLength(5));
    default:
        throw new UnsupportedOperationException("Unkown SortField.Type: " + type);
    }
}

/**
 * Test the WordScorer emitted by the smoothing model
 */
public void testBuildWordScorer() throws IOException {
    SmoothingModel testModel = createTestModel();
    Map<String, Analyzer> mapping = new HashMap<>();
    mapping.put("field", new WhitespaceAnalyzer());
    PerFieldAnalyzerWrapper wrapper = new PerFieldAnalyzerWrapper(new WhitespaceAnalyzer(), mapping);
    IndexWriter writer = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(wrapper));
    Document doc = new Document();
    doc.add(new Field("field", "someText", TextField.TYPE_NOT_STORED));
    writer.addDocument(doc);
    DirectoryReader ir = DirectoryReader.open(writer);

    WordScorer wordScorer = testModel.buildWordScorerFactory().newScorer(ir, MultiFields.getTerms(ir, "field"), "field", 0.9d,
            BytesRefs.toBytesRef(" "));
    assertWordScorer(wordScorer, testModel);
}

public void testGetParentIdNoParentField() throws Exception {
    ParentFieldMapper fieldMapper = createParentFieldMapper();
    Directory directory = newDirectory();
    IndexWriter indexWriter = new IndexWriter(directory, newIndexWriterConfig());
    Document document = new Document();
    document.add(new SortedDocValuesField("different_field", new BytesRef("1")));
    indexWriter.addDocument(document);
    indexWriter.close();

    IndexReader indexReader = DirectoryReader.open(directory);
    String id = ParentFieldSubFetchPhase.getParentId(fieldMapper, indexReader.leaves().get(0).reader(), 0);
    assertNull(id);

    indexReader.close();
    directory.close();
}

public void testSingleValued() throws IOException {
    Directory dir = newDirectory();
    // we need the default codec to check for singletons
    IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null).setCodec(TestUtil.getDefaultCodec()));
    Document doc = new Document();
    for (IndexableField f : NumberFieldMapper.NumberType.HALF_FLOAT.createFields("half_float", 3f, false, true, false)) {
        doc.add(f);
    }
    w.addDocument(doc);
    final DirectoryReader dirReader = DirectoryReader.open(w);
    LeafReader reader = getOnlyLeafReader(dirReader);
    SortedNumericDoubleValues values = new SortedNumericDVIndexFieldData.SortedNumericHalfFloatFieldData(
            reader, "half_float").getDoubleValues();
    assertNotNull(FieldData.unwrapSingleton(values));
    values.setDocument(0);
    assertEquals(1, values.count());
    assertEquals(3f, values.valueAt(0), 0f);
    IOUtils.close(dirReader, w, dir);
}

public static InternalEngine createInternalEngine(@Nullable final IndexWriterFactory indexWriterFactory,
                                                  @Nullable final Supplier<SequenceNumbersService> sequenceNumbersServiceSupplier,
                                                  final EngineConfig config) {
    return new InternalEngine(config) {
            @Override
            IndexWriter createWriter(Directory directory, IndexWriterConfig iwc) throws IOException {
                return (indexWriterFactory != null) ?
                    indexWriterFactory.createWriter(directory, iwc) :
                    super.createWriter(directory, iwc);
            }

            @Override
            public SequenceNumbersService seqNoService() {
                return (sequenceNumbersServiceSupplier != null) ? sequenceNumbersServiceSupplier.get() : super.seqNoService();
            }
        };
}

/**
 * Main entry point. 
 * 
 * @param args the command line arguments.
 * @throws IOException in case of I/O failure.
 * @throws ParseException in case of Query parse exception.
 */
public static void main(String[] args) throws IOException, ParseException {
    // 1. Creates a directory reference. This is where index datafiles will be created.
    Directory directory = FSDirectory.open(new File("/tmp").toPath());

    // 2. Creates an IndexWriter
    try (IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig())) {

        // 3. Add some data
        indexSomeData(writer);

        // 4. Search
        search(directory);          

        writer.deleteAll();
    } 
}

@Override
public void process(ProcessingContext<Corpus> ctx, Corpus corpus) throws ModuleException {
    try (KeywordAnalyzer kwa = new KeywordAnalyzer()) {
        IndexWriterConfig writerConfig = new IndexWriterConfig(Version.LUCENE_36, kwa);
        writerConfig.setOpenMode(append ? OpenMode.CREATE_OR_APPEND : OpenMode.CREATE);
        try (Directory dir = FSDirectory.open(indexDir)) {
            try (IndexWriter writer = new IndexWriter(dir, writerConfig)) {
                AlvisDBIndexerResolvedObjects resObj = getResolvedObjects();
                Logger logger = getLogger(ctx);
                EvaluationContext evalCtx = new EvaluationContext(logger);
                for (ADBElements.Resolved ent : resObj.elements) {
                    ent.indexElements(logger, writer, evalCtx, corpus);
                }
            }
        }
        catch (IOException e) {
            rethrow(e);
        }
    }
}

public void testCanOpenIndex() throws IOException {
    final ShardId shardId = new ShardId("index", "_na_", 1);
    IndexWriterConfig iwc = newIndexWriterConfig();
    Path tempDir = createTempDir();
    final BaseDirectoryWrapper dir = newFSDirectory(tempDir);
    assertFalse(Store.canOpenIndex(logger, tempDir, shardId, (id, l) -> new DummyShardLock(id)));
    IndexWriter writer = new IndexWriter(dir, iwc);
    Document doc = new Document();
    doc.add(new StringField("id", "1", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
    writer.addDocument(doc);
    writer.commit();
    writer.close();
    assertTrue(Store.canOpenIndex(logger, tempDir, shardId, (id, l) -> new DummyShardLock(id)));

    DirectoryService directoryService = new DirectoryService(shardId, INDEX_SETTINGS) {

        @Override
        public Directory newDirectory() throws IOException {
            return dir;
        }
    };
    Store store = new Store(shardId, INDEX_SETTINGS, directoryService, new DummyShardLock(shardId));
    store.markStoreCorrupted(new CorruptIndexException("foo", "bar"));
    assertFalse(Store.canOpenIndex(logger, tempDir, shardId, (id, l) -> new DummyShardLock(id)));
    store.close();
}

public void testVectorHighlighter() throws Exception {
    Directory dir = new RAMDirectory();
    IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));

    Document document = new Document();
    document.add(new TextField("_id", "1", Field.Store.YES));
    FieldType vectorsType = new FieldType(TextField.TYPE_STORED);
    vectorsType.setStoreTermVectors(true);
    vectorsType.setStoreTermVectorPositions(true);
    vectorsType.setStoreTermVectorOffsets(true);
    document.add(new Field("content", "the big bad dog", vectorsType));
    indexWriter.addDocument(document);

    IndexReader reader = DirectoryReader.open(indexWriter);
    IndexSearcher searcher = new IndexSearcher(reader);
    TopDocs topDocs = searcher.search(new TermQuery(new Term("_id", "1")), 1);

    assertThat(topDocs.totalHits, equalTo(1));

    FastVectorHighlighter highlighter = new FastVectorHighlighter();
    String fragment = highlighter.getBestFragment(highlighter.getFieldQuery(new TermQuery(new Term("content", "bad"))),
            reader, topDocs.scoreDocs[0].doc, "content", 30);
    assertThat(fragment, notNullValue());
    assertThat(fragment, equalTo("the big <b>bad</b> dog"));
}

/**
 * Index all text files under a directory.
 */
public void main(final File INDEX_DIR, final String[] args) throws IOException {
  IndexWriter writer = new IndexWriter(INDEX_DIR, new StandardAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
  for (int arg = 0; arg < args.length; arg++) {
    final File docDir = new File(args[arg]);
    if (!docDir.exists() || !docDir.canRead()) {
      System.out.println("Document directory '" + docDir.getAbsolutePath() + "' does not exist or is not readable, please check the path");
      throw new IOException("Cannot read from document directory");
    }

    indexDocs(writer, docDir);
    System.out.println("Optimizing...");
    writer.optimize();
  }
  writer.close();
}

public void testSimpleNumericOps() throws Exception {
    Directory dir = new RAMDirectory();
    IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));

    Document document = new Document();
    document.add(new TextField("_id", "1", Field.Store.YES));
    document.add(new LegacyIntField("test", 2, LegacyIntField.TYPE_STORED));
    indexWriter.addDocument(document);

    IndexReader reader = DirectoryReader.open(indexWriter);
    IndexSearcher searcher = new IndexSearcher(reader);
    TopDocs topDocs = searcher.search(new TermQuery(new Term("_id", "1")), 1);
    Document doc = searcher.doc(topDocs.scoreDocs[0].doc);
    IndexableField f = doc.getField("test");
    assertThat(f.stringValue(), equalTo("2"));

    BytesRefBuilder bytes = new BytesRefBuilder();
    LegacyNumericUtils.intToPrefixCoded(2, 0, bytes);
    topDocs = searcher.search(new TermQuery(new Term("test", bytes.get())), 1);
    doc = searcher.doc(topDocs.scoreDocs[0].doc);
    f = doc.getField("test");
    assertThat(f.stringValue(), equalTo("2"));

    indexWriter.close();
}

/** Test that version map cache behaves properly with a filtered reader */
public void testCacheFilterReader() throws Exception {
    int size = Versions.lookupStates.size();

    Directory dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));
    Document doc = new Document();
    doc.add(new Field(UidFieldMapper.NAME, "6", UidFieldMapper.Defaults.FIELD_TYPE));
    doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 87));
    writer.addDocument(doc);
    DirectoryReader reader = DirectoryReader.open(writer);
    assertEquals(87, Versions.loadVersion(reader, new Term(UidFieldMapper.NAME, "6")));
    assertEquals(size+1, Versions.lookupStates.size());
    // now wrap the reader
    DirectoryReader wrapped = ElasticsearchDirectoryReader.wrap(reader, new ShardId("bogus", "_na_", 5));
    assertEquals(87, Versions.loadVersion(wrapped, new Term(UidFieldMapper.NAME, "6")));
    // same size map: core cache key is shared
    assertEquals(size+1, Versions.lookupStates.size());

    reader.close();
    writer.close();
    // core should be evicted from the map
    assertEquals(size, Versions.lookupStates.size());
    dir.close();
}

public void testNoTokens() throws Exception {
    Directory dir = new RAMDirectory();
    IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.KEYWORD_ANALYZER));

    FieldType allFt = getAllFieldType();
    Document doc = new Document();
    doc.add(new Field("_id", "1", StoredField.TYPE));
    doc.add(new AllField("_all", "", 2.0f, allFt));
    indexWriter.addDocument(doc);

    IndexReader reader = DirectoryReader.open(indexWriter);
    IndexSearcher searcher = new IndexSearcher(reader);

    TopDocs docs = searcher.search(new MatchAllDocsQuery(), 10);
    assertThat(docs.totalHits, equalTo(1));
    assertThat(docs.scoreDocs[0].doc, equalTo(0));
}

/**
 * Manage getting a lucene index writer for transactional data - looks after registration and checking there is no
 * active reader.
 * 
 * @param id String
 * @param analyzer Analyzer
 * @return IndexWriter
 * @throws IOException
 */
public IndexWriter getDeltaIndexWriter(String id, Analyzer analyzer) throws IOException
{
    if (id == null)
    {
        throw new IndexerException("\"null\" is not a valid identifier for a transaction");
    }

    // No read lock required as the delta should be bound to one thread only
    IndexWriter writer = indexWriters.get(id);
    if (writer == null)
    {
        // close index writer if required
        closeDeltaIndexReader(id);
        File location = ensureDeltaIsRegistered(id);
        writer = makeDeltaIndexWriter(location, analyzer);
        indexWriters.put(id, writer);
    }
    return writer;
}

static void indexDocs(final IndexWriter writer, Path path) throws IOException {
    if (Files.isDirectory(path)) {
        Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
            @Override
            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
                try {
                    indexDoc(writer, file, attrs.lastModifiedTime().toMillis());
                } catch (IOException ignore) {
                }
                return FileVisitResult.CONTINUE;
            }
        }
                );
    } else {
        indexDoc(writer, path, Files.getLastModifiedTime(path).toMillis());
    }
}

public void addIndex(UUser user) throws Exception {
  IndexWriter writer = getWriter();
  Document doc = new Document();
  /*
 * yes是会将数据存进索引，如果查询结果中需要将记录显示出来就要存进去，如果查询结果
 * 只是显示标题之类的就可以不用存，而且内容过长不建议存进去
 * 使用TextField类是可以用于查询的。
 */
  try {
    doc.add(new StringField("userid", String.valueOf(user.getId()), Field.Store.YES));
    doc.add(new TextField("username", user.getUsername(), Field.Store.YES));

    writer.addDocument(doc);
  } catch (Exception e) {
    e.printStackTrace();
    throw e;
  } finally {
    writer.close();
  }
}

private static void indexDoc(IndexWriter writer, FileBean t) throws Exception {
        Document doc = new Document();
        if (t.getContent() != null) {
            doc.add(new TextField(LuceneConstants.PATH, t.getFilepath(), Field.Store.YES));
            doc.add(new StringField(LuceneConstants.MODIFIED, UtilsTool.getDateStrByLastModified(t.getLastModified()), Field.Store.YES));
            doc.add(new TextField(LuceneConstants.CONTENT, t.getContent(), CommonConstants.IS_OPEN_CONTEXT ? Field.Store.YES : Field.Store.NO));
//            System.out.println("added to document:" + t.getFilepath());
            if (writer.getConfig().getOpenMode() == IndexWriterConfig.OpenMode.CREATE){
                writer.addDocument(doc);
            } else{
                writer.updateDocument(new Term(LuceneConstants.PATH, t.getFilepath()), doc);
            }
        }
    }

private static void mergeTickets(String firstId, String secondId) throws IOException, ParseException, IndexException {

    if (firstId == null || secondId == null) {
        throw new IllegalArgumentException(ERROR_NULL_ARGUMENT);
    }

    Directory directory = IndexProvider.getInstance().getDirectory();
    try (IndexReader indexReader = DirectoryReader.open(directory);) {
        IndexSearcher searcher = new IndexSearcher(indexReader);

        BooleanQuery firstQuery = getQuery(firstId);
        BooleanQuery secondQuery = getQuery(secondId);

        ScoreDoc[] topHitsDocsIdFirst = getTopHitsDoc(firstId, firstQuery, searcher);
        ScoreDoc[] topHitsDocsIdSecond = getTopHitsDoc(secondId, secondQuery, searcher);

        if (topHitsDocsIdFirst[0].doc == topHitsDocsIdSecond[0].doc) {
            String message = MessageFormat.format(WARN_DOCUMENTS_ALREADY_MERGED, firstId, secondId);
            LOGGER.warn(message);
            return;
        }

        Document newDocument = mergeDocuments(searcher, topHitsDocsIdFirst, topHitsDocsIdSecond);

        IndexWriterConfig config = new IndexWriterConfig(analyzer);
        try (IndexWriter indexWriter = new IndexWriter(directory, config);) {
            indexWriter.deleteDocuments(firstQuery);
            indexWriter.deleteDocuments(secondQuery);
            indexWriter.addDocument(newDocument);
        }
    }       
}

void indexElements(Logger logger, IndexWriter writer, EvaluationContext ctx, Corpus corpus) throws IOException {
    for (Element item : Iterators.loop(items.evaluateElements(ctx, corpus))) {
        Document doc = createDocument(logger, ctx, item);
        if (doc != null) {
            writer.addDocument(doc);
        }
    }
}

private static void indexJavaDocAllClasses(IndexWriter writer, File javaDocRoot)  throws Exception {
        StringBuilder builder = new StringBuilder();
        BufferedReader reader = new BufferedReader(new FileReader(new File(javaDocRoot,"allclasses-noframe.html")));
        String line;
        while((line = reader.readLine()) != null) {
            builder.append(line);
            builder.append('\n');
        }
        reader.close();
        // parse package
        Matcher matcher = findClassUrl.matcher(builder);
        while (matcher.find()) {
            String classUrl = matcher.group(1);
            ///System.out.println("\n\nclassUrl = " + classUrl);
            indexDocs(writer, new File(javaDocRoot, classUrl));
        }

        /*
SampleInfo{
    sourceFileUrl='file:/Volumes/Store/Projects/presidio/jfx/apps/internal/Ensemble/src/ensemble/samples/animation/timelines/InterpolatorSample.java',
    packageName='ensemble.samples.animation.timelines',
    className='ensemble.samples.animation.timelines.InterpolatorSample',
    description='This sample demostrates the interpolator property of the KeyValues: from default linear interpolation of the values between the KeyFrames to powerful Spline interpolator. There are 5 circles with adjustable opacity in the demo. Each circle moves with different interpolator, the first visibles are ones with default(linear) and easy_both interpolators. ',
    apiClasspaths=[javafx.animation.Interpolator, javafx.animation.KeyFrame, javafx.animation.KeyValue, javafx.animation.Timeline, javafx.util.Duration],
    relatesSamplePaths=[],
    resourceUrls=[]
}
         */
    }

/**
 * Create a new document and write it to the given writer
 *
 * @param writer       The writer to write out to
 * @param documentType The document type to save in the doc
 * @param fields       The searchable and data fields to write into doc
 * @throws IOException If there was problem writing doc
 */
private static void addDocument(IndexWriter writer, DocumentType documentType, Field... fields) throws IOException {
    // make a new, empty document
    Document doc = new Document();
    // add doc type field
    doc.add(new Field("documentType", documentType.toString(), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
    // add other fields
    if (fields != null) {
        for (Field field : fields) doc.add(field);
    }
    // write into index, assuming we are recreating every time
    writer.addDocument(doc);
}

public void testRedisDirectoryWithJedisPool() throws IOException {
    long start = System.currentTimeMillis();
    IndexWriterConfig indexWriterConfig = new IndexWriterConfig(new WhitespaceAnalyzer()).setOpenMode(IndexWriterConfig
            .OpenMode.CREATE);
    //indexWriterConfig.setInfoStream(System.out);
    //indexWriterConfig.setRAMBufferSizeMB(2048);
    //LogByteSizeMergePolicy logByteSizeMergePolicy = new LogByteSizeMergePolicy();
    //logByteSizeMergePolicy.setMinMergeMB(1);
    //logByteSizeMergePolicy.setMaxMergeMB(64);
    //logByteSizeMergePolicy.setMaxCFSSegmentSizeMB(64);
    //indexWriterConfig.setRAMBufferSizeMB(1024).setMergePolicy(logByteSizeMergePolicy).setUseCompoundFile(false);
    //GenericObjectPoolConfig genericObjectPoolConfig = new GenericObjectPoolConfig();
    //获取连接等待时间
    //genericObjectPoolConfig.setMaxWaitMillis(3000);
    //10s超时时间
    JedisPool jedisPool = new JedisPool(new JedisPoolConfig(), "localhost", 6379, Constants.TIME_OUT);
    RedisDirectory redisDirectory = new RedisDirectory(new JedisPoolStream(jedisPool));
    IndexWriter indexWriter = new IndexWriter(redisDirectory, indexWriterConfig);
    for (int i = 0; i < 10000000; i++) {
        indexWriter.addDocument(addDocument(i));
    }
    indexWriter.commit();
    indexWriter.close();
    redisDirectory.close();
    long end = System.currentTimeMillis();
    log.error("RedisDirectoryWithJedisPool consumes {}s!", (end - start) / 1000);
    start = System.currentTimeMillis();
    IndexSearcher indexSearcher = new IndexSearcher(DirectoryReader.open(new RedisDirectory(new JedisStream("localhost",
            6379))));
    int total = 0;
    for (int i = 0; i < 10000000; i++) {
        TermQuery key1 = new TermQuery(new Term("key1", "key" + i));
        TopDocs search = indexSearcher.search(key1, 10);
        total += search.totalHits;
    }
    System.out.println(total);
    end = System.currentTimeMillis();
    log.error("RedisDirectoryWithJedisPool search consumes {}ms!", (end - start));
}

public void testRedisDirectoryWithJedis() throws IOException {
    long start = System.currentTimeMillis();
    IndexWriterConfig indexWriterConfig = new IndexWriterConfig(new WhitespaceAnalyzer()).setOpenMode(IndexWriterConfig
            .OpenMode.CREATE);
    //indexWriterConfig.setInfoStream(System.out);
    //indexWriterConfig.setRAMBufferSizeMB(2048);
    //LogByteSizeMergePolicy logByteSizeMergePolicy = new LogByteSizeMergePolicy();
    //logByteSizeMergePolicy.setMinMergeMB(1);
    //logByteSizeMergePolicy.setMaxMergeMB(64);
    //logByteSizeMergePolicy.setMaxCFSSegmentSizeMB(64);
    //indexWriterConfig.setRAMBufferSizeMB(1024).setMergePolicy(logByteSizeMergePolicy).setUseCompoundFile(false);
    //GenericObjectPoolConfig genericObjectPoolConfig = new GenericObjectPoolConfig();
    //获取连接等待时间
    //genericObjectPoolConfig.setMaxWaitMillis(3000);
    //10s超时时间
    RedisDirectory redisDirectory = new RedisDirectory(new JedisStream("localhost", 6379));
    IndexWriter indexWriter = new IndexWriter(redisDirectory, indexWriterConfig);
    for (int i = 0; i < 10000000; i++) {
        indexWriter.addDocument(addDocument(i));
    }
    indexWriter.commit();
    indexWriter.close();
    redisDirectory.close();
    long end = System.currentTimeMillis();
    log.error("RedisDirectoryWithJedis consumes {}s!", (end - start) / 1000);
    start = System.currentTimeMillis();
    IndexSearcher indexSearcher = new IndexSearcher(DirectoryReader.open(new RedisDirectory(new JedisStream("localhost",
            6379))));
    int total = 0;
    for (int i = 0; i < 10000000; i++) {
        TermQuery key1 = new TermQuery(new Term("key1", "key" + i));
        TopDocs search = indexSearcher.search(key1, 10);
        total += search.totalHits;
    }
    System.out.println(total);
    end = System.currentTimeMillis();
    log.error("RedisDirectoryWithJedis search consumes {}ms!", (end - start));
}

@Before
public void init() throws Exception {
    directory = newDirectory();
    IndexWriterConfig config = new IndexWriterConfig(new WhitespaceAnalyzer());
    config.setMergePolicy(NoMergePolicy.INSTANCE);
    indexWriter = new IndexWriter(directory, config);
}

public LuceneWordSearch() {
    directory = new RAMDirectory();
    IndexWriterConfig writerConfig = new IndexWriterConfig();
    try {
        writer = new IndexWriter(directory, writerConfig);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

/**
 * 创建内存目录
 */
public void createRAMDirectory() throws Exception {
    this.directory = new RAMDirectory();
    IndexWriterConfig indexWriterConfig = new IndexWriterConfig(Version.LUCENE_44, this.analyzer);
    IndexWriter indexWriter = new IndexWriter(this.directory, indexWriterConfig);
    indexWriter.close();
}

private Translog openTranslog(EngineConfig engineConfig, IndexWriter writer, LongSupplier globalCheckpointSupplier) throws IOException {
    assert openMode != null;
    final TranslogConfig translogConfig = engineConfig.getTranslogConfig();
    Translog.TranslogGeneration generation = null;
    if (openMode == EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG) {
        generation = loadTranslogIdFromCommit(writer);
        // We expect that this shard already exists, so it must already have an existing translog else something is badly wrong!
        if (generation == null) {
            throw new IllegalStateException("no translog generation present in commit data but translog is expected to exist");
        }
        if (generation.translogUUID == null) {
            throw new IndexFormatTooOldException("trasnlog", "translog has no generation nor a UUID - this might be an index from a previous version consider upgrading to N-1 first");
        }
    }
    final Translog translog = new Translog(translogConfig, generation, globalCheckpointSupplier);
    if (generation == null || generation.translogUUID == null) {
        assert openMode != EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG : "OpenMode must not be "
            + EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG;
        if (generation == null) {
            logger.debug("no translog ID present in the current generation - creating one");
        } else if (generation.translogUUID == null) {
            logger.debug("upgraded translog to pre 2.0 format, associating translog with index - writing translog UUID");
        }
        boolean success = false;
        try {
            commitIndexWriter(writer, translog, openMode == EngineConfig.OpenMode.OPEN_INDEX_CREATE_TRANSLOG
                ? commitDataAsMap(writer).get(SYNC_COMMIT_ID) : null);
            success = true;
        } finally {
            if (success == false) {
                IOUtils.closeWhileHandlingException(translog);
            }
        }
    }
    return translog;
}

@Test
public void testNumericValues() throws IOException {
    LuceneIndex index = new LuceneIndex();
    try (Reference<IndexWriter> writer = index.provideWriter()) {
        Document doc1 = new Document();
        LuceneFields.Long.add(doc1, "id", 1L, LuceneFields.FieldOptions.STORE_INDEX);
        LuceneFields.Double.add(doc1, "val", 5.0d, LuceneFields.FieldOptions.INDEX_DOCVALUE);
        writer.use().addDocument(doc1);

        Document doc2 = new Document();
        LuceneFields.Long.add(doc2, "id", 2L, LuceneFields.FieldOptions.STORE_INDEX);
        LuceneFields.Double.add(doc2, "val", 3.0d, LuceneFields.FieldOptions.INDEX_DOCVALUE);
        writer.use().addDocument(doc2);

        Document doc3 = new Document();
        LuceneFields.Long.add(doc3, "id", 3L, LuceneFields.FieldOptions.STORE_INDEX);
        LuceneFields.Double.add(doc3, "val", 7.0d, LuceneFields.FieldOptions.INDEX_DOCVALUE);
        writer.use().addDocument(doc3);
    }

    LuceneSearch search = LuceneSearch.builder().sort(new Sort(new SortField("val", SortField.Type.DOUBLE))).build();
    List<LuceneSearchHit> hits = index.search(search).toList();
    assertEquals(3, hits.size());
    assertEquals(2L, (long)LuceneFields.Long.get(hits.get(0).getField("id")));
    assertEquals(1L, (long)LuceneFields.Long.get(hits.get(1).getField("id")));
    assertEquals(3L, (long)LuceneFields.Long.get(hits.get(2).getField("id")));
}

private static void index(final List<ParseContext.Document> docs, final IndexWriter indexWriter) throws IOException {
    if (docs.size() > 1) {
        indexWriter.addDocuments(docs);
    } else {
        indexWriter.addDocument(docs.get(0));
    }
}

private static void update(final Term uid, final List<ParseContext.Document> docs, final IndexWriter indexWriter) throws IOException {
    if (docs.size() > 1) {
        indexWriter.updateDocuments(uid, docs);
    } else {
        indexWriter.updateDocument(uid, docs.get(0));
    }
}

private IndexWriter createWriter(boolean create) throws IOException {
    try {
        final IndexWriterConfig iwc = getIndexWriterConfig(create);
        return createWriter(store.directory(), iwc);
    } catch (LockObtainFailedException ex) {
        logger.warn("could not lock IndexWriter", ex);
        throw ex;
    }
}

/**
 * Gets the commit data from {@link IndexWriter} as a map.
 */
private static Map<String, String> commitDataAsMap(final IndexWriter indexWriter) {
    Map<String, String> commitData = new HashMap<>(6);
    for (Map.Entry<String, String> entry : indexWriter.getLiveCommitData()) {
        commitData.put(entry.getKey(), entry.getValue());
    }
    return commitData;
}