Java 类org.apache.lucene.analysis.Analyzer 实例源码

private Fields generateTermVectors(Collection<GetField> getFields, boolean withOffsets, @Nullable Map<String, String> perFieldAnalyzer, Set<String> fields)
        throws IOException {
    /* store document in memory index */
    MemoryIndex index = new MemoryIndex(withOffsets);
    for (GetField getField : getFields) {
        String field = getField.getName();
        if (fields.contains(field) == false) {
            // some fields are returned even when not asked for, eg. _timestamp
            continue;
        }
        Analyzer analyzer = getAnalyzerAtField(field, perFieldAnalyzer);
        for (Object text : getField.getValues()) {
            index.addField(field, text.toString(), analyzer);
        }
    }
    /* and read vectors from it */
    return MultiFields.getFields(index.createSearcher().getIndexReader());
}

/**
 * Creates a new {@link Index.Transactional} for given folder with given lucene Analyzer.
 * The returned {@link Index} is not cached, next call with the same arguments returns a different instance
 * of {@link Index}. The caller is responsible to cache the returned {@link Index}.
 * @param cacheFolder the folder in which the index is stored
 * @param analyzer the lucene Analyzer used to split fields into tokens.
 * @param isWritable <code>false</code> if we will use it as read only
 * @return the created {@link Index.Transactional}
 * @throws IOException in case of IO problem.
 * @since 2.27.1
 */
@NonNull
public static Index.Transactional createTransactionalIndex(final @NonNull File cacheFolder, final @NonNull Analyzer analyzer, boolean isWritable) throws IOException {
    Parameters.notNull("cacheFolder", cacheFolder); //NOI18N
    Parameters.notNull("analyzer", analyzer);       //NOI18N
    if (!cacheFolder.canRead()) {
        throw new IOException(String.format("Cannot read cache folder: %s.", cacheFolder.getAbsolutePath()));   //NOI18N
    }
    if (isWritable && !cacheFolder.canWrite()) {
        throw new IOException(String.format("Cannot write to cache folder: %s.", cacheFolder.getAbsolutePath()));   //NOI18N
    }
    final Index.Transactional index = factory.createIndex(cacheFolder, analyzer);
    assert index != null;
    indexes.put(cacheFolder, new Ref(cacheFolder,index));
    return index;
}

/**
 * 查询索引
 * 
 * @param keywords
 * @return
 * @throws Exception
 */
public List<Document> searchIndex(Integer typeId, String keywords) throws Exception {
    // 1.init searcher
    Analyzer analyzer = new PaodingAnalyzer();
    IndexReader reader = IndexReader.open(typeId == appConfig.getGameTypeId() ? appConfig.getGameIndexDir()
            : appConfig.getSoftIndexDir());
    BooleanClause.Occur[] flags = new BooleanClause.Occur[] { BooleanClause.Occur.SHOULD,
            BooleanClause.Occur.SHOULD };
    Query query = MultiFieldQueryParser.parse(keywords, appConfig.getQueryFields(), flags, analyzer);
    query = query.rewrite(reader);

    // 2.search
    List<Document> docs = new ArrayList<Document>();
    Hits hits = (typeId == appConfig.getGameTypeId() ? gameSearcher.search(query, Sort.RELEVANCE) : softSearcher
            .search(query, Sort.RELEVANCE));// searcher.search(query,
                                            // Sort.RELEVANCE);
    for (int i = 0; i < hits.length(); i++) {
        docs.add(hits.doc(i));
    }

    // 3.return
    reader.close();
    return docs;
}

public DocumentFieldMappers(Collection<FieldMapper> mappers, Analyzer defaultIndex, Analyzer defaultSearch, Analyzer defaultSearchQuote) {
    Map<String, FieldMapper> fieldMappers = new HashMap<>();
    Map<String, Analyzer> indexAnalyzers = new HashMap<>();
    Map<String, Analyzer> searchAnalyzers = new HashMap<>();
    Map<String, Analyzer> searchQuoteAnalyzers = new HashMap<>();
    for (FieldMapper mapper : mappers) {
        fieldMappers.put(mapper.name(), mapper);
        MappedFieldType fieldType = mapper.fieldType();
        put(indexAnalyzers, fieldType.names().indexName(), fieldType.indexAnalyzer(), defaultIndex);
        put(searchAnalyzers, fieldType.names().indexName(), fieldType.searchAnalyzer(), defaultSearch);
        put(searchQuoteAnalyzers, fieldType.names().indexName(), fieldType.searchQuoteAnalyzer(), defaultSearchQuote);
    }
    this.fieldMappers = Collections.unmodifiableMap(fieldMappers);
    this.indexAnalyzer = new FieldNameAnalyzer(indexAnalyzers);
    this.searchAnalyzer = new FieldNameAnalyzer(searchAnalyzers);
    this.searchQuoteAnalyzer = new FieldNameAnalyzer(searchQuoteAnalyzers);
}

/**
 * Test the WordScorer emitted by the smoothing model
 */
public void testBuildWordScorer() throws IOException {
    SmoothingModel testModel = createTestModel();
    Map<String, Analyzer> mapping = new HashMap<>();
    mapping.put("field", new WhitespaceAnalyzer());
    PerFieldAnalyzerWrapper wrapper = new PerFieldAnalyzerWrapper(new WhitespaceAnalyzer(), mapping);
    IndexWriter writer = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(wrapper));
    Document doc = new Document();
    doc.add(new Field("field", "someText", TextField.TYPE_NOT_STORED));
    writer.addDocument(doc);
    DirectoryReader ir = DirectoryReader.open(writer);

    WordScorer wordScorer = testModel.buildWordScorerFactory().newScorer(ir, MultiFields.getTerms(ir, "field"), "field", 0.9d,
            BytesRefs.toBytesRef(" "));
    assertWordScorer(wordScorer, testModel);
}

private static int findGoodEndForNoHighlightExcerpt(int noMatchSize, Analyzer analyzer, String fieldName, String contents) throws IOException {
    try (TokenStream tokenStream = analyzer.tokenStream(fieldName, contents)) {
        if (!tokenStream.hasAttribute(OffsetAttribute.class)) {
            // Can't split on term boundaries without offsets
            return -1;
        }
        int end = -1;
        tokenStream.reset();
        while (tokenStream.incrementToken()) {
            OffsetAttribute attr = tokenStream.getAttribute(OffsetAttribute.class);
            if (attr.endOffset() >= noMatchSize) {
                // Jump to the end of this token if it wouldn't put us past the boundary
                if (attr.endOffset() == noMatchSize) {
                    end = noMatchSize;
                }
                return end;
            }
            end = attr.endOffset();
        }
        tokenStream.end();
        // We've exhausted the token stream so we should just highlight everything.
        return end;
    }
}

public void testSimple() throws IOException {
    Analyzer analyzer = new Analyzer() {
        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer t = new MockTokenizer(MockTokenizer.WHITESPACE, false);
            return new TokenStreamComponents(t, new UniqueTokenFilter(t));
        }
    };

    TokenStream test = analyzer.tokenStream("test", "this test with test");
    test.reset();
    CharTermAttribute termAttribute = test.addAttribute(CharTermAttribute.class);
    assertThat(test.incrementToken(), equalTo(true));
    assertThat(termAttribute.toString(), equalTo("this"));

    assertThat(test.incrementToken(), equalTo(true));
    assertThat(termAttribute.toString(), equalTo("test"));

    assertThat(test.incrementToken(), equalTo(true));
    assertThat(termAttribute.toString(), equalTo("with"));

    assertThat(test.incrementToken(), equalTo(false));
}

public DirectCandidateGenerator(DirectSpellChecker spellchecker, String field, SuggestMode suggestMode, IndexReader reader,
        double nonErrorLikelihood, int numCandidates, Analyzer preFilter, Analyzer postFilter, Terms terms) throws IOException {
    if (terms == null) {
        throw new IllegalArgumentException("generator field [" + field + "] doesn't exist");
    }
    this.spellchecker = spellchecker;
    this.field = field;
    this.numCandidates = numCandidates;
    this.suggestMode = suggestMode;
    this.reader = reader;
    final long dictSize = terms.getSumTotalTermFreq();
    this.useTotalTermFrequency = dictSize != -1;
    this.dictSize =  dictSize == -1 ? reader.maxDoc() : dictSize;
    this.preFilter = preFilter;
    this.postFilter = postFilter;
    this.nonErrorLikelihood = nonErrorLikelihood;
    float thresholdFrequency = spellchecker.getThresholdFrequency();
    this.frequencyPlateau = thresholdFrequency >= 1.0f ? (int) thresholdFrequency: (int)(dictSize * thresholdFrequency);
    termsEnum = terms.iterator();
}

public void testCommonTermsQuery() throws IOException {
    Directory dir = newDirectory();
    String value = "The quick brown fox.";
    Analyzer analyzer = new StandardAnalyzer();
    IndexReader ir = indexOneDoc(dir, "text", value, analyzer);
    CommonTermsQuery query = new CommonTermsQuery(BooleanClause.Occur.SHOULD, BooleanClause.Occur.SHOULD, 128);
    query.add(new Term("text", "quick"));
    query.add(new Term("text", "brown"));
    query.add(new Term("text", "fox"));
    IndexSearcher searcher = newSearcher(ir);
    TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
    assertThat(topDocs.totalHits, equalTo(1));
    int docId = topDocs.scoreDocs[0].doc;
    CustomPassageFormatter passageFormatter = new CustomPassageFormatter("<b>", "</b>", new DefaultEncoder());
    CustomUnifiedHighlighter highlighter = new CustomUnifiedHighlighter(searcher, analyzer,
        passageFormatter, null, value, false);
    Snippet[] snippets = highlighter.highlightField("text", query, docId, 5);
    assertThat(snippets.length, equalTo(1));
    assertThat(snippets[0].getText(), equalTo("The <b>quick</b> <b>brown</b> <b>fox</b>."));
    ir.close();
    dir.close();
}

/**
 * @deprecated Use {@link #parse(String, String[], BooleanClause.Occur[], Analyzer)}
 */
@Deprecated
public static Query parse(Version matchVersion, String query, String[] fields,
      BooleanClause.Occur[] flags, Analyzer analyzer) throws ParseException {
  if (fields.length != flags.length)
    throw new IllegalArgumentException("fields.length != flags.length");
  BooleanQuery bQuery = new BooleanQuery();
  for (int i = 0; i < fields.length; i++) {
    QueryParser qp = new QueryParser(matchVersion, fields[i], analyzer);
    Query q = qp.parse(query);
    if (q!=null && // q never null, just being defensive 
        (!(q instanceof BooleanQuery) || ((BooleanQuery)q).getClauses().length>0)) {
      bQuery.add(q, flags[i]);
    }
  }
  return bQuery;
}

private void assertLuceneAnalyzersAreNotClosed(Map<PreBuiltAnalyzers, List<Version>> loadedAnalyzers) throws IOException {
    for (Map.Entry<PreBuiltAnalyzers, List<Version>> preBuiltAnalyzerEntry : loadedAnalyzers.entrySet()) {
        for (Version version : preBuiltAnalyzerEntry.getValue()) {
            Analyzer analyzer = preBuiltAnalyzerEntry.getKey().getCache().get(version);
            try (TokenStream stream = analyzer.tokenStream("foo", "bar")) {
                stream.reset();
                while (stream.incrementToken()) {
                }
                stream.end();
            }
        }
    }
}

public void indexInit() throws Exception {
    Analyzer analyzer = new IKAnalyzer();
//  Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_44);
    this.indexSettings = new LuceneIndexSettings(analyzer);
    this.indexSettings.createFSDirectory("f:\\file");
    this.luceneIndex = new LuceneIndex(this.indexSettings);
    this.luceneIndexSearch = new LuceneIndexSearch(indexSettings, new LuceneResultCollector(indexSettings));
}

@Override
public TokenStream tokenStream(Analyzer analyzer, TokenStream previous) {
    if (fieldType().indexOptions() != IndexOptions.NONE) {
        return getCachedStream().setIntValue(number);
    }
    return null;
}

/**
 * Returns the {@link Locale} specific {@link Analyzer}.
 * If {@link Locale} is null than the {@link #DEFAULT_ANALYZER} will be returned.
 * @param locale the {@link Locale} to get the {@link Analyzer}.
 * @return the {@link Analyzer}. Null if no {@link Analyzer} was defined for the given {@link Locale}.
 */
Analyzer getAnalyzer(final Locale locale) {
    if(locale == null) {
        return DEFAULT_ANALYZER;
    }

    return ANALYZERS.get(locale.getLanguage());
}

@Override
public Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> getAnalyzers() {
  Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> extra = new HashMap<>();

  extra.put("jieba_search", JiebaAnalyzerProvider::getJiebaSearchAnalyzerProvider);
  extra.put("jieba_index", JiebaAnalyzerProvider::getJiebaIndexAnalyzerProvider);

  return extra;
}

private boolean openIndex(String foldername, String folderpath) {
    try {
        Analyzer analyzer = new StandardAnalyzer();

        //diskConfig = new IndexWriterConfig(analyzer);
        //diskConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
        //diskConfig.setRAMBufferSizeMB(256.0);
        //init director

        FSDirectory diskDirectory = FSDirectory.open(Paths.get(folderpath));

        //init reader
        DirectoryReader diskReader = DirectoryReader.open(diskDirectory);

        //every thing is ok
        if (analyzer != null && diskDirectory != null && diskReader != null) {
            analyzerList.put(foldername, analyzer);
            directorList.put(foldername, diskDirectory);
            readerList.put(foldername, diskReader);

            log.info("Load Index Success:" + foldername + " path:" + folderpath);
            return true;
        } else {
            log.error("Load Index Fail:" + foldername);
            return false;
        }

    } catch (org.apache.lucene.index.IndexNotFoundException xe) {
        log.error("Load Index Not Found:" + foldername);
        //throw new Exception(e.getMessage());
        //e.printStackTrace();
    } catch (Exception xxe) {
        //do nothing
        xxe.printStackTrace();
        log.error("load index Exception:" + xxe.getMessage());
        //throw new Exception(e.getMessage());
    }
    return false;
}

public static AnalyzerProvider<? extends Analyzer> getJiebaIndexAnalyzerProvider(IndexSettings indexSettings,
                                                                                 Environment environment,
                                                                                 String s,
                                                                                 Settings settings) {
  JiebaAnalyzerProvider jiebaAnalyzerProvider = new JiebaAnalyzerProvider(indexSettings,
      environment,
      s,
      settings,
      JiebaSegmenter.SegMode.INDEX);

  return jiebaAnalyzerProvider;
}

public Indexer(String indexDirectoryPath) throws IOException {
    // Directory directory = new RAMDirectory();
    Directory indexDirectory = FSDirectory.open(Paths.get(indexDirectoryPath));
    // create the indexer
    Analyzer analyzer = new StandardAnalyzer();
    IndexWriterConfig config = new IndexWriterConfig(analyzer);
    writer = new IndexWriter(indexDirectory, config);
}

@Override
protected Map<String, Analyzer> getAnalyzerFieldMap(Analyzer autoComplete, Analyzer nonStemmed)
{
    return ImmutableMap.of(FreeTextQuery.FIELD_BODY_NOSTEM, nonStemmed, FreeTextQuery.FIELD_NAME_AUTOCOMPLETE,
        autoComplete, FreeTextQuery.FIELD_NAME_VECTORED_NOSTEM, nonStemmed, FreeTextQuery.FIELD_BOOKMARK_TAGS,
        nonStemmed, FreeTextQuery.FIELD_ATTACHMENT_VECTORED_NOSTEM, nonStemmed);
}

@Override
protected Analyzer getWrappedAnalyzer(String fieldName) {
    Analyzer analyzer = analyzers.get(fieldName);
    if (analyzer != null) {
        return analyzer;
    }
    // Don't be lenient here and return the default analyzer
    // Fields need to be explicitly added
    throw new IllegalArgumentException("Field [" + fieldName + "] has no associated analyzer");
}

private Query getRegexpQuerySingle(String field, String termStr) throws ParseException {
    currentFieldType = null;
    Analyzer oldAnalyzer = getAnalyzer();
    try {
        currentFieldType = parseContext.fieldMapper(field);
        if (currentFieldType != null) {
            if (!forcedAnalyzer) {
                setAnalyzer(parseContext.getSearchAnalyzer(currentFieldType));
            }
            Query query = null;
            if (currentFieldType.useTermQueryWithQueryString()) {
                query = currentFieldType.regexpQuery(termStr, RegExp.ALL, maxDeterminizedStates, multiTermRewriteMethod, parseContext);
            }
            if (query == null) {
                query = super.getRegexpQuery(field, termStr);
            }
            return query;
        }
        return super.getRegexpQuery(field, termStr);
    } catch (RuntimeException e) {
        if (settings.lenient()) {
            return null;
        }
        throw e;
    } finally {
        setAnalyzer(oldAnalyzer);
    }
}

@Override
public TokenStream tokenStream(Analyzer analyzer, TokenStream previous) {
    if (fieldType().indexOptions() != IndexOptions.NONE) {
        return getCachedStream().setDoubleValue(number);
    }
    return null;
}

/**
 * Test turkish lowercasing
 */
public void testTurkish() throws Exception {
  Analyzer a = new SnowballAnalyzer("Turkish");

  assertAnalyzesTo(a, "ağacı", new String[] { "ağaç" });
  assertAnalyzesTo(a, "AĞACI", new String[] { "ağaç" });
}

private Query getRegexpQuerySingle(String field, String termStr) throws ParseException {
    currentFieldType = null;
    Analyzer oldAnalyzer = getAnalyzer();
    try {
        currentFieldType = context.fieldMapper(field);
        if (currentFieldType != null) {
            if (!settings.forceAnalyzer()) {
                setAnalyzer(context.getSearchAnalyzer(currentFieldType));
            }
            Query query = null;
            if (currentFieldType.tokenized() == false) {
                query = currentFieldType.regexpQuery(termStr, RegExp.ALL,
                    getMaxDeterminizedStates(), getMultiTermRewriteMethod(), context);
            }
            if (query == null) {
                query = super.getRegexpQuery(field, termStr);
            }
            return query;
        }
        return super.getRegexpQuery(field, termStr);
    } catch (RuntimeException e) {
        if (settings.lenient()) {
            return null;
        }
        throw e;
    } finally {
        setAnalyzer(oldAnalyzer);
    }
}

private Analyzer getAnalyzerAtField(String field, @Nullable Map<String, String> perFieldAnalyzer) {
    MapperService mapperService = indexShard.mapperService();
    Analyzer analyzer;
    if (perFieldAnalyzer != null && perFieldAnalyzer.containsKey(field)) {
        analyzer = mapperService.analysisService().analyzer(perFieldAnalyzer.get(field).toString());
    } else {
        analyzer = mapperService.smartNameFieldType(field).indexAnalyzer();
    }
    if (analyzer == null) {
        analyzer = mapperService.analysisService().defaultIndexAnalyzer();
    }
    return analyzer;
}

/** Initializes a query parser.  Called by the QueryParser constructor
 *  @param matchVersion  Lucene version to match. See <a href="QueryParser.html#version">here</a>.
 *  @param f  the default field for query terms.
 *  @param a   used to find terms in the query text.
 */
public void init(Version matchVersion, String f, Analyzer a) {
  init(f, a);
  if (matchVersion.onOrAfter(Version.LUCENE_3_1) == false) {
    setAutoGeneratePhraseQueries(true);
  }
}

/** Gets the search analyzer for the given field, or the default if there is none present for the field
 * TODO: remove this by moving defaults into mappers themselves
 */
public Analyzer getSearchAnalyzer(MappedFieldType fieldType) {
    if (fieldType.searchAnalyzer() != null) {
        return fieldType.searchAnalyzer();
    }
    return mapperService().searchAnalyzer();
}

/**
 * Queries a feature index of a list of files. A default index order sorting is being used.
 *
 * @param files a {@link List} of {@link FeatureFile}, which indexes to search
 * @param query a query string to search in index
 * @param vcfInfoFields list of info fields to retrieve
 * @return a {List} of {@code FeatureIndexEntry} objects that satisfy index query
 * @throws FeatureIndexException if something is wrong in the filesystem or query syntax is wrong
 */
public IndexSearchResult searchFileIndexes(List<? extends FeatureFile> files, String query,
                                                     List<String> vcfInfoFields) throws FeatureIndexException {
    try (Analyzer analyzer = new StandardAnalyzer()) {
        QueryParser queryParser = new QueryParser(FeatureIndexFields.FEATURE_ID.getFieldName(), analyzer);
        return searchFileIndexes(files, queryParser.parse(query), vcfInfoFields, null, null);
    } catch (IOException | ParseException e) {
        throw new FeatureIndexException("Failed to perform index search for files " +
                                    files.stream().map(BaseEntity::getName).collect(Collectors.joining(", ")), e);
    }
}

/**
 * Creates a new suggester.
 *
 * @param indexAnalyzer Analyzer that will be used for
 *   analyzing suggestions while building the index.
 * @param queryAnalyzer Analyzer that will be used for
 *   analyzing query text during lookup
 * @param options see {@link #EXACT_FIRST}, {@link #PRESERVE_SEP}
 * @param maxSurfaceFormsPerAnalyzedForm Maximum number of
 *   surface forms to keep for a single analyzed form.
 *   When there are too many surface forms we discard the
 *   lowest weighted ones.
 * @param maxGraphExpansions Maximum number of graph paths
 *   to expand from the analyzed form.  Set this to -1 for
 *   no limit.
 */
public XAnalyzingSuggester(Analyzer indexAnalyzer, Automaton queryPrefix, Analyzer queryAnalyzer, int options, int maxSurfaceFormsPerAnalyzedForm, int maxGraphExpansions,
                           boolean preservePositionIncrements, FST<Pair<Long, BytesRef>> fst, boolean hasPayloads, int maxAnalyzedPathsForOneInput,
                           int sepLabel, int payloadSep, int endByte, int holeCharacter) {
    // SIMON EDIT: I added fst, hasPayloads and maxAnalyzedPathsForOneInput
  this.indexAnalyzer = indexAnalyzer;
  this.queryAnalyzer = queryAnalyzer;
  this.fst = fst;
  this.hasPayloads = hasPayloads;
  if ((options & ~(EXACT_FIRST | PRESERVE_SEP)) != 0) {
    throw new IllegalArgumentException("options should only contain EXACT_FIRST and PRESERVE_SEP; got " + options);
  }
  this.exactFirst = (options & EXACT_FIRST) != 0;
  this.preserveSep = (options & PRESERVE_SEP) != 0;

  // FLORIAN EDIT: I added <code>queryPrefix</code> for context dependent suggestions
  this.queryPrefix = queryPrefix;

  // NOTE: this is just an implementation limitation; if
  // somehow this is a problem we could fix it by using
  // more than one byte to disambiguate ... but 256 seems
  // like it should be way more then enough.
  if (maxSurfaceFormsPerAnalyzedForm <= 0 || maxSurfaceFormsPerAnalyzedForm > 256) {
    throw new IllegalArgumentException("maxSurfaceFormsPerAnalyzedForm must be > 0 and < 256 (got: " + maxSurfaceFormsPerAnalyzedForm + ")");
  }
  this.maxSurfaceFormsPerAnalyzedForm = maxSurfaceFormsPerAnalyzedForm;

  if (maxGraphExpansions < 1 && maxGraphExpansions != -1) {
    throw new IllegalArgumentException("maxGraphExpansions must -1 (no limit) or > 0 (got: " + maxGraphExpansions + ")");
  }
  this.maxGraphExpansions = maxGraphExpansions;
  this.maxAnalyzedPathsForOneInput = maxAnalyzedPathsForOneInput;
  this.preservePositionIncrements = preservePositionIncrements;
  this.sepLabel = sepLabel;
  this.payloadSep = payloadSep;
  this.endByte = endByte;
  this.holeCharacter = holeCharacter;
}

@Override
public TokenStream tokenStream(Analyzer analyzer, TokenStream previous) {
    if (fieldType().indexOptions() != IndexOptions.NONE) {
        return getCachedStream().setLongValue(number);
    }
    return null;
}

/**
 * Count position increments in a token stream.  Package private for testing.
 * @param analyzer analyzer to create token stream
 * @param fieldName field name to pass to analyzer
 * @param fieldValue field value to pass to analyzer
 * @return number of position increments in a token stream
 * @throws IOException if tokenStream throws it
 */
static int countPositions(Analyzer analyzer, String fieldName, String fieldValue) throws IOException {
    try (TokenStream tokenStream = analyzer.tokenStream(fieldName, fieldValue)) {
        int count = 0;
        PositionIncrementAttribute position = tokenStream.addAttribute(PositionIncrementAttribute.class);
        tokenStream.reset();
        while (tokenStream.incrementToken()) {
            count += position.getPositionIncrement();
        }
        tokenStream.end();
        count += position.getPositionIncrement();
        return count;
    }
}

public void run() throws java.io.IOException {
  Analyzer analyzer = new StandardAnalyzer();
  QueryParser parser = new QueryParser(field, analyzer);

  while (true) {
    String line = in.readLine();

    if (line == null || line.length() == -1)
      break;

    line = line.trim();
    if (line.length() == 0)
      break;

    Query query = null;
    try {
      query = parser.parse(line);
    } catch (Exception e) {
      e.printStackTrace();
    }
    searcher.search(query, null, 10);

    doPagingSearch(query);
  }

  reader.close();
  out.flush();
  out.close();
  synchronized (parent) {
    parent.completed++;
    if (parent.completed % 4 == 0) {
      System.out.println(parent.completed + " query batches completed");
    }
    parent.notify();
  }
}

protected void string(StringQuery<T> stringQuery) {
    try {
        String[] fields = stringQuery.getFields();
        IndexedTypeIdentifier iti = getIndexedTypeIdentifier(stringQuery.getEntityClass());
        Analyzer analyzer = getSearchIntegrator().getAnalyzer(iti);
        QueryParser parser = new MultiFieldQueryParser(fields, analyzer);
        Query luceneQuery = parser.parse(stringQuery.getQuery());
        fillQuery(stringQuery, luceneQuery);
    } catch (ParseException e) {
        throw new RuntimeException(e);
    }
}

@Override
public TokenStream tokenStream(Analyzer analyzer, TokenStream previous) {
    if (fieldType().indexOptions() != IndexOptions.NONE) {
        return getCachedStream().setIntValue(number);
    }
    return null;
}

/**
 * Count position increments in a token stream.  Package private for testing.
 * @param analyzer analyzer to create token stream
 * @param fieldName field name to pass to analyzer
 * @param fieldValue field value to pass to analyzer
 * @return number of position increments in a token stream
 * @throws IOException if tokenStream throws it
 */
static int countPositions(Analyzer analyzer, String fieldName, String fieldValue) throws IOException {
    try (TokenStream tokenStream = analyzer.tokenStream(fieldName, fieldValue)) {
        int count = 0;
        PositionIncrementAttribute position = tokenStream.addAttribute(PositionIncrementAttribute.class);
        tokenStream.reset();
        while (tokenStream.incrementToken()) {
            count += position.getPositionIncrement();
        }
        tokenStream.end();
        count += position.getPositionIncrement();
        return count;
    }
}

public Iterable<Document> doSearch(Environment env, Site site, Application application, Directory directory,
        String term, String language, String[] parseFields, Analyzer analyzer, String highlightWith,
        Map<String, String> parameters) throws IOException {
    List<Document> documents = new ArrayList<>();

    if("en".equalsIgnoreCase(language) && analyzer instanceof EnglishAnalyzer){
        List<String> fieldList = Arrays.asList(parseFields);
        if(fieldList.contains(Document.FIELD_TITLE)){
            List<Person> persons = service.getPersonsByLastNameLikeOrNameLike(term);
            persons.forEach(p -> documents.add(p.toDocument(site, application)));
        }
    }

    return documents;
}

public void testThatDefaultAndStandardAnalyzerAreTheSameInstance() {
    Analyzer currentStandardAnalyzer = PreBuiltAnalyzers.STANDARD.getAnalyzer(Version.CURRENT);
    Analyzer currentDefaultAnalyzer = PreBuiltAnalyzers.DEFAULT.getAnalyzer(Version.CURRENT);

    // special case, these two are the same instance
    assertThat(currentDefaultAnalyzer, is(currentStandardAnalyzer));
}

public Analyzer searchQuoteAnalyzer() {
    return this.searchQuoteAnalyzer;
}

public TermsFilterBuilder(Analyzer analyzer) {
  this.analyzer = analyzer;
}

public PreBuiltAnalyzerProviderFactory(String name, AnalyzerScope scope, Analyzer analyzer) {
    analyzerProvider = new PreBuiltAnalyzerProvider(name, scope, analyzer);
}