Java 类org.apache.lucene.analysis.miscellaneous.UniqueTokenFilter 实例源码

@Override
public TokenStream create(TokenStream tokenStream) {
    return new UniqueTokenFilter(tokenStream, onlyOnSamePosition);
}

@Override
public TokenStream create(TokenStream tokenStream) {
    return new UniqueTokenFilter(tokenStream, onlyOnSamePosition);
}

@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
    TermExtractorTokenizer tokenizer = new TermExtractorTokenizer(reader, new PhoneTermExtractor(), new EmailTermExtractor());
    return new TokenStreamComponents(tokenizer, new UniqueTokenFilter(tokenizer));
}

@Override
protected TokenStreamComponents createComponents(String field, Reader reader) {
    Tokenizer tokenizer = new TermExtractorTokenizer(reader, new PhoneTermExtractor());
    return new TokenStreamComponents(tokenizer, new UniqueTokenFilter(tokenizer));
}