Java 类org.apache.lucene.analysis.standard.StandardTokenizerInterface 实例源码

@Override
public final boolean incrementToken() throws IOException {
    clearAttributes();
    skippedPositions = 0;
    while (true) {
        int tokenType = scanner.getNextToken();
        if (tokenType == StandardTokenizerInterface.YYEOF) {
            return false;
        }
        if (scanner.yylength() <= maxTokenLength) {
            posIncrAtt.setPositionIncrement(skippedPositions + 1);
            scanner.getText(termAtt);
            final int start = scanner.yychar();
            offsetAtt.setOffset(correctOffset(start), correctOffset(start + termAtt.length()));
            typeAtt.setType(HyphenTokenizer.TOKEN_TYPES[tokenType]);
            return true;
        } else {
            skippedPositions++;
        }
    }
}

@Override
public boolean incrementToken() throws IOException {
    clearAttributes();
    int posIncr = 1;

    while (true) {
        int tokenType = scanner.getNextToken();

        if (tokenType == StandardTokenizerInterface.YYEOF) {
            return false;
        }

        if (scanner.yylength() <= maxTokenLength) {
            posIncrAtt.setPositionIncrement(posIncr);
            scanner.getText(termAtt);
            final int start = scanner.yychar();
            offsetAtt.setOffset(correctOffset(start), correctOffset(start + termAtt.length()));

            if (tokenType == StandardTokenizer.ACRONYM_DEP) {
                typeAtt.setType(StandardTokenizer.TOKEN_TYPES[StandardTokenizer.HOST]);
                termAtt.setLength(termAtt.length() - 1);
            } else
                typeAtt.setType(StandardTokenizer.TOKEN_TYPES[tokenType]);
            return true;
        } else
            posIncr++;
    }
}

@Override
public boolean incrementToken() throws IOException {
    clearAttributes();
    int posIncr = 1;

    while (true) {
        int tokenType = scanner.getNextToken();

        if (tokenType == StandardTokenizerInterface.YYEOF) {
            return false;
        }

        if (scanner.yylength() <= maxTokenLength) {
            posIncrAtt.setPositionIncrement(posIncr);
            scanner.getText(termAtt);
            final int start = scanner.yychar();
            offsetAtt.setOffset(correctOffset(start), correctOffset(start + termAtt.length()));

            if (tokenType == StandardTokenizer.ACRONYM_DEP) {
                typeAtt.setType(StandardTokenizer.TOKEN_TYPES[StandardTokenizer.HOST]);
                termAtt.setLength(termAtt.length() - 1);
            } else
                typeAtt.setType(StandardTokenizer.TOKEN_TYPES[tokenType]);
            return true;
        } else
            posIncr++;
    }
}