Java 类com.intellij.lang.cacheBuilder.WordOccurrence 实例源码

public void processWords(CharSequence fileText, Processor<WordOccurrence> processor) {
    myLexer.start(fileText);
    WordOccurrence occurrence = new WordOccurrence(fileText, 0, 0, null); // shared occurrence

    IElementType type;
    while ((type = myLexer.getTokenType()) != null) {
        if (myIdentifierTokenSet.contains(type)) {
            if (!stripWords(processor, fileText, myLexer.getTokenStart(), myLexer.getTokenEnd(), WordOccurrence.Kind.CODE, occurrence, false, myKeepCodeTokensWhole, myUseSpaceBreaks)) return;
        } else if (myCommentTokenSet.contains(type)) {
            if (!stripWords(processor, fileText, myLexer.getTokenStart(), myLexer.getTokenEnd(), WordOccurrence.Kind.COMMENTS, occurrence, false, false, myUseSpaceBreaks)) return;
        } else if (myLiteralTokenSet.contains(type)) {
            if (!stripWords(processor, fileText, myLexer.getTokenStart(), myLexer.getTokenEnd(), WordOccurrence.Kind.LITERALS, occurrence, myMayHaveFileRefsInLiterals, false, myUseSpaceBreaks)) return;
        } else if (myDefaultKind != null && !mySkipCodeContextTokenSet.contains(type)) {
            if (!stripWords(processor, fileText, myLexer.getTokenStart(), myLexer.getTokenEnd(), myDefaultKind, occurrence, false, myDefaultKind == WordOccurrence.Kind.CODE && myKeepCodeTokensWhole, myUseSpaceBreaks)) return;
        }
        myLexer.advance();
    }
}

@Override
public void processWords(CharSequence fileText, Processor<WordOccurrence> processor) {
  myLexer.start(fileText);
  WordOccurrence occurrence = new WordOccurrence(fileText, 0, 0, null); // shared occurrence

  IElementType type;
  while ((type = myLexer.getTokenType()) != null) {
    if (myIdentifierTokenSet.contains(type)) {
      if (!stripWordsSimple(processor, fileText, myLexer.getTokenStart(), myLexer.getTokenEnd(), WordOccurrence.Kind.CODE, occurrence))
        return;
    } else if (myCommentTokenSet.contains(type)) {
      if (!stripWordsSimple(processor, fileText, myLexer.getTokenStart(), myLexer.getTokenEnd(), WordOccurrence.Kind.COMMENTS, occurrence))
        return;
    } else if (myLiteralTokenSet.contains(type)) {
      if (!stripWordsSimple(processor, fileText, myLexer.getTokenStart(), myLexer.getTokenEnd(), WordOccurrence.Kind.LITERALS, occurrence))
        return;
    }
    myLexer.advance();
  }
}

private static void processClassAndPackagesNames(String qName, final Processor<WordOccurrence> processor) {
  WordOccurrence occurrence = new WordOccurrence(qName, 0, qName.length(), WordOccurrence.Kind.FOREIGN_LANGUAGE);
  processor.process(occurrence);
  int idx = qName.lastIndexOf('.');

  while (idx > 0) {
    qName = qName.substring(0, idx);
    occurrence.init(qName, 0,qName.length(),WordOccurrence.Kind.FOREIGN_LANGUAGE);
    processor.process(occurrence);
    idx = qName.lastIndexOf('.');
  }
}

/**
 * Creates a new instance of the words scanner.
 *
 * @param lexer                   the lexer used for breaking the text into tokens.
 * @param identifierTokenSet      the set of token types which represent identifiers.
 * @param commentTokenSet         the set of token types which represent comments.
 * @param literalTokenSet         the set of token types which represent literals.
 * @param skipCodeContextTokenSet the set of token types which should not be considered as code context.
 */
public MultiMarkdownWordsScanner(final Lexer lexer, final TokenSet identifierTokenSet, final TokenSet commentTokenSet,
        final TokenSet literalTokenSet, @NotNull TokenSet skipCodeContextTokenSet, int version) {
    myLexer = lexer;
    myIdentifierTokenSet = identifierTokenSet;
    myCommentTokenSet = commentTokenSet;
    myLiteralTokenSet = literalTokenSet;
    mySkipCodeContextTokenSet = skipCodeContextTokenSet;
    myDefaultKind = WordOccurrence.Kind.CODE;
    myVersion = version;
}

protected static boolean stripWordsSimple(final Processor<WordOccurrence> processor,
                                          final CharSequence tokenText,
                                          int from,
                                          int to,
                                          final WordOccurrence.Kind kind,
                                          @NotNull WordOccurrence occurrence
) {
  // Simply grab everything in a token to use for the Word Index

  occurrence.init(tokenText, from, to, kind);

  return processor.process(occurrence);
}

private static void processClassAndPackagesNames(String qName, final Processor<WordOccurrence> processor) {
  WordOccurrence occurrence = new WordOccurrence(qName, 0, qName.length(), WordOccurrence.Kind.FOREIGN_LANGUAGE);
  processor.process(occurrence);
  int idx = qName.lastIndexOf('.');

  while (idx > 0) {
    qName = qName.substring(0, idx);
    occurrence.init(qName, 0,qName.length(),WordOccurrence.Kind.FOREIGN_LANGUAGE);
    processor.process(occurrence);
    idx = qName.lastIndexOf('.');
  }
}

private static void processClassAndPackagesNames(String qName, final Processor<WordOccurrence> processor) {
  WordOccurrence occurrence = new WordOccurrence(qName, 0, qName.length(), WordOccurrence.Kind.FOREIGN_LANGUAGE);
  processor.process(occurrence);
  int idx = qName.lastIndexOf('.');

  while (idx > 0) {
    qName = qName.substring(0, idx);
    occurrence.init(qName, 0,qName.length(),WordOccurrence.Kind.FOREIGN_LANGUAGE);
    processor.process(occurrence);
    idx = qName.lastIndexOf('.');
  }
}

public void setDefaultKind(WordOccurrence.Kind defaultKind) {
    this.myDefaultKind = defaultKind;
}