public void testCpp() throws Exception { EditorHighlighter highlighter = HighlighterFactory.createHighlighter(getProject(), "A.cpp"); // 0123456789012345678 9 0123 45 6 7 highlighter.setText("#include try enum \"\\xff\\z\\\"xxx\""); HighlighterIterator iterator = highlighter.createIterator(2); assertEquals(CustomHighlighterTokenType.KEYWORD_1, iterator.getTokenType()); iterator = highlighter.createIterator(9); assertEquals(CustomHighlighterTokenType.KEYWORD_2, iterator.getTokenType()); iterator = highlighter.createIterator(15); assertEquals(CustomHighlighterTokenType.KEYWORD_1, iterator.getTokenType()); iterator = highlighter.createIterator(19); assertEquals(StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN, iterator.getTokenType()); iterator = highlighter.createIterator(23); assertEquals(StringEscapesTokenTypes.INVALID_CHARACTER_ESCAPE_TOKEN, iterator.getTokenType()); iterator = highlighter.createIterator(25); assertEquals(StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN, iterator.getTokenType()); iterator = highlighter.createIterator(27); assertEquals(CustomHighlighterTokenType.STRING, iterator.getTokenType()); }
public static void addWordHonoringEscapeSequences(CharSequence editorText, TextRange literalTextRange, int cursorOffset, Lexer lexer, List<TextRange> result) { lexer.start(editorText, literalTextRange.getStartOffset(), literalTextRange.getEndOffset()); while (lexer.getTokenType() != null) { if (lexer.getTokenStart() <= cursorOffset && cursorOffset < lexer.getTokenEnd()) { if (StringEscapesTokenTypes.STRING_LITERAL_ESCAPES.contains(lexer.getTokenType())) { result.add(new TextRange(lexer.getTokenStart(), lexer.getTokenEnd())); } else { TextRange word = getWordSelectionRange(editorText, cursorOffset, JAVA_IDENTIFIER_PART_CONDITION); if (word != null) { result.add(new TextRange(Math.max(word.getStartOffset(), lexer.getTokenStart()), Math.min(word.getEndOffset(), lexer.getTokenEnd()))); } } break; } lexer.advance(); } }
@Nullable private IElementType locateToken() { if (myEnd >= myBufferEnd) return null; myStart = myEnd; if (checkForSimpleValidEscape(myStart)) { myEnd = myStart + 2; return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN; } else if (checkForHexCodeStart(myStart)) { for (myEnd = myStart + 2; myEnd < myStart + 6; myEnd++) { if (myEnd >= myBufferEnd || !StringUtil.isHexDigit(myBuffer.charAt(myEnd))) { return StringEscapesTokenTypes.INVALID_UNICODE_ESCAPE_TOKEN; } } return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN; } else if (checkForInvalidSimpleEscape(myStart)) { myEnd = myStart + 2; return StringEscapesTokenTypes.INVALID_CHARACTER_ESCAPE_TOKEN; } while (myEnd < myBufferEnd && !checkForSimpleValidEscape(myEnd) && !checkForHexCodeStart(myEnd)) myEnd++; return myContentElementType; }
public static void addWordHonoringEscapeSequences(CharSequence editorText, TextRange literalTextRange, int cursorOffset, Lexer lexer, List<TextRange> result) { lexer.start(editorText, literalTextRange.getStartOffset(), literalTextRange.getEndOffset()); while (lexer.getTokenType() != null) { if (lexer.getTokenStart() <= cursorOffset && cursorOffset < lexer.getTokenEnd()) { if (StringEscapesTokenTypes.STRING_LITERAL_ESCAPES.contains(lexer.getTokenType())) { result.add(new TextRange(lexer.getTokenStart(), lexer.getTokenEnd())); } else { TextRange word = getWordSelectionRange(editorText, cursorOffset); if (word != null) { result.add(new TextRange(Math.max(word.getStartOffset(), lexer.getTokenStart()), Math.min(word.getEndOffset(), lexer.getTokenEnd()))); } } break; } lexer.advance(); } }
@Nullable private IElementType locateToken() { if (myEnd >= myBufferEnd) return null; myStart = myEnd; if (checkForSlashEscape(myStart)) { myEnd = myStart + 2; return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN; } else if (checkForHexCodeStart(myStart)) { for (myEnd = myStart + 2; myEnd < myStart + 6; myEnd++) { if (myEnd >= myBufferEnd || !StringUtil.isHexDigit(myBuffer.charAt(myEnd))) { return StringEscapesTokenTypes.INVALID_UNICODE_ESCAPE_TOKEN; } } return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN; } while (myEnd < myBufferEnd && !checkForSlashEscape(myEnd) && !checkForHexCodeStart(myEnd)) myEnd++; return GroovyTokenTypes.mREGEX_CONTENT; }
@Override public boolean isOpeningQuote(HighlighterIterator iterator, int offset) { boolean openingQuote = super.isOpeningQuote(iterator, offset); if(openingQuote) { // check escape next if(!iterator.atEnd()) { iterator.retreat(); if(!iterator.atEnd() && StringEscapesTokenTypes.STRING_LITERAL_ESCAPES.contains(iterator.getTokenType())) { openingQuote = false; } iterator.advance(); } } return openingQuote; }
@Override public boolean isClosingQuote(HighlighterIterator iterator, int offset) { boolean closingQuote = super.isClosingQuote(iterator, offset); if(closingQuote) { // check escape next if(!iterator.atEnd()) { iterator.advance(); if(!iterator.atEnd() && StringEscapesTokenTypes.STRING_LITERAL_ESCAPES.contains(iterator.getTokenType())) { closingQuote = false; } iterator.retreat(); } } return closingQuote; }
private static boolean isInStringLiteral(@Nonnull Editor editor, @Nonnull DataContext dataContext, int offset) { Language language = EnterHandler.getLanguage(dataContext); if (offset > 0 && language != null) { QuoteHandler quoteHandler = TypedHandler.getLanguageQuoteHandler(language); if (quoteHandler == null) { FileType fileType = language.getAssociatedFileType(); quoteHandler = fileType != null ? TypedHandler.getQuoteHandlerForType(fileType) : null; } if (quoteHandler != null) { EditorHighlighter highlighter = ((EditorEx)editor).getHighlighter(); HighlighterIterator iterator = highlighter.createIterator(offset - 1); return StringEscapesTokenTypes.STRING_LITERAL_ESCAPES.contains(iterator.getTokenType()) || quoteHandler.isInsideLiteral(iterator); } } return false; }
/** * The 'heavy' constructor that initializes everything. PySyntaxHighlighterFactory caches such instances per level. * @param languageLevel */ public PyHighlighter(LanguageLevel languageLevel) { myLanguageLevel = languageLevel; keys = new HashMap<IElementType, TextAttributesKey>(); fillMap(keys, PythonDialectsTokenSetProvider.INSTANCE.getKeywordTokens(), PY_KEYWORD); fillMap(keys, PyTokenTypes.OPERATIONS, PY_OPERATION_SIGN); keys.put(PyTokenTypes.INTEGER_LITERAL, PY_NUMBER); keys.put(PyTokenTypes.FLOAT_LITERAL, PY_NUMBER); keys.put(PyTokenTypes.IMAGINARY_LITERAL, PY_NUMBER); keys.put(PyTokenTypes.SINGLE_QUOTED_STRING, PY_BYTE_STRING); keys.put(PyTokenTypes.TRIPLE_QUOTED_STRING, PY_BYTE_STRING); keys.put(PyTokenTypes.SINGLE_QUOTED_UNICODE, PY_UNICODE_STRING); keys.put(PyTokenTypes.TRIPLE_QUOTED_UNICODE, PY_UNICODE_STRING); keys.put(PyTokenTypes.DOCSTRING, PY_DOC_COMMENT); keys.put(PyTokenTypes.LPAR, PY_PARENTHS); keys.put(PyTokenTypes.RPAR, PY_PARENTHS); keys.put(PyTokenTypes.LBRACE, PY_BRACES); keys.put(PyTokenTypes.RBRACE, PY_BRACES); keys.put(PyTokenTypes.LBRACKET, PY_BRACKETS); keys.put(PyTokenTypes.RBRACKET, PY_BRACKETS); keys.put(PyTokenTypes.COMMA, PY_COMMA); keys.put(PyTokenTypes.DOT, PY_DOT); keys.put(PyTokenTypes.END_OF_LINE_COMMENT, PY_LINE_COMMENT); keys.put(PyTokenTypes.BAD_CHARACTER, HighlighterColors.BAD_CHARACTER); keys.put(StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN, PY_VALID_STRING_ESCAPE); keys.put(StringEscapesTokenTypes.INVALID_CHARACTER_ESCAPE_TOKEN, PY_INVALID_STRING_ESCAPE); keys.put(StringEscapesTokenTypes.INVALID_UNICODE_ESCAPE_TOKEN, PY_INVALID_STRING_ESCAPE); }
private static boolean distinctTokens(@Nullable IElementType token1, @Nullable IElementType token2) { if (token1 == token2) return false; if (token1 == null || token2 == null) return true; if (StringEscapesTokenTypes.STRING_LITERAL_ESCAPES.contains(token1) || StringEscapesTokenTypes.STRING_LITERAL_ESCAPES.contains(token2)) return false; if (!token1.getLanguage().is(token2.getLanguage())) return true; BidiRegionsSeparator separator = LanguageBidiRegionsSeparator.INSTANCE.forLanguage(token1.getLanguage()); return separator.createBorderBetweenTokens(token1, token2); }
@Override public IElementType getTokenType() { if (myStart >= myEnd) return null; if (myBuffer.charAt(myStart) != '\\') { mySeenEscapedSpacesOnly = false; return myOriginalLiteralToken; } if (myStart + 1 >= myEnd) return StringEscapesTokenTypes.INVALID_CHARACTER_ESCAPE_TOKEN; char nextChar = myBuffer.charAt(myStart + 1); mySeenEscapedSpacesOnly &= nextChar == ' '; if (myCanEscapeEolOrFramingSpaces && (nextChar == '\n' || nextChar == ' ' && (mySeenEscapedSpacesOnly || isTrailingSpace(myStart+2))) ) { return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN; } if (nextChar == 'u') { for(int i = myStart + 2; i < myStart + 6; i++) { if (i >= myEnd || !StringUtil.isHexDigit(myBuffer.charAt(i))) return StringEscapesTokenTypes.INVALID_UNICODE_ESCAPE_TOKEN; } return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN; } if (nextChar == 'x' && myAllowHex) { for(int i = myStart + 2; i < myStart + 4; i++) { if (i >= myEnd || !StringUtil.isHexDigit(myBuffer.charAt(i))) return StringEscapesTokenTypes.INVALID_UNICODE_ESCAPE_TOKEN; } return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN; } switch (nextChar) { case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': if (!myAllowOctal) return StringEscapesTokenTypes.INVALID_CHARACTER_ESCAPE_TOKEN; //noinspection fallthrough case 'n': case 'r': case 'b': case 't': case 'f': case '\'': case '\"': case '\\': return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN; } if (myAdditionalValidEscapes != null && myAdditionalValidEscapes.indexOf(nextChar) != -1) { return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN; } return StringEscapesTokenTypes.INVALID_CHARACTER_ESCAPE_TOKEN; }
@Override public Result preprocessEnter(@NotNull final PsiFile file, @NotNull final Editor editor, @NotNull Ref<Integer> caretOffsetRef, @NotNull final Ref<Integer> caretAdvanceRef, @NotNull final DataContext dataContext, final EditorActionHandler originalHandler) { int caretOffset = caretOffsetRef.get().intValue(); int caretAdvance = caretAdvanceRef.get().intValue(); PsiElement psiAtOffset = file.findElementAt(caretOffset); if (psiAtOffset != null && psiAtOffset.getTextOffset() < caretOffset) { Document document = editor.getDocument(); CharSequence text = document.getText(); ASTNode token = psiAtOffset.getNode(); JavaLikeQuoteHandler quoteHandler = getJavaLikeQuoteHandler(editor, psiAtOffset); if (quoteHandler != null && quoteHandler.getConcatenatableStringTokenTypes() != null && quoteHandler.getConcatenatableStringTokenTypes().contains(token.getElementType())) { TextRange range = token.getTextRange(); final char literalStart = token.getText().charAt(0); final StringLiteralLexer lexer = new StringLiteralLexer(literalStart, token.getElementType()); lexer.start(text, range.getStartOffset(), range.getEndOffset()); while (lexer.getTokenType() != null) { if (lexer.getTokenStart() < caretOffset && caretOffset < lexer.getTokenEnd()) { if (StringEscapesTokenTypes.STRING_LITERAL_ESCAPES.contains(lexer.getTokenType())) { caretOffset = lexer.getTokenEnd(); } break; } lexer.advance(); } if (quoteHandler.needParenthesesAroundConcatenation(psiAtOffset)) { document.insertString(psiAtOffset.getTextRange().getEndOffset(), ")"); document.insertString(psiAtOffset.getTextRange().getStartOffset(), "("); caretOffset++; caretAdvance++; } final String insertedFragment = literalStart + " " + quoteHandler.getStringConcatenationOperatorRepresentation(); document.insertString(caretOffset, insertedFragment + " " + literalStart); caretOffset += insertedFragment.length(); caretAdvance = 1; CommonCodeStyleSettings langSettings = CodeStyleSettingsManager.getSettings(file.getProject()).getCommonSettings(file.getLanguage()); if (langSettings.BINARY_OPERATION_SIGN_ON_NEXT_LINE) { caretOffset -= 1; caretAdvance = 3; } caretOffsetRef.set(caretOffset); caretAdvanceRef.set(caretAdvance); return Result.DefaultForceIndent; } } return Result.Continue; }
@Override public IElementType getTokenType() { if (myStart >= myEnd) return null; if (myBuffer.charAt(myStart) != '\\') { mySeenEscapedSpacesOnly = false; return myOriginalLiteralToken; } if (myStart + 1 >= myEnd) return StringEscapesTokenTypes.INVALID_CHARACTER_ESCAPE_TOKEN; char nextChar = myBuffer.charAt(myStart + 1); mySeenEscapedSpacesOnly &= nextChar == ' '; if (myCanEscapeEolOrFramingSpaces && (nextChar == '\n' || nextChar == ' ' && (mySeenEscapedSpacesOnly || isTrailingSpace(myStart+2))) ) { return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN; } if (nextChar == 'u') { for(int i = myStart + 2; i < myStart + 6; i++) { if (i >= myEnd || !StringUtil.isHexDigit(myBuffer.charAt(i))) return StringEscapesTokenTypes.INVALID_UNICODE_ESCAPE_TOKEN; } return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN; } if (nextChar == 'x' && myAllowHex) { for(int i = myStart + 2; i < myStart + 4; i++) { if (i >= myEnd || !StringUtil.isHexDigit(myBuffer.charAt(i))) return StringEscapesTokenTypes.INVALID_UNICODE_ESCAPE_TOKEN; } return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN; } switch (nextChar) { case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': if (!myAllowOctal) return StringEscapesTokenTypes.INVALID_CHARACTER_ESCAPE_TOKEN; case 'n': case 'r': case 'b': case 't': case 'f': case '\'': case '\"': case '\\': return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN; } if (myAdditionalValidEscapes != null && myAdditionalValidEscapes.indexOf(nextChar) != -1) { return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN; } return StringEscapesTokenTypes.INVALID_CHARACTER_ESCAPE_TOKEN; }
@Override public Result preprocessEnter(@NotNull final PsiFile file, @NotNull final Editor editor, @NotNull Ref<Integer> caretOffsetRef, @NotNull final Ref<Integer> caretAdvanceRef, @NotNull final DataContext dataContext, final EditorActionHandler originalHandler) { int caretOffset = caretOffsetRef.get().intValue(); int caretAdvance = caretAdvanceRef.get().intValue(); PsiElement psiAtOffset = file.findElementAt(caretOffset); if (psiAtOffset != null && psiAtOffset.getTextOffset() < caretOffset) { Document document = editor.getDocument(); CharSequence text = document.getText(); ASTNode token = psiAtOffset.getNode(); JavaLikeQuoteHandler quoteHandler = getJavaLikeQuoteHandler(editor, psiAtOffset); if (quoteHandler != null && quoteHandler.getConcatenatableStringTokenTypes() != null && quoteHandler.getConcatenatableStringTokenTypes().contains(token.getElementType())) { TextRange range = token.getTextRange(); final char literalStart = token.getText().charAt(0); final StringLiteralLexer lexer = new StringLiteralLexer(literalStart, token.getElementType()); lexer.start(text, range.getStartOffset(), range.getEndOffset()); while (lexer.getTokenType() != null) { if (lexer.getTokenStart() < caretOffset && caretOffset < lexer.getTokenEnd()) { if (StringEscapesTokenTypes.STRING_LITERAL_ESCAPES.contains(lexer.getTokenType())) { caretOffset = lexer.getTokenEnd(); } break; } lexer.advance(); } if (quoteHandler.needParenthesesAroundConcatenation(psiAtOffset)) { document.insertString(psiAtOffset.getTextRange().getEndOffset(), ")"); document.insertString(psiAtOffset.getTextRange().getStartOffset(), "("); caretOffset++; caretAdvance++; } final String insertedFragment = literalStart + " " + quoteHandler.getStringConcatenationOperatorRepresentation(); document.insertString(caretOffset, insertedFragment + " " + literalStart); caretOffset += insertedFragment.length(); caretAdvance = 1; if (CodeStyleSettingsManager.getSettings(file.getProject()).BINARY_OPERATION_SIGN_ON_NEXT_LINE) { caretOffset -= 1; caretAdvance = 3; } caretOffsetRef.set(caretOffset); caretAdvanceRef.set(caretAdvance); return Result.DefaultForceIndent; } } return Result.Continue; }
@Override public Result preprocessEnter(@Nonnull final PsiFile file, @Nonnull final Editor editor, @Nonnull Ref<Integer> caretOffsetRef, @Nonnull final Ref<Integer> caretAdvanceRef, @Nonnull final DataContext dataContext, final EditorActionHandler originalHandler) { int caretOffset = caretOffsetRef.get().intValue(); int caretAdvance = caretAdvanceRef.get().intValue(); if (!isInStringLiteral(editor, dataContext, caretOffset)) return Result.Continue; PsiDocumentManager.getInstance(file.getProject()).commitDocument(editor.getDocument()); PsiElement psiAtOffset = file.findElementAt(caretOffset); if (psiAtOffset != null && psiAtOffset.getTextOffset() < caretOffset) { Document document = editor.getDocument(); CharSequence text = document.getText(); ASTNode token = psiAtOffset.getNode(); JavaLikeQuoteHandler quoteHandler = getJavaLikeQuoteHandler(editor, psiAtOffset); if (quoteHandler != null && quoteHandler.getConcatenatableStringTokenTypes() != null && quoteHandler.getConcatenatableStringTokenTypes().contains(token.getElementType())) { TextRange range = token.getTextRange(); final char literalStart = token.getText().charAt(0); final StringLiteralLexer lexer = new StringLiteralLexer(literalStart, token.getElementType()); lexer.start(text, range.getStartOffset(), range.getEndOffset()); while (lexer.getTokenType() != null) { if (lexer.getTokenStart() < caretOffset && caretOffset < lexer.getTokenEnd()) { if (StringEscapesTokenTypes.STRING_LITERAL_ESCAPES.contains(lexer.getTokenType())) { caretOffset = lexer.getTokenEnd(); } break; } lexer.advance(); } if (quoteHandler.needParenthesesAroundConcatenation(psiAtOffset)) { document.insertString(psiAtOffset.getTextRange().getEndOffset(), ")"); document.insertString(psiAtOffset.getTextRange().getStartOffset(), "("); caretOffset++; caretAdvance++; } final String insertedFragment = literalStart + " " + quoteHandler.getStringConcatenationOperatorRepresentation(); document.insertString(caretOffset, insertedFragment + " " + literalStart); caretOffset += insertedFragment.length(); caretAdvance = 1; CommonCodeStyleSettings langSettings = CodeStyleSettingsManager.getSettings(file.getProject()).getCommonSettings(file.getLanguage()); if (langSettings.BINARY_OPERATION_SIGN_ON_NEXT_LINE) { caretOffset -= 1; caretAdvance = 3; } caretOffsetRef.set(caretOffset); caretAdvanceRef.set(caretAdvance); return Result.DefaultForceIndent; } } return Result.Continue; }