@Override public void recover(final LexerNoViableAltException lnvae) { final CharStream charStream = lnvae.getInputStream(); final int startIndex = lnvae.getStartIndex(); final String text = charStream.getText(Interval.of(startIndex, charStream.index())); Location location = new Location(sourceName, _tokenStartCharIndex); String message = "unexpected character [" + getErrorDisplay(text) + "]."; char firstChar = text.charAt(0); if ((firstChar == '\'' || firstChar == '"') && text.length() - 2 > 0 && text.charAt(text.length() - 2) == '\\') { /* Use a simple heuristic to guess if the unrecognized characters were trying to be a string but has a broken escape sequence. * If it was add an extra message about valid string escape sequences. */ message += " The only valid escape sequences in strings starting with [" + firstChar + "] are [\\\\] and [\\" + firstChar + "]."; } throw location.createError(new IllegalArgumentException(message, lnvae)); }
/** "If an operator has whitespace on the right side only, it is treated as a postfix unary operator. As an example, the ++ operator in a++ b is treated as a postfix unary operator." "If an operator has no whitespace on the left but is followed immediately by a dot (.), it is treated as a postfix unary operator. As an example, the ++ operator in a++.b is treated as a postfix unary operator (a++ .b rather than a ++ .b)." */ public static boolean isPostfixOp(TokenStream tokens) { int stop = getLastOpTokenIndex(tokens); if ( stop==-1 ) return false; int start = tokens.index(); Token prevToken = tokens.get(start-1); // includes hidden-channel tokens Token nextToken = tokens.get(stop+1); boolean prevIsWS = isLeftOperatorWS(prevToken); boolean nextIsWS = isRightOperatorWS(nextToken); boolean result = !prevIsWS && nextIsWS || !prevIsWS && nextToken.getType()==SwiftParser.DOT; String text = tokens.getText(Interval.of(start, stop)); //System.out.println("isPostfixOp: '"+prevToken+"','"+text+"','"+nextToken+"' is "+result); return result; }
public static void print(final ParseTree node, final int level, CommonTokenStream stream) { final Interval sourceInterval = node.getSourceInterval(); final Token firstToken = stream.get(sourceInterval.a); int line = firstToken.getLine(); int charStart = firstToken.getCharPositionInLine(); int endLine = line; int endChar = charStart + firstToken.getText().length(); String data = "@(" + line + ":" + charStart + "," + endLine + ":" + endChar + ") with text: " + firstToken.getText(); final int tmp = level + 1; final StringBuilder sb = new StringBuilder(); sb.append(StringUtils.repeat("\t", level)); sb.append(node.getClass().getSimpleName() + ": " + data + " :" + node.getText()); System.out.println(sb.toString()); final int n = node.getChildCount(); for (int i = 0; i < n; i++) { final ParseTree c = node.getChild(i); print(c, tmp, stream); } }
public List<String> getStatementStrings(String ksqlString) { List<SqlBaseParser.SingleStatementContext> statementContexts = new KsqlParser().getStatements(ksqlString); List<String> result = new ArrayList<>(statementContexts.size()); for (SqlBaseParser.SingleStatementContext statementContext : statementContexts) { // Taken from http://stackoverflow.com/questions/16343288/how-do-i-get-the-original-text-that-an-antlr4-rule-matched CharStream charStream = statementContext.start.getInputStream(); result.add( charStream.getText( new Interval( statementContext.start.getStartIndex(), statementContext.stop.getStopIndex() ) ) ); } return result; }
@NotNull private String getText(TokenStream tokens, Interval interval) { int start = interval.a; int stop = interval.b; if (start < 0 || stop < 0) return ""; if (stop >= tokens.size()) stop = tokens.size() - 1; StringBuilder buf = new StringBuilder(); for (int i = start; i <= stop; i++) { Token t = tokens.get(i); if (t.getType() == Token.EOF) break; buf.append(t.getText()); if (i != stop) { buf.append(" "); } } return buf.toString(); }
@Override public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) { if ( offendingSymbol==null ) { final Lexer lexer = (Lexer) recognizer; int i = lexer.getCharIndex(); final int n = lexer.getInputStream().size(); if (i >= n) { i = n - 1; } final String text = lexer.getInputStream().getText(new Interval(i, i)); CommonToken t = (CommonToken) lexer.getTokenFactory().create(Token.INVALID_TYPE, text); t.setStartIndex(i); t.setStopIndex(i); t.setLine(line); t.setCharPositionInLine(charPositionInLine); offendingSymbol = t; } // System.out.println("lex error: " + offendingSymbol); issues.add(new Issue(msg, (Token)offendingSymbol)); }
@Override public Token emit() { if (_type == ID) { String firstChar = _input.getText(Interval.of(_tokenStartCharIndex, _tokenStartCharIndex)); if (Grammar.isTokenName(firstChar)) { _type = TOKEN_REF; } else { _type = RULE_REF; } if (_ruleType == Token.INVALID_TYPE) { _ruleType = _type; } } else if (_type == SEMI) { _ruleType = Token.INVALID_TYPE; } return super.emit(); }
private void updateVersionNumber(TerminalNode currentVersionToken, final int newVersion) { Interval sourceInterval = ParseTrees.getSourceInterval(currentVersionToken); OffsetRegion region = OffsetRegion.fromBounds(sourceInterval.a, sourceInterval.b + 1); TrackingPositionRegion trackingRegion = _snapshot.createTrackingRegion(region, TrackingPositionRegion.Bias.Forward); final SnapshotPositionRegion currentRegion = trackingRegion.getRegion(_snapshot.getVersionedDocument().getCurrentSnapshot()); final BaseDocument baseDocument = (BaseDocument)_snapshot.getVersionedDocument().getDocument(); if (baseDocument == null) { throw new UnsupportedOperationException("No document available"); } baseDocument.runAtomicAsUser(new Runnable() { @Override public void run() { try { baseDocument.remove(currentRegion.getStart().getOffset(), currentRegion.getLength()); baseDocument.insertString(currentRegion.getStart().getOffset(), Integer.toString(newVersion), null); } catch (BadLocationException ex) { Exceptions.printStackTrace(ex); } } }); }
@CheckForNull @RuleDependency(recognizer=GrammarParser.class, rule=GrammarParser.RULE_parserRuleSpec, version=0, dependents=Dependents.SELF) private static ParserRuleSpecContext findRuleForDescription(@NonNull Map<ParserRuleSpecContext, String> rules, @NonNull Description description) { for (Map.Entry<ParserRuleSpecContext, String> entry : rules.entrySet()) { if (!description.getName().equals(entry.getValue())) { continue; } Interval sourceInterval = ParseTrees.getSourceInterval(entry.getKey()); if (sourceInterval.a <= description.getOffset() && sourceInterval.b >= description.getOffset()) { return entry.getKey(); } } return null; }
@Override @RuleDependencies({ @RuleDependency(recognizer=GoParser.class, rule=GoParser.RULE_constSpec, version=0), @RuleDependency(recognizer=GoParser.class, rule=GoParser.RULE_identifierList, version=0), }) public void enterConstSpec(ConstSpecContext ctx) { if (!isTopLevel(ctx)) { return; } IdentifierListContext idListContext = ctx.identifierList(); List<? extends TerminalNode> identifiers = idListContext.IDENTIFIER(); String type = ctx.type() != null ? String.format(" : <font color='808080'>%s</font>", HtmlSignatureVisitor.UNCOLORED.visit(ctx.type())) : ""; for (TerminalNode identifier : identifiers) { Interval sourceInterval = new Interval(identifier.getSymbol().getStartIndex(), ParseTrees.getStopSymbol(ctx).getStopIndex()); String name = identifier.getSymbol().getText(); String signature = name + type; GoNode.DeclarationDescription description = new GoNode.DeclarationDescription(name, DeclarationKind.CONSTANT); description.setOffset(snapshot, getCurrentParent().getFileObject(), sourceInterval.a); description.setHtmlHeader(signature); getCurrentParent().getChildren().add(description); } }
@Override @RuleDependencies({ @RuleDependency(recognizer=GoParser.class, rule=GoParser.RULE_shortVarDecl, version=1), @RuleDependency(recognizer=GoParser.class, rule=GoParser.RULE_identifierList, version=0), }) public void enterShortVarDecl(ShortVarDeclContext ctx) { // no locals in navigator if (blockLevel > 0) { return; } IdentifierListContext idListContext = ctx.identifierList(); List<? extends TerminalNode> identifiers = idListContext.IDENTIFIER(); for (TerminalNode identifier : identifiers) { Interval sourceInterval = new Interval(identifier.getSymbol().getStartIndex(), ParseTrees.getStopSymbol(ctx).getStopIndex()); String name = identifier.getSymbol().getText(); String signature = name; GoNode.DeclarationDescription description = new GoNode.DeclarationDescription(name, DeclarationKind.VARIABLE); description.setOffset(snapshot, getCurrentParent().getFileObject(), sourceInterval.a); description.setHtmlHeader(String.format("%s", Description.htmlEscape(signature))); getCurrentParent().getChildren().add(description); } }
@Override @RuleDependency(recognizer=GoParser.class, rule=GoParser.RULE_interfaceType, version=0) public void enterInterfaceType(InterfaceTypeContext ctx) { if (isAnonymousType(ctx)) { return; } Interval sourceInterval = ParseTrees.getSourceInterval(ctx); String name = typeNameStack.isEmpty() ? "?interface?" : typeNameStack.peek(); String signature = name; GoNode.DeclarationDescription description = new GoNode.DeclarationDescription(name, DeclarationKind.INTERFACE); description.setOffset(snapshot, getCurrentParent().getFileObject(), sourceInterval.a); description.setHtmlHeader(String.format("%s", signature)); getCurrentParent().getChildren().add(description); descriptionStack.push(description); }
@Override @RuleDependency(recognizer=GoParser.class, rule=GoParser.RULE_structType, version=0) public void enterStructType(StructTypeContext ctx) { if (isAnonymousType(ctx)) { return; } Interval sourceInterval = ParseTrees.getSourceInterval(ctx); String name = typeNameStack.isEmpty() ? "?struct?" : typeNameStack.peek(); String signature = name; GoNode.DeclarationDescription description = new GoNode.DeclarationDescription(name, DeclarationKind.STRUCT); description.setOffset(snapshot, getCurrentParent().getFileObject(), sourceInterval.a); description.setHtmlHeader(String.format("%s", signature)); getCurrentParent().getChildren().add(description); descriptionStack.push(description); }
@Override @RuleDependencies({ @RuleDependency(recognizer=GoParser.class, rule=GoParser.RULE_methodDecl, version=0), @RuleDependency(recognizer=GoParser.class, rule=GoParser.RULE_methodName, version=0), @RuleDependency(recognizer=GoParser.class, rule=GoParser.RULE_receiver, version=0), @RuleDependency(recognizer=GoParser.class, rule=GoParser.RULE_baseTypeName, version=0), }) public void enterMethodDecl(MethodDeclContext ctx) { Interval sourceInterval = ParseTrees.getSourceInterval(ctx); String name = ctx.methodName() != null && ctx.methodName().IDENTIFIER() != null ? ctx.methodName().IDENTIFIER().getSymbol().getText() : "?"; String signature = HtmlSignatureVisitor.COLORED.visit(ctx); GoNode.DeclarationDescription description = new GoNode.DeclarationDescription(name, DeclarationKind.METHOD); description.setOffset(snapshot, getCurrentParent().getFileObject(), sourceInterval.a); description.setHtmlHeader(signature); getCurrentParent().getChildren().add(description); ReceiverContext receiverContext = ctx.receiver(); BaseTypeNameContext baseTypeNameContext = receiverContext != null ? receiverContext.baseTypeName() : null; if (baseTypeNameContext != null && baseTypeNameContext.IDENTIFIER() != null) { String receiverTypeName = baseTypeNameContext.IDENTIFIER().getText(); _methodDescriptions.add(Tuple.create(receiverTypeName, getCurrentParent(), description)); } descriptionStack.push(description); }
@Override public String getText(Interval interval) { int startIndex = interval.a; int endIndexInclusive = interval.b; if (currentSnapshotLine != null) { if (startIndex >= currentSnapshotLineStartIndex && (endIndexInclusive + 1) <= currentSnapshotLineStartIndex + currentSnapshotLine.length()) { return currentSnapshotLine.substring(startIndex - currentSnapshotLineStartIndex, endIndexInclusive - currentSnapshotLineStartIndex + 1); } } try { return document.getText(startIndex, endIndexInclusive - startIndex + 1).toString(); } catch (BadLocationException ex) { LOGGER.log(Level.WARNING, ex.getMessage(), ex); return null; } }
@Override public String getText(Interval interval) { int start = interval.a; int stop = interval.b; if (currentSnapshotLine != null) { if (start >= currentSnapshotLineStartIndex && stop < currentSnapshotLineStartIndex + currentSnapshotLine.length()) { return currentSnapshotLine.substring(start - currentSnapshotLineStartIndex, stop + 1 - currentSnapshotLineStartIndex); } } // HACK: special handling due to Lexer passing invalid indexes. if (stop >= size()) { stop = size() - 1; } return getSnapshot().subSequence(start, stop + 1).toString(); }
public static Interval getSourceInterval(@NonNull ParserRuleContext context) { Parameters.notNull("context", context); int startIndex = context.start.getStartIndex(); Token stopSymbol = getStopSymbol(context); if (stopSymbol == null) { return new Interval(startIndex, startIndex - 1); } int stopIndex; if (stopSymbol.getType() != Token.EOF) { stopIndex = stopSymbol.getStopIndex(); } else { TokenSource tokenSource = context.getStart().getTokenSource(); CharStream inputStream = tokenSource != null ? tokenSource.getInputStream() : null; if (inputStream != null) { stopIndex = inputStream.size() - 1; } else { stopIndex = context.start.getStartIndex() - 1; } } stopIndex = Math.max(stopIndex, startIndex - 1); return new Interval(startIndex, stopIndex); }
public static Interval getSourceInterval(@NonNull ParseTree context) { Parameters.notNull("context", context); if (context instanceof TerminalNode) { TerminalNode terminalNode = (TerminalNode)context; Token token = terminalNode.getSymbol(); return new Interval(token.getStartIndex(), token.getStopIndex()); } else if (context instanceof RuleNode) { RuleNode ruleNode = (RuleNode)context; RuleContext ruleContext = ruleNode.getRuleContext(); if (ruleContext instanceof ParserRuleContext) { return getSourceInterval((ParserRuleContext)ruleContext); } else { Token startSymbol = getStartSymbol(context); Token stopSymbol = getStopSymbol(context); if (startSymbol == null || stopSymbol == null) { return Interval.INVALID; } return new Interval(startSymbol.getStartIndex(), stopSymbol.getStopIndex()); } } else { return Interval.INVALID; } }
/** * Gets whether or not {@code a} starts after the start of {@code b}. * * @param a The first tree. * @param b The second tree. * @return {@code true} if {@code a} starts after the start of {@code b}, otherwise {@code false}. */ public static boolean startsAfterStartOf(@NonNull ParseTree a, @NonNull ParseTree b) { //TerminalNode<? extends Token> startNodeA = getStartNode(a); //TerminalNode<? extends Token> startNodeB = getStartNode(b); //if (startNodeA == null || startNodeB == null) { // throw new NotImplementedException(); //} Interval sourceIntervalA = a.getSourceInterval(); Interval sourceIntervalB = b.getSourceInterval(); //if (sourceIntervalA.a == sourceIntervalB.a) { // if (isAncestorOf(a, b)) { // return true; // } // // if (isEpsilon(a) || isEpsilon(b)) { // // b could be a child of a later sibling of some ancestor of a // throw new NotImplementedException(); // } //} return sourceIntervalA.a > sourceIntervalB.a; }
@Override public final void syntaxError(final Recognizer<?, ?> recognizer, final Object offendingSymbol, final int line, final int charPositionInLine, final String msg, final RecognitionException e) { String input; if (recognizer instanceof Lexer) { final CharStream cs = ((Lexer) recognizer).getInputStream(); input = cs.getText(new Interval(0, cs.size())); } else if (recognizer instanceof Parser) { final TokenStream tokens = ((Parser) recognizer).getInputStream(); if (tokens != null) { input = tokens.getText(); } else { input = "<unknown input>"; } } else { input = "<unknown input>"; } throw new AntlrParseException(input, line, charPositionInLine, msg); }
@NotNull @Override public String getText(Interval interval) { int start = interval.a; int stop = interval.b; if ( start<0 || stop<0 ) return ""; lazyInit(); if ( stop>=tokens.size() ) stop = tokens.size()-1; StringBuilder buf = new StringBuilder(); for (int i = start; i <= stop; i++) { Token t = tokens.get(i); if ( t.getType()==Token.EOF ) break; buf.append(t.getText()); } return buf.toString(); }
@Override public CommonToken create(Pair<TokenSource, CharStream> source, int type, String text, int channel, int start, int stop, int line, int charPositionInLine) { CommonToken t = new CommonToken(source, type, channel, start, stop); t.setLine(line); t.setCharPositionInLine(charPositionInLine); if ( text!=null ) { t.setText(text); } else if ( copyText && source.b != null ) { t.setText(source.b.getText(Interval.of(start,stop))); } return t; }
@Override public void reportAmbiguity(@NotNull Parser recognizer, @NotNull DFA dfa, int startIndex, int stopIndex, boolean exact, @Nullable BitSet ambigAlts, @NotNull ATNConfigSet configs) { if (exactOnly && !exact) { return; } String format = "reportAmbiguity d=%s: ambigAlts=%s, input='%s'"; String decision = getDecisionDescription(recognizer, dfa); BitSet conflictingAlts = getConflictingAlts(ambigAlts, configs); String text = recognizer.getTokenStream().getText(Interval.of(startIndex, stopIndex)); String message = String.format(format, decision, conflictingAlts, text); recognizer.notifyErrorListeners(message); }
@Override public String getText() { if ( text!=null ) { return text; } CharStream input = getInputStream(); if ( input==null ) return null; int n = input.size(); if ( start<n && stop<n) { return input.getText(Interval.of(start,stop)); } else { return "<EOF>"; } }
@Override public String getText(Interval interval) { if (interval.a < 0 || interval.b < interval.a - 1) { throw new IllegalArgumentException("invalid interval"); } int bufferStartIndex = getBufferStartIndex(); if (n > 0 && data[n - 1] == Character.MAX_VALUE) { if (interval.a + interval.length() > bufferStartIndex + n) { throw new IllegalArgumentException("the interval extends past the end of the stream"); } } if (interval.a < bufferStartIndex || interval.b >= bufferStartIndex + n) { throw new UnsupportedOperationException("interval "+interval+" outside buffer: "+ bufferStartIndex+".."+(bufferStartIndex+n-1)); } // convert from absolute to local index int i = interval.a - bufferStartIndex; return new String(data, i, interval.length()); }
private void parseFile(File child) { try { currentJavaFile = child; intervals = new ArrayList<Interval>(); input = new ANTLRFileStream(child.getPath()); Java7Lexer lexer = new Java7Lexer(input); CommonTokenStream tokens = new CommonTokenStream(lexer); Java7Parser parser = new Java7Parser(tokens); ParserRuleContext tree = parser.compilationUnit(); ParseTreeWalker walker = new ParseTreeWalker(); walker.walk(this, tree); if (intervals.size()>0) { writeOutputFile(); } } catch (IOException e) { System.err.println("Could not parse " + child.getPath()); e.printStackTrace(); } }
public void highlightAndOfferHint(Editor editor, int offset, Interval sourceInterval, JBColor color, EffectType effectType, String hintText) { CaretModel caretModel = editor.getCaretModel(); final TextAttributes attr = new TextAttributes(); attr.setForegroundColor(color); attr.setEffectColor(color); attr.setEffectType(effectType); MarkupModel markupModel = editor.getMarkupModel(); markupModel.addRangeHighlighter( sourceInterval.a, sourceInterval.b, InputPanel.TOKEN_INFO_LAYER, // layer attr, HighlighterTargetArea.EXACT_RANGE ); if ( hintText.contains("<") ) { hintText = hintText.replaceAll("<", "<"); } // HINT caretModel.moveToOffset(offset); // info tooltip only shows at cursor :( HintManager.getInstance().showInformationHint(editor, hintText); }
public void setCursorToGrammarElement(Project project, PreviewState previewState, int offset) { Token tokenUnderCursor = ParsingUtils.getTokenUnderCursor(previewState, offset); if ( tokenUnderCursor==null ) { return; } PreviewParser parser = (PreviewParser) previewState.parsingResult.parser; Integer atnState = parser.inputTokenToStateMap.get(tokenUnderCursor); if ( atnState==null ) { // likely an error token //LOG.error("no ATN state for input token " + tokenUnderCursor); return; } Interval region = previewState.g.getStateToGrammarRegion(atnState); CommonToken token = (CommonToken) previewState.g.tokenStream.get(region.a); jumpToGrammarPosition(project, token.getStartIndex()); }
@Override public Object visitProcedureConst(ProcedureConstContext ctx) { Token startToken = ctx.getStart(); Token stopToken = ctx.getStop(); // we need skipped token, such as WhiteSpace String rawText = stopToken .getTokenSource() .getInputStream() .getText( Interval.of(startToken.getStartIndex(), stopToken.getStopIndex())); CreateObject ins = new CreateObject(ctx.start.getLine()); ins.varName = getNextTempVar("link"); ins.objType = ObjectType.PROCEDURE; ins.value = rawText; addInstruction(ins); return ins.varName; }
public static boolean isOpNext(TokenStream tokens) { int start = tokens.index(); Token lt = tokens.get(start); int stop = getLastOpTokenIndex(tokens); if ( stop==-1 ) return false; System.out.printf("isOpNext: i=%d t='%s'", start, lt.getText()); System.out.printf(", op='%s'\n", tokens.getText(Interval.of(start,stop))); return true; }
/** "If an operator has whitespace around both sides or around neither side, it is treated as a binary operator. As an example, the + operator in a+b and a + b is treated as a binary operator." */ public static boolean isBinaryOp(TokenStream tokens) { int stop = getLastOpTokenIndex(tokens); if ( stop==-1 ) return false; int start = tokens.index(); Token prevToken = tokens.get(start-1); // includes hidden-channel tokens Token nextToken = tokens.get(stop+1); boolean prevIsWS = isLeftOperatorWS(prevToken); boolean nextIsWS = isRightOperatorWS(nextToken); boolean result = prevIsWS && nextIsWS || (!prevIsWS && !nextIsWS); String text = tokens.getText(Interval.of(start, stop)); //System.out.println("isBinaryOp: '"+prevToken+"','"+text+"','"+nextToken+"' is "+result); return result; }