protected String getStateLabel(ATNState s) { if ( s==null ) return "null"; String stateLabel = ""; if (s instanceof BlockStartState) { stateLabel += "→\\n"; } else if (s instanceof BlockEndState) { stateLabel += "←\\n"; } stateLabel += String.valueOf(s.stateNumber); if (s instanceof PlusBlockStartState || s instanceof PlusLoopbackState) { stateLabel += "+"; } else if (s instanceof StarBlockStartState || s instanceof StarLoopEntryState || s instanceof StarLoopbackState) { stateLabel += "*"; } if ( s instanceof DecisionState && ((DecisionState)s).decision>=0 ) { stateLabel = stateLabel+"\\nd="+((DecisionState)s).decision; } return stateLabel; }
/** identify the ATN states where we need to set the outer alt number. * For regular rules, that's the block at the target to rule start state. * For left-recursive rules, we track the primary block, which looks just * like a regular rule's outer block, and the star loop block (always * there even if 1 alt). */ public BitSet findOuterMostDecisionStates() { BitSet track = new BitSet(atn.states.size()); int numberOfDecisions = atn.getNumberOfDecisions(); for (int i = 0; i < numberOfDecisions; i++) { DecisionState decisionState = atn.getDecisionState(i); RuleStartState startState = atn.ruleToStartState[decisionState.ruleIndex]; // Look for StarLoopEntryState that is in any left recursive rule if ( decisionState instanceof StarLoopEntryState) { StarLoopEntryState loopEntry = (StarLoopEntryState)decisionState; if ( loopEntry.isPrecedenceDecision ) { // Recursive alts always result in a (...)* in the transformed // left recursive rule and that always has a BasicBlockStartState // even if just 1 recursive alt exists. ATNState blockStart = loopEntry.transition(0).target; // track the StarBlockStartState associated with the recursive alternatives track.set(blockStart.stateNumber); } } else if ( startState.transition(0).target == decisionState ) { // always track outermost block for any rule if it exists track.set(decisionState.stateNumber); } } return track; }
public LL1OptionalBlockSingleAlt(OutputModelFactory factory, GrammarAST blkAST, List<CodeBlockForAlt> alts) { super(factory, blkAST, alts); this.decision = ((DecisionState)blkAST.atnState).decision; /** Lookahead for each alt 1..n */ // IntervalSet[] altLookSets = LinearApproximator.getLL1LookaheadSets(dfa); IntervalSet[] altLookSets = factory.getGrammar().decisionLOOK.get(decision); altLook = getAltLookaheadAsStringLists(altLookSets); IntervalSet look = altLookSets[0]; IntervalSet followLook = altLookSets[1]; IntervalSet expecting = look.or(followLook); this.error = getThrowNoViableAlt(factory, blkAST, expecting); expr = addCodeForLookaheadTempVar(look); followExpr = factory.getLL1Test(followLook, blkAST); }
@Override public Choice getChoiceBlock(BlockAST blkAST, List<CodeBlockForAlt> alts, GrammarAST labelAST) { int decision = ((DecisionState)blkAST.atnState).decision; Choice c; if ( !g.tool.force_atn && AnalysisPipeline.disjoint(g.decisionLOOK.get(decision)) ) { c = getLL1ChoiceBlock(blkAST, alts); } else { c = getComplexChoiceBlock(blkAST, alts); } if ( labelAST!=null ) { // for x=(...), define x or x_list String label = labelAST.getText(); Decl d = getTokenLabelDecl(label); c.label = d; getCurrentRuleFunction().addContextDecl(labelAST.getAltLabel(), d); if ( labelAST.parent.getType() == ANTLRParser.PLUS_ASSIGN ) { String listLabel = gen.getTarget().getListLabel(label); TokenListDecl l = new TokenListDecl(this, listLabel); getCurrentRuleFunction().addContextDecl(labelAST.getAltLabel(), l); } } return c; }
@Override public Choice getEBNFBlock(GrammarAST ebnfRoot, List<CodeBlockForAlt> alts) { if (!g.tool.force_atn) { int decision; if ( ebnfRoot.getType()==ANTLRParser.POSITIVE_CLOSURE ) { decision = ((PlusLoopbackState)ebnfRoot.atnState).decision; } else if ( ebnfRoot.getType()==ANTLRParser.CLOSURE ) { decision = ((StarLoopEntryState)ebnfRoot.atnState).decision; } else { decision = ((DecisionState)ebnfRoot.atnState).decision; } if ( AnalysisPipeline.disjoint(g.decisionLOOK.get(decision)) ) { return getLL1EBNFBlock(ebnfRoot, alts); } } return getComplexEBNFBlock(ebnfRoot, alts); }
/** Override this method so that we can record which alternative * was taken at each decision point. For non-left recursive rules, * it's simple. Set decisionStatesThatSetOuterAltNumInContext * indicates which decision states should set the outer alternative number. * * <p>Left recursive rules are much more complicated to deal with: * there is typically a decision for the primary alternatives and a * decision to choose between the recursive operator alternatives. * For example, the following left recursive rule has two primary and 2 * recursive alternatives.</p> * e : e '*' e | '-' INT | e '+' e | ID ; * <p>ANTLR rewrites that rule to be</p> e[int precedence] : ('-' INT | ID) ( {...}? '*' e[5] | {...}? '+' e[3] )* ; * * <p>So, there are two decisions associated with picking the outermost alt. * This complicates our tracking significantly. The outermost alternative number * is a function of the decision (ATN state) within a left recursive rule and the * predicted alternative coming back from adaptivePredict(). * * We use stateToAltsMap as a cache to avoid expensive calls to * getRecursiveOpAlts(). */ @Override protected int visitDecisionState(DecisionState p) { int predictedAlt = super.visitDecisionState(p); if( p.getNumberOfTransitions() > 1) { // System.out.println("decision "+p.decision+": "+predictedAlt); if( p.decision == this.overrideDecision && this._input.index() == this.overrideDecisionInputIndex ) { overrideDecisionRoot = (GrammarInterpreterRuleContext)getContext(); } } GrammarInterpreterRuleContext ctx = (GrammarInterpreterRuleContext)_ctx; if ( decisionStatesThatSetOuterAltNumInContext.get(p.stateNumber) ) { ctx.outerAltNum = predictedAlt; Rule r = g.getRule(p.ruleIndex); if ( atn.ruleToStartState[r.index].isLeftRecursiveRule ) { int[] alts = stateToAltsMap[p.stateNumber]; LeftRecursiveRule lr = (LeftRecursiveRule) g.getRule(p.ruleIndex); if (p.getStateType() == ATNState.BLOCK_START) { if ( alts==null ) { alts = lr.getPrimaryAlts(); stateToAltsMap[p.stateNumber] = alts; // cache it } } else if ( p.getStateType() == ATNState.STAR_BLOCK_START ) { if ( alts==null ) { alts = lr.getRecursiveOpAlts(); stateToAltsMap[p.stateNumber] = alts; // cache it } } ctx.outerAltNum = alts[predictedAlt]; } } return predictedAlt; }
public LL1AltBlock(OutputModelFactory factory, GrammarAST blkAST, List<CodeBlockForAlt> alts) { super(factory, blkAST, alts); this.decision = ((DecisionState)blkAST.atnState).decision; /** Lookahead for each alt 1..n */ IntervalSet[] altLookSets = factory.getGrammar().decisionLOOK.get(decision); altLook = getAltLookaheadAsStringLists(altLookSets); IntervalSet expecting = IntervalSet.or(altLookSets); // combine alt sets this.error = getThrowNoViableAlt(factory, blkAST, expecting); }
public static int getQidDecision(@NonNull ATN atn) { ATNState decisionState = atn.ruleToStartState[GoParser.RULE_qualifiedIdentifier].transition(0).target; if (decisionState instanceof DecisionState) { return ((DecisionState)decisionState).decision; } else { return -1; } }
@Override protected int visitDecisionState(DecisionState p) { int predictedAlt = super.visitDecisionState(p); if ( p.getNumberOfTransitions()>1 ) { // System.out.println("decision "+p.decision+": "+predictedAlt); if ( p.decision==this.overrideDecision && this._input.index()==this.overrideDecisionInputIndex ) { ((PreviewInterpreterRuleContext)overrideDecisionRoot).isDecisionOverrideRoot = true; } } return predictedAlt; }
/** Return a list of parse trees, one for each alternative in a decision * given the same input. * * Very similar to {@link #getAllPossibleParseTrees} except * that it re-parses the input for every alternative in a decision, * not just the ambiguous ones (there is no alts parameter here). * This method also tries to reduce the size of the parse trees * by stripping away children of the tree that are completely out of range * of startIndex..stopIndex. Also, because errors are expected, we * use a specialized error handler that more or less bails out * but that also consumes the first erroneous token at least. This * ensures that an error node will be in the parse tree for display. * * NOTES: // we must parse the entire input now with decision overrides // we cannot parse a subset because it could be that a decision // above our decision of interest needs to read way past // lookaheadInfo.stopIndex. It seems like there is no escaping // the use of a full and complete token stream if we are // resetting to token index 0 and re-parsing from the start symbol. // It's not easy to restart parsing somewhere in the middle like a // continuation because our call stack does not match the // tree stack because of left recursive rule rewriting. grrrr! * * @since 4.5.1 */ public static List<ParserRuleContext> getLookaheadParseTrees(Grammar g, ParserInterpreter originalParser, TokenStream tokens, int startRuleIndex, int decision, int startIndex, int stopIndex) { List<ParserRuleContext> trees = new ArrayList<ParserRuleContext>(); // Create a new parser interpreter to parse the ambiguous subphrase ParserInterpreter parser = deriveTempParserInterpreter(g, originalParser, tokens); BailButConsumeErrorStrategy errorHandler = new BailButConsumeErrorStrategy(); parser.setErrorHandler(errorHandler); DecisionState decisionState = originalParser.getATN().decisionToState.get(decision); for (int alt=1; alt<=decisionState.getTransitions().length; alt++) { // re-parse entire input for all ambiguous alternatives // (don't have to do first as it's been parsed, but do again for simplicity // using this temp parser.) parser.reset(); parser.addDecisionOverride(decision, startIndex, alt); ParserRuleContext tt = parser.parse(startRuleIndex); int stopTreeAt = stopIndex; if ( errorHandler.firstErrorTokenIndex>=0 ) { stopTreeAt = errorHandler.firstErrorTokenIndex; // cut off rest at first error } ParserRuleContext subtree = Trees.getRootOfSubtreeEnclosingRegion(tt, startIndex, stopTreeAt); // Use higher of overridden decision tree or tree enclosing all tokens if ( Trees.isAncestorOf(parser.getOverrideDecisionRoot(), subtree) ) { subtree = parser.getOverrideDecisionRoot(); } Trees.stripChildrenOutOfRange(subtree, parser.getOverrideDecisionRoot(), startIndex, stopTreeAt); trees.add(subtree); } return trees; }
public DFA(@NotNull DecisionState atnStartState) { this(atnStartState, 0); }
public DFA(@NotNull DecisionState atnStartState, int decision) { this.atnStartState = atnStartState; this.decision = decision; }
public void selectDecisionInGrammar(PreviewState previewState, int decision) { final ANTLRv4PluginController controller = ANTLRv4PluginController.getInstance(previewState.project); if ( controller==null ) return; final Editor grammarEditor = controller.getEditor(previewState.grammarFile); if ( grammarEditor==null ) return; DecisionState decisionState = previewState.g.atn.getDecisionState(decision); Interval region = previewState.g.getStateToGrammarRegion(decisionState.stateNumber); if ( region==null ) { System.err.println("decision "+decision+" has state "+decisionState.stateNumber+" but no region"); return; } InputPanel.removeHighlighters(grammarEditor, ProfilerPanel.DECISION_INFO_KEY); org.antlr.runtime.TokenStream tokens = previewState.g.tokenStream; if ( region.a>=tokens.size() || region.b>=tokens.size() ) { // System.out.println("out of range: " + region + " tokens.size()=" + tokens.size()); return; } CommonToken startToken = (CommonToken) tokens.get(region.a); CommonToken stopToken = (CommonToken) tokens.get(region.b); JBColor effectColor = JBColor.darkGray; DecisionInfo decisionInfo = previewState.parsingResult.parser.getParseInfo().getDecisionInfo()[decision]; if ( decisionInfo.predicateEvals.size()>0 ) { effectColor = new JBColor(PREDEVAL_COLOR, AMBIGUITY_COLOR); } if ( decisionInfo.contextSensitivities.size()>0 ) { effectColor = new JBColor(FULLCTX_COLOR, AMBIGUITY_COLOR); } if ( decisionInfo.ambiguities.size()>0 ) { effectColor = new JBColor(AMBIGUITY_COLOR, AMBIGUITY_COLOR); } TextAttributes attr = new TextAttributes(JBColor.BLACK, JBColor.WHITE, effectColor, EffectType.ROUNDED_BOX, Font.PLAIN); MarkupModel markupModel = grammarEditor.getMarkupModel(); final RangeHighlighter rangeHighlighter = markupModel.addRangeHighlighter( startToken.getStartIndex(), stopToken.getStopIndex()+1, HighlighterLayer.SELECTION, // layer attr, HighlighterTargetArea.EXACT_RANGE ); rangeHighlighter.putUserData(DECISION_INFO_KEY, decisionInfo); // System.out.println("dec " + decision + " from " + startToken + " to " + stopToken); ScrollingModel scrollingModel = grammarEditor.getScrollingModel(); CaretModel caretModel = grammarEditor.getCaretModel(); caretModel.moveToOffset(startToken.getStartIndex()); scrollingModel.scrollToCaret(ScrollType.MAKE_VISIBLE); }