@Override public SortedMap<Integer, String> parseString(String fieldString) { CharStream is = CharStreams.fromString(fieldString); DrawGrammarLexer lex = new DrawGrammarLexer(is); CommonTokenStream tokens = new CommonTokenStream(lex); DrawGrammarParser parser = new DrawGrammarParser(tokens); palette = Palette.makeDefaultPalette("DarkSpectrum"); final SortedMap<Integer, String> resultMap; try { resultMap = parser.root().map; } catch (RecognitionException | NullPointerException | StringIndexOutOfBoundsException | RasterFormatException re) { //Something wrong with the parsing do not update. return null; } return resultMap; }
/** * @param args the command line arguments * @throws java.io.IOException * @throws java.net.URISyntaxException */ public static void main(String[] args) throws IOException, URISyntaxException { final String entryPoint; final URL res; switch (args.length) { case 2: res = Paths.get(args[0]).toUri().toURL(); entryPoint = args[1]; break; default: System.err.println("Supply two parameters in the following order:\n- file name of the main function\n- name of the main function\n\nFor example: hello.ct main:argc:argv"); return; } final CharStream inp = CharStreams.fromStream(res.openStream()); final GrammarLexer lex = new GrammarLexer(inp); final TokenStream toks = new CommonTokenStream(lex); final GrammarParser parser = new GrammarParser(toks); System.out.println(new Translator(res.toURI()).generate(parser.program(), entryPoint)); }
/** * @throws InvalidQueryException if the given query contains invalid characters or is incomplete */ public SqlGrammarParser.SelectStatementContext parse(String query) { log.trace("About to parse [{}]", query); long now = currentTimeMillis(); CharStream input = CharStreams.fromString(query); SqlGrammarLexer lexer = new SqlGrammarLexer(input); TokenStream tokenStream = new CommonTokenStream(lexer); SqlGrammarParser parser = new SqlGrammarParser(tokenStream); parser.removeErrorListeners(); parser.addErrorListener(new RaiseExceptionErrorListener()); SqlGrammarParser.SelectStatementContext result = parser.selectStatement(); log.trace("Successfully parsed [{}] into [{}] in [{}ms]", query, result.toStringTree(parser), currentTimeMillis() - now); return result; }
@Override public Token nextToken() { if (stashedNext != null) { previous = stashedNext; stashedNext = null; return previous; } Token next = super.nextToken(); if (insertSemicolon(previous, next)) { stashedNext = next; previous = _factory.create(new Pair<TokenSource, CharStream>(this, _input), PainlessLexer.SEMICOLON, ";", Lexer.DEFAULT_TOKEN_CHANNEL, next.getStartIndex(), next.getStopIndex(), next.getLine(), next.getCharPositionInLine()); return previous; } else { previous = next; return next; } }
@Override public void recover(final LexerNoViableAltException lnvae) { final CharStream charStream = lnvae.getInputStream(); final int startIndex = lnvae.getStartIndex(); final String text = charStream.getText(Interval.of(startIndex, charStream.index())); Location location = new Location(sourceName, _tokenStartCharIndex); String message = "unexpected character [" + getErrorDisplay(text) + "]."; char firstChar = text.charAt(0); if ((firstChar == '\'' || firstChar == '"') && text.length() - 2 > 0 && text.charAt(text.length() - 2) == '\\') { /* Use a simple heuristic to guess if the unrecognized characters were trying to be a string but has a broken escape sequence. * If it was add an extra message about valid string escape sequences. */ message += " The only valid escape sequences in strings starting with [" + firstChar + "] are [\\\\] and [\\" + firstChar + "]."; } throw location.createError(new IllegalArgumentException(message, lnvae)); }
private void parseInputStream(CharStream inputStream, OboParseResultListener listener) { final OboLexer l = new OboLexer(inputStream); final Antlr4OboParser p = new Antlr4OboParser(new CommonTokenStream(l)); p.addErrorListener(new BaseErrorListener() { @Override public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) { throw new IllegalStateException("Failed to parse at line " + line + " due to " + msg, e); } }); if (debug) { p.addErrorListener(new DiagnosticErrorListener()); } p.addParseListener(new OboParserListener(listener)); p.oboFile(); }
public EObject parse(final String typeExpression, final Scope scope) { final CharStream charStream = CharStreams.fromString(typeExpression); final TypeExpressionLexer lexer = new TypeExpressionLexer(charStream); final TokenStream tokenStream = new CommonTokenStream(lexer); final TypeExpressionParser typeExpressionParser = new TypeExpressionParser(tokenStream); lexer.removeErrorListeners(); typeExpressionParser.removeErrorListeners(); final ParserErrorCollector errorCollector = new ParserErrorCollector(); lexer.addErrorListener(errorCollector); typeExpressionParser.addErrorListener(errorCollector); final TypeExpressionParser.Type_exprContext typeExpr = typeExpressionParser.type_expr(); final EObject anyType = new TypeExpressionBuilder(scope, ARRAY_TYPE).visit(typeExpr); scope.getResource().getErrors().addAll(errorCollector.getErrors()); return anyType; }
public static <L extends Lexer, P extends Parser> P newParser( Function<CharStream, L> lexerFactory, Function<TokenStream, P> parserFactory, String input, boolean useBailErrorStrategy, boolean removeErrorListeners) { CharStream charStream = new ANTLRInputStream(input); L lexer = lexerFactory.apply(charStream); if (removeErrorListeners) { lexer.removeErrorListeners(); } TokenStream tokenStream = new CommonTokenStream(lexer); P parser = parserFactory.apply(tokenStream); if (useBailErrorStrategy) { parser.setErrorHandler(new BailErrorStrategy()); } if (removeErrorListeners) { parser.removeErrorListeners(); } return parser; }
/** * Parse an interval, for example <tt>[1,-]</tt> or <tt>-</tt> (a wildcard) or <tt>[1,4]</tt>. * Only fixed values are allowed, no variables. * * @param intervalAsString the string to be parsed. * @return a LowerBoundedInterval as the runtime representation of interval strings. * @throws ParseException in case the string doesn't fit the given fixed-interval grammar. */ public static LowerBoundedInterval parse(String intervalAsString) throws ParseException { CharStream charStream = new ANTLRInputStream(intervalAsString); CellExpressionLexer lexer = new CellExpressionLexer(charStream); TokenStream tokens = new CommonTokenStream(lexer); CellExpressionParser parser = new CellExpressionParser(tokens); parser.removeErrorListeners(); parser.addErrorListener(new ThrowingErrorListener()); try { CellExpressionParser.Fixed_intervalContext ctx = parser.fixed_interval(); if (ctx == null) { throw new ParseException(0, 0, "Expected fixed interval"); } return INSTANCE.visit(ctx); } catch (ParseRuntimeException runtimeException) { throw runtimeException.getParseException(); } }
public static FilterPredicate compile(String predicateString){ predicateString = predicateString.trim(); if(predicateString.isEmpty()){ return FilterPredicate.empty(); } PredicateCompileErrorListener errorListener = new PredicateCompileErrorListener(predicateString); CharStream stream = CharStreams.fromString(predicateString); PredicateLexer lexer = new PredicateLexer(stream); lexer.removeErrorListener(ConsoleErrorListener.INSTANCE); lexer.addErrorListener(errorListener); CommonTokenStream tokens = new CommonTokenStream(lexer); PredicateParser parser = new PredicateParser(tokens); parser.removeErrorListener(ConsoleErrorListener.INSTANCE); parser.addErrorListener(errorListener); FilterPredicateVisitor filterPredicateVisitor = new FilterPredicateVisitor(); return filterPredicateVisitor.visit(parser.compilationUnit()); }
@Test public void test() { final CharStream charStream = CharStreams.fromString("\r\nSELECT\r\n 1"); final TSqlLexer lexer = new TSqlLexer(charStream); final CommonTokenStream stream = new CommonTokenStream(lexer); stream.fill(); TSqlParser parser = new TSqlParser(stream); ParseTree child = parser.tsql_file().getChild(0); DefaultLinesProvider lines = new DefaultLinesProvider(stream); int line = lines.getLine(new ParsedNode(child)); Assert.assertEquals(2, line); }
/** * Parses a source file using ANTLR4, and walks over the parse tree to * interpret this source file as a Java object. * * @param c * input character stream * @param path * location of the file * @return an interpreted source file, or null in case of an error. */ @Nullable private ReoFile<T> parse(CharStream c, String path) { ReoLexer lexer = new ReoLexer(c); CommonTokenStream tokens = new CommonTokenStream(lexer); ReoParser parser = new ReoParser(tokens); ErrorListener errListener = new ErrorListener(m); parser.removeErrorListeners(); parser.addErrorListener(errListener); ParseTree tree = parser.file(); if (errListener.hasError) return null; ParseTreeWalker walker = new ParseTreeWalker(); listener.setFileName(path); walker.walk(listener, tree); return listener.getMain(); }
static MysqlParser.ScriptContext parse(CharStream cs) { MysqlLexer lexer = new MysqlLexer(cs); CommonTokenStream tokens = new CommonTokenStream(lexer); tokens.setTokenSource(lexer); MysqlParser parser = new MysqlParser(tokens); parser.setErrorHandler(new BailErrorStrategy()); boolean success = false; try { MysqlParser.ScriptContext script = parser.script(); success = true; return script; } finally { if (!success && (parser.lastStatement != null)) { _log.debug("last passed statement: {}", ((ParseTree)parser.lastStatement).getText()); } } }
private static byte[] getBytes(Literal_value_binaryContext rule) { Token token = rule.STRING_LITERAL().getSymbol(); byte[] bytes = new byte[token.getStopIndex() - token.getStartIndex() - 1]; CharStream cs = token.getInputStream(); int pos = cs.index(); cs.seek(token.getStartIndex() + 1); int j = 0; for (int i = 0; i < bytes.length; i++) { int ch = cs.LA(i + 1); if (ch == '\\') { i++; ch = cs.LA(i + 1); if (ch == '0') { ch = 0; } else if (ch == 'n') { ch = '\n'; } else if (ch == 'r') { ch = '\r'; } else if (ch == 'Z') { ch = '\032'; } } bytes[j] = (byte) ch; j++; } cs.seek(pos); if (j != bytes.length) { // esacpe characters byte[] old = bytes; bytes = new byte[j]; System.arraycopy(old, 0, bytes, 0, j); } return bytes; }
private void compileFiles(List<RawFile> files, OOPSourceCodeModel srcModel, List<String> projectFileTypes) { for (RawFile file : files) { try { CharStream charStream = new ANTLRInputStream(file.content()); GolangLexer lexer = new GolangLexer(charStream); TokenStream tokens = new CommonTokenStream(lexer); GolangParser parser = new GolangParser(tokens); SourceFileContext sourceFileContext = parser.sourceFile(); parser.setErrorHandler(new BailErrorStrategy()); parser.getInterpreter().setPredictionMode(PredictionMode.SLL); ParseTreeWalker walker = new ParseTreeWalker(); GolangBaseListener listener = new GoLangTreeListener(srcModel, projectFileTypes, file); walker.walk(listener, sourceFileContext); } catch (Exception e) { e.printStackTrace(); } } }
private SyntaxErrorListener getSyntaxTreeErrors(String iFlowResource) { InputStream inputStream = null; try { File file = new File(getClass().getResource(iFlowResource).getFile()); inputStream = new FileInputStream(file); CharStream cs = new ANTLRInputStream(inputStream); TokenStream tokenStream = new CommonTokenStream(new WUMLLexer(cs)); WUMLParser parser = new WUMLParser(tokenStream); SyntaxErrorListener errorListener = new SyntaxErrorListener(); parser.addErrorListener(errorListener); ParseTree tree = parser.sourceFile(); return errorListener; } catch (Exception e) { return null; } }
private static Program combineInput(boolean literate, String... fileNames) throws IOException { final Program result = new Program(); for (String fileName : fileNames) { CharStream stream; if (!literate) { stream = CharStreams.fromFileName(fileName); } else { stream = CharStreams.fromChannel( streamToChannel(literate(lines(Paths.get(fileName)))), 4096, CodingErrorAction.REPLACE, fileName ); } result.accumulate(PARSER.parse(stream)); } return result; }
public static FillerRequest createRequest(final SourceLinesProvider linesProvider, final InputFile file, final Charset encoding) throws IOException, FileNotFoundException { final CharStream mainStream = CharStreams.fromPath(file.path(), encoding); final CharStream charStream = new CaseChangingCharStream(mainStream, true); final TSqlLexer lexer = new TSqlLexer(charStream); lexer.removeErrorListeners(); final CommonTokenStream stream = new CommonTokenStream(lexer); stream.fill(); final TSqlParser parser = new TSqlParser(stream); parser.removeErrorListeners(); final ParseTree root = parser.tsql_file(); final FillerRequest antrlFile = new FillerRequest(file, stream, root, linesProvider.getLines(new FileInputStream(file.file()), encoding)); return antrlFile; }
private static TravnyParser.SchemaFileContext parseContext(String dsl) { CharStream cs = new ANTLRInputStream(dsl); TravnyLexer lexer = new TravnyLexer(cs); lexer.removeErrorListeners(); lexer.addErrorListener(new BreakingErrorListener()); CommonTokenStream tokens = new CommonTokenStream(lexer); TravnyParser parser = new TravnyParser(tokens); parser.removeErrorListeners(); parser.addErrorListener(new BreakingErrorListener()); TravnyParser.SchemaFileContext parsed = parser.schemaFile(); return parsed; }
static long getResult(String equationString) throws LogicGrammarParsingException { CharStream is = CharStreams.fromString(equationString); LogicGrammarLexer lex = new LogicGrammarLexer(is); CommonTokenStream tokens = new CommonTokenStream(lex); LogicGrammarParser parser = new LogicGrammarParser(tokens); System.out.println("Parsing equation: " + equationString); long final_result; try { final_result = parser.root().result; } catch (RecognitionException | NumberFormatException nfe) { throw new LogicGrammarParsingException(); } return final_result; }
public String processCharStream(CharStream st) throws RecognitionException { final GrammarLexer lex = new GrammarLexer(st); final TokenStream toks = new CommonTokenStream(lex); final GrammarParser parser = new GrammarParser(toks); final String body = this.visitProgram(parser.program()); return body; }
private int read() { int result = input.read(); if (result == LexerInput.EOF) { result = CharStream.EOF; } return result; }
private Constructor<? extends Lexer> getLexerCtor() { if (lexerCtor == null) { try { lexerCtor = lexerClass.getConstructor(CharStream.class); } catch (NoSuchMethodException | SecurityException e) { throw new RuntimeException(e); } } return lexerCtor; }
private static Expressie doParse(final String expressieString, final Context context) { // TEAMBRP-2535 de expressie syntax kan niet goed omgaan met een string waarachter ongedefinieerde velden staan. // Door haakjes toe te voegen zal dan een fout gesignaleerd worden, aangezien de content dan niet meer precies // gematched kan worden. final String expressieStringMetHaakjes = String.format("(%s)", expressieString); // Parsing gebeurt met een door ANTLR gegenereerde visitor. Om die te kunnen gebruiken, moet een treintje // opgetuigd worden (String->CharStream->Lexer->TokenStream). final CharStream cs = CharStreams.fromString(expressieStringMetHaakjes); final ParserErrorListener parserErrorListener = new ParserErrorListener(); final BRPExpressietaalLexer lexer = new BRPExpressietaalLexer(cs); // verwijdert de interne listener van de lexer die naar system/out/err wegschrijft // expressies als ***bla*** logt bijvoorbeeld unrecognized token errors naar system/out // in plaats hiervan neem een eigen error listener op. lexer.removeErrorListeners(); lexer.addErrorListener(parserErrorListener); final CommonTokenStream tokens = new CommonTokenStream(lexer); final BRPExpressietaalParser parser = new BRPExpressietaalParser(tokens); parser.getInterpreter().setPredictionMode(PredictionMode.LL_EXACT_AMBIG_DETECTION); // Verwijder bestaande (default) error listeners en voeg de eigen error listener toe. parser.removeErrorListeners(); parser.addErrorListener(parserErrorListener); // Maak de parse tree. Hier gebeurt het feitelijke parsing. final BRPExpressietaalParser.Brp_expressieContext tree = parser.brp_expressie(); // Maak een visitor voor parsing. final ExpressieVisitor visitor = new ExpressieVisitor(context); // De visitor zet een parse tree om in een Expressie. Tenzij er een fout optreedt. return visitor.visit(tree); }
public static SQLTemplate parse(String sql) { if (StringUtils.isBlank(sql)) { throw new ElasticSearchSQLException("blank sql is not allowed"); } CharStream input = CharStreams.fromString(sql); ThrowingExceptionErrorStrategy errorHandler = new ThrowingExceptionErrorStrategy(); ElasticSearchLexer lexer = new ElasticSearchLexer(input); lexer.removeErrorListeners(); // Lexer error handler lexer.addErrorListener(new LexerErrorListener()); CommonTokenStream token = new CommonTokenStream(lexer); ElasticSearchParser parser = new ElasticSearchParser(token); // Parser error handler parser.setErrorHandler(errorHandler); parser.removeErrorListeners(); parser.addErrorListener(new SyntaxErrorListener()); ElasticSearchParser.ProgContext progTree = parser.prog(); ElasticSearchVisitor visitor = new ElasticSearchVisitor(); ParserResult parsedResult = visitor.visit(progTree); if (parsedResult.isSuccess()) { return parsedResult.getTemplate(); } throw new ElasticSearchSQLException("SQL parser error !!! " + parsedResult.getFailureReason()); }
public static String getSourceText(ParserRuleContext ctx){ if (ctx.start == null || ctx.stop == null) { return ctx.getText(); } int startIndex = ctx.start.getStartIndex(); int stopIndex = ctx.stop.getStopIndex(); if (stopIndex < startIndex) { return ""; // Just return the empty string. } CharStream inputStream = ctx.start.getInputStream(); return inputStream.getText(new Interval(startIndex, stopIndex)); }
@Override public void parse( Snapshot snapshot, Task task, SourceModificationEvent event) throws ParseException { Assert.notNull(snapshot); this.snapshot = snapshot; CharStream input = new ANTLRInputStream(snapshot.getText().toString()); Lexer lexer = new ProtobufLexer(input); CommonTokenStream tokens = new CommonTokenStream(lexer); ProtobufParser parser = new ProtobufParser(tokens); parser.removeErrorListeners(); ParsingErrorListener listener = new ParsingErrorListener(); parser.addErrorListener(listener); ParseTree tree = parser.proto(); // TODO def and ref phases this.errors.clear(); this.errors.addAll(listener.errors()); }
/** * Parses while matching an AQL expression. * * @param queryEnvironment * the {@link IReadOnlyQueryEnvironment} * @param type * the type to parse * @return the corresponding {@link AstResult} */ private AstResult parseWhileAqlTypeLiteral(IReadOnlyQueryEnvironment queryEnvironment, String type) { final IQueryBuilderEngine.AstResult result; if (type != null && type.length() > 0) { AstBuilderListener astBuilder = AQL56Compatibility .createAstBuilderListener((IQueryEnvironment) queryEnvironment); CharStream input = new UnbufferedCharStream(new StringReader(type), type.length()); QueryLexer lexer = new QueryLexer(input); lexer.setTokenFactory(new CommonTokenFactory(true)); lexer.removeErrorListeners(); lexer.addErrorListener(astBuilder.getErrorListener()); TokenStream tokens = new UnbufferedTokenStream<CommonToken>(lexer); QueryParser parser = new QueryParser(tokens); parser.addParseListener(astBuilder); parser.removeErrorListeners(); parser.addErrorListener(astBuilder.getErrorListener()); // parser.setTrace(true); parser.typeLiteral(); result = astBuilder.getAstResult(); } else { ErrorTypeLiteral errorTypeLiteral = (ErrorTypeLiteral) EcoreUtil .create(AstPackage.eINSTANCE.getErrorTypeLiteral()); List<org.eclipse.acceleo.query.ast.Error> errors = new ArrayList<org.eclipse.acceleo.query.ast.Error>(1); errors.add(errorTypeLiteral); final Map<Object, Integer> positions = new HashMap<Object, Integer>(); if (type != null) { positions.put(errorTypeLiteral, Integer.valueOf(0)); } final BasicDiagnostic diagnostic = new BasicDiagnostic(); diagnostic.add(new BasicDiagnostic(Diagnostic.ERROR, AstBuilderListener.PLUGIN_ID, 0, "null or empty type.", new Object[] {errorTypeLiteral })); result = new AstResult(errorTypeLiteral, positions, positions, errors, diagnostic); } return result; }
private static AntrlResult getFromStream(final CharStream charStream) { final TSqlLexer lexer = new TSqlLexer(charStream); final CommonTokenStream stream = new CommonTokenStream(lexer); stream.fill(); final TSqlParser parser = new TSqlParser(stream); AntrlResult result = new AntrlResult(); result.setTree(parser.tsql_file()); result.setStream(stream); result.setParser(parser); return result; }
public static void main(String[] args) { String someLangSourceCode = "a=10;\nb=20;\na * b"; CharStream charStream = new ANTLRInputStream(someLangSourceCode); CalculatorLexer lexer = new CalculatorLexer(charStream); TokenStream tokens = new CommonTokenStream(lexer); CalculatorParser parser = new CalculatorParser(tokens); MyCalculatorVisitor visitor = new MyCalculatorVisitor(); BigDecimal result = visitor.visit(parser.root()); System.out.println(result); }
public static void main(String[] args) { String someLangSourceCode = "1 + 2 - 3 + 4 * SUM(1,2,3,4,5,6,7,8,9,10) / 8"; CharStream charStream = new ANTLRInputStream(someLangSourceCode); ExpressionLexer lexer = new ExpressionLexer(charStream); TokenStream tokens = new CommonTokenStream(lexer); ExpressionParser parser = new ExpressionParser(tokens); ItemExpressionVisitor visitor = new ItemExpressionVisitor(); BigDecimal result = visitor.visit(parser.root()); System.out.println(result); }
@Override public int LA(int i) { if (i == 0) { return 0; } if (i < 0) { i++; // e.g., translate LA(-1) to use offset 0 } if ((p + i - 1) >= n) { return CharStream.EOF; } return Character.toLowerCase(data[p + i - 1]); }
public ConditionValidator(Map<String, FieldValidatorImpl> fieldValidatorMap, String id, String name, String expression, String error) { super(id); this.fieldValidatorMap = fieldValidatorMap; this.expression = expression; this.error = error; final CharStream charStream = new ANTLRInputStream(expression); logger.info("Constructing new lexer and parser for expression " + expression); checklistConditionLexer = new ChecklistConditionLexer(charStream); final CommonTokenStream commonTokenStream = new CommonTokenStream(checklistConditionLexer); checklistConditionParser = new ChecklistConditionParser(commonTokenStream); }
public void query(ChannelHandlerContext ctx, QueryPacket packet) throws Exception { CharStream sql= packet.getSql(); Object result = null; if (sql == null) { serverHandler.writeErrMessage(ctx, MysqlErrorCode.ER_ERROR_WHEN_EXECUTING_COMMAND, "Empty query."); } else { result = serverHandler.session.run(sql); Helper.writeResonpse(ctx, serverHandler, result, true); } }
private Integer asyncInsert(CharStream cs) { if (this.asyncExecutor.getError() != null) { throw new OrcaException(this.asyncExecutor.getError()); } this.asyncExecutor.add(cs); return 1; }
@Override public Script parse(Session session, CharStream cs) { ScriptContext scriptCtx = parse(cs); GeneratorContext ctx = new GeneratorContext(session); InstructionGenerator.scan(ctx, scriptCtx); Instruction code = (Instruction)InstructionGenerator.generate(ctx, scriptCtx); Script script = new Script(code, ctx.getParameterCount(), ctx.getVariableCount(), cs.toString()); return script; }
@Override public Object run(VdmContext ctx, Parameters params, CharStream cs) { ScriptContext rule = parse(cs); GeneratorContext gctx = new GeneratorContext(ctx.getSession()); gctx.setCompileDdl(true); InstructionGenerator.scan(gctx, rule); Object result = run(gctx, ctx, params, rule); return result; }