public static void main(String[] args) throws Exception { LangDescriptor[] languages = new LangDescriptor[] { JAVA_DESCR, JAVA8_DESCR, JAVA_GUAVA_DESCR, JAVA8_GUAVA_DESCR, ANTLR4_DESCR, SQLITE_CLEAN_DESCR, TSQL_CLEAN_DESCR, SQLITE_NOISY_DESCR, TSQL_NOISY_DESCR, // QUORUM_DESCR, }; List<String> corpusDirs = map(languages, l -> l.corpusDir); String[] dirs = corpusDirs.toArray(new String[languages.length]); String python = testAllLanguages(languages, dirs, "leave_one_out.pdf"); String fileName = "python/src/leave_one_out.py"; Utils.writeFile(fileName, python); System.out.println("wrote python code to "+fileName); }
public static void format(LangDescriptor language, String testFileName, String outputFileName) throws Exception { // load all files up front List<String> allFiles = getFilenames(new File(language.corpusDir), language.fileRegex); List<InputDocument> documents = load(allFiles, language); // if in corpus, don't include in corpus final String path = new File(testFileName).getAbsolutePath(); List<InputDocument> others = filter(documents, d -> !d.fileName.equals(path)); InputDocument testDoc = parse(testFileName, language); Corpus corpus = new Corpus(others, language); corpus.train(); Formatter formatter = new Formatter(corpus, language.indentSize, Formatter.DEFAULT_K, FEATURES_INJECT_WS, FEATURES_HPOS); String output = formatter.format(testDoc, false); if ( outputFileName!=null ) { Utils.writeFile(outputFileName, output); } else { System.out.print(output); } }
/** Print out a whole tree in LISP form. Arg nodeTextProvider is used on the * node payloads to get the text for the nodes. * * @since 4.5.1 */ public static String toStringTree(Tree t, TreeTextProvider nodeTextProvider) { if ( t==null ) return "null"; String s = Utils.escapeWhitespace(nodeTextProvider.getText(t), false); if ( t.getChildCount()==0 ) return s; StringBuilder buf = new StringBuilder(); buf.append("("); s = Utils.escapeWhitespace(nodeTextProvider.getText(t), false); buf.append(s); buf.append(' '); for (int i = 0; i<t.getChildCount(); i++) { if ( i>0 ) buf.append(' '); buf.append(toStringTree(t.getChild(i), nodeTextProvider)); } buf.append(")"); return buf.toString(); }
private String toStringTree(Tree tree, List<String> ruleNames) { String s = Utils.escapeWhitespace(getNodeText(tree, ruleNames), false); if(tree.getChildCount() == 0) return s; StringBuilder buf = new StringBuilder(); buf.append("("); s = Utils.escapeWhitespace(getNodeText(tree, ruleNames), false); buf.append(s); buf.append(' '); for(int i = 0; i < tree.getChildCount(); i++) { if(i > 0) buf.append(' '); buf.append(toStringTree(tree.getChild(i), ruleNames)); } buf.append(")"); return buf.toString(); }
private String toStringTree(@NotNull final Tree t, @Nullable final List<String> ruleNames) { if (t.getChildCount() == 0) { return Utils.escapeWhitespace(getNodeText(t, ruleNames), true); } StringBuilder buf = new StringBuilder(); buf.append(" ( "); String s = Utils.escapeWhitespace(getNodeText(t, ruleNames), true); buf.append(s); buf.append(' '); for (int i = 0; i < t.getChildCount(); i++) { if (i > 0) { buf.append(' '); } buf.append(toStringTree(t.getChild(i), ruleNames)); } buf.append(" ) "); return buf.toString(); }
/** Print out a whole tree in LISP form. {@link #getNodeText} is used on the * node payloads to get the text for the nodes. Detect * parse trees and extract data appropriately. */ public static String toStringTree(@NotNull Tree t, @Nullable List<String> ruleNames) { String s = Utils.escapeWhitespace(getNodeText(t, ruleNames), false); if ( t.getChildCount()==0 ) return s; StringBuilder buf = new StringBuilder(); buf.append("("); s = Utils.escapeWhitespace(getNodeText(t, ruleNames), false); buf.append(s); buf.append(' '); for (int i = 0; i<t.getChildCount(); i++) { if ( i>0 ) buf.append(' '); buf.append(toStringTree(t.getChild(i), ruleNames)); } buf.append(")"); return buf.toString(); }
/** * Get a map from token names to token types. * * <p>Used for XPath and tree pattern compilation.</p> */ @NotNull public Map<String, Integer> getTokenTypeMap() { String[] tokenNames = getTokenNames(); if (tokenNames == null) { throw new UnsupportedOperationException("The current recognizer does not provide a list of token names."); } synchronized (tokenTypeMapCache) { Map<String, Integer> result = tokenTypeMapCache.get(tokenNames); if (result == null) { result = Utils.toMap(tokenNames); result.put("EOF", Token.EOF); result = Collections.unmodifiableMap(result); tokenTypeMapCache.put(tokenNames, result); } return result; } }
/** * Get a map from rule names to rule indexes. * * <p>Used for XPath and tree pattern compilation.</p> */ @NotNull public Map<String, Integer> getRuleIndexMap() { String[] ruleNames = getRuleNames(); if (ruleNames == null) { throw new UnsupportedOperationException("The current recognizer does not provide a list of rule names."); } synchronized (ruleIndexMapCache) { Map<String, Integer> result = ruleIndexMapCache.get(ruleNames); if (result == null) { result = Collections.unmodifiableMap(Utils.toMap(ruleNames)); ruleIndexMapCache.put(ruleNames, result); } return result; } }
public void checkCExec(String filename) throws Exception { URL testFolderURL = TestCGen.class.getClassLoader().getResource(SAMPLES_DIR); String testFolder = testFolderURL.getPath(); String workingDir = getWorkingDir(); String J_pathToFile = testFolder+"/"+filename; String C_filename = basename(filename)+".c"; JTran jTran = new JTran(); String C_code = jTran.translate(J_pathToFile, C_filename, false, false); Utils.writeFile(workingDir+"/"+C_filename, C_code); // compile String[] cc = {"cc", "-o", basename(filename), C_filename}; Triple<Integer, String, String> cc_result = exec(cc, getWorkingDir()); int execCode = cc_result.a; String stdout = cc_result.b; String stderr = cc_result.c; assertEquals("", stdout); assertEquals("", stderr); assertEquals(0, execCode); // execute String[] exec_cmd = {"./"+basename(filename)}; Triple<Integer, String, String> result = exec(exec_cmd, getWorkingDir()); execCode = result.a; stdout = result.b; stderr = result.c; String expected_output_filename = basename(filename)+".txt"; String expected_output = readFile(testFolder+"/"+expected_output_filename); assertEquals(expected_output, stdout); assertEquals("", stderr); assertEquals(0, execCode); }
public static void writePython(LangDescriptor[] languages, List<Integer> ks, Float[][] medians) throws IOException { StringBuilder data = new StringBuilder(); StringBuilder plot = new StringBuilder(); for (int i = 0; i<languages.length; i++) { LangDescriptor language = languages[i]; List<Float> filteredMedians = BuffUtils.filter(Arrays.asList(medians[i]), m -> m!=null); data.append(language.name+'='+filteredMedians+'\n'); plot.append(String.format("ax.plot(ks, %s, label=\"%s\", marker='%s', color='%s')\n", language.name, language.name, nameToGraphMarker.get(language.name), nameToGraphColor.get(language.name))); } String python = "#\n"+ "# AUTO-GENERATED FILE. DO NOT EDIT\n" + "# CodeBuff %s '%s'\n" + "#\n"+ "import numpy as np\n"+ "import matplotlib.pyplot as plt\n\n" + "%s\n" + "ks = %s\n"+ "fig = plt.figure()\n"+ "ax = plt.subplot(111)\n"+ "%s"+ "ax.tick_params(axis='both', which='major', labelsize=18)\n" + "ax.set_xlabel(\"$k$ nearest neighbors\", fontsize=20)\n"+ "ax.set_ylabel(\"Median error rate\", fontsize=20)\n" + "#ax.set_title(\"k Nearest Neighbors vs\\nLeave-one-out Validation Error Rate\")\n"+ "plt.legend(fontsize=18)\n\n" + "fig.savefig('images/vary_k.pdf', format='pdf')\n"+ "plt.show()\n"; String code = String.format(python, Tool.version, new Date(), data, ks, plot); String fileName = "python/src/vary_k.py"; Utils.writeFile(fileName, code); System.out.println("wrote python code to "+fileName); }
public static void main(String[] args) throws Exception { LangDescriptor[] languages = new LangDescriptor[] { JAVA_DESCR, JAVA8_DESCR, JAVA_GUAVA_DESCR, }; List<String> corpusDirs = map(languages, l -> l.corpusDir); String[] dirs = corpusDirs.toArray(new String[languages.length]); String python = testAllLanguages(languages, dirs, "all_java_leave_one_out.pdf"); String fileName = "python/src/all_java_leave_one_out.py"; Utils.writeFile(fileName, python); System.out.println("wrote python code to "+fileName); }
public static void main(String[] args) throws Exception { LangDescriptor[] languages = new LangDescriptor[] { SQLITE_NOISY_DESCR, SQLITE_CLEAN_DESCR, TSQL_NOISY_DESCR, TSQL_CLEAN_DESCR, }; List<String> corpusDirs = map(languages, l -> l.corpusDir); String[] dirs = corpusDirs.toArray(new String[languages.length]); String python = testAllLanguages(languages, dirs, "all_sql_leave_one_out.pdf"); String fileName = "python/src/all_sql_leave_one_out.py"; Utils.writeFile(fileName, python); System.out.println("wrote python code to "+fileName); }
public static Triple<Formatter,Float,Float> validate(LangDescriptor language, List<InputDocument> documents, InputDocument testDoc, boolean saveOutput, boolean computeEditDistance) throws Exception { // kNNClassifier.resetCache(); Corpus corpus = new Corpus(documents, language); corpus.train(); // System.out.printf("%d feature vectors\n", corpus.featureVectors.size()); Formatter formatter = new Formatter(corpus, language.indentSize); String output = formatter.format(testDoc, false); float editDistance = 0; if ( computeEditDistance ) { editDistance = normalizedLevenshteinDistance(testDoc.content, output); } ClassificationAnalysis analysis = new ClassificationAnalysis(testDoc, formatter.getAnalysisPerToken()); // System.out.println(testDoc.fileName+": edit distance = "+editDistance+", error rate = "+analysis.getErrorRate()); if ( saveOutput ) { File dir = new File(outputDir+"/"+language.name); if ( saveOutput ) { dir = new File(outputDir+"/"+language.name); dir.mkdir(); } Utils.writeFile(dir.getPath()+"/"+new File(testDoc.fileName).getName(), output); } return new Triple<>(formatter, editDistance, analysis.getErrorRate()); }
@Override public void visitTerminal(TerminalNode node) { if (builder.length() > 0) { builder.append(' '); } builder.append(Utils.escapeWhitespace(Trees.getNodeText(node, ruleNames), false)); }
@Override public void visitErrorNode(ErrorNode node) { if (builder.length() > 0) { builder.append(' '); } builder.append(Utils.escapeWhitespace(Trees.getNodeText(node, ruleNames), false)); }
public static void writeFile(String dir, String fileName, String content) { try { Utils.writeFile(dir+"/"+fileName, content, "UTF-8"); } catch (IOException ioe) { System.err.println("can't write file"); ioe.printStackTrace(System.err); } }
private String process(final Tree t, final List<String> ruleNames) { if (t.getChildCount() == 0) return Utils.escapeWhitespace(Trees.getNodeText(t, ruleNames), false); StringBuilder sb = new StringBuilder(); sb.append(lead(level)); level++; String s = Utils.escapeWhitespace(Trees.getNodeText(t, ruleNames), false); sb.append(s + ' '); for (int i = 0; i < t.getChildCount(); i++) { sb.append(process(t.getChild(i), ruleNames)); } level--; sb.append(lead(level)); return sb.toString(); }
@Override public void visitTerminal(TerminalNode node) { String text = Utils.escapeWhitespace(Trees.getNodeText(node, ruleNames), false); if(text.startsWith(" ") || text.endsWith(" ")){ text = "'" + text + "'"; } stack.get(node.getParent()).add(text); }
/** * Append the symbol and its position in the input text to the parse tree * buffer. * * @param symbol the symbol which should be appended to the parse tree. * @return boolean true if symbol is not null, false otherwise. */ protected boolean appendToken(Token symbol) { if (symbol != null) { buf.append("["); String s = Utils.escapeWhitespace(symbol.getText(), false); buf.append(s); buf.append("] (line "); buf.append(String.valueOf(symbol.getLine())); buf.append(":"); buf.append(String.valueOf(symbol.getCharPositionInLine())); buf.append(")"); return true; } return false; }
/** * Determine some string representation for the given node and append it to * the parse tree buffer. * * @param t The sub-tree whose current node needs a string representation. */ protected void appendNodeText(@NotNull Tree t) { if ( ruleNames!=null ) { if ( t instanceof RuleNode ) { if (appendRule(((RuleNode)t).getRuleContext())) return; } if ( t instanceof ErrorNode) { buf.append("ERROR: "); appendToken(((ErrorNode)t).getSymbol()); return; } if ( t instanceof TerminalNode) { if (appendToken(((TerminalNode)t).getSymbol())) return; } } // no recog for rule names Object payload = t.getPayload(); if ( payload instanceof Token ) { if (appendToken((Token)payload)) return; } if ( payload instanceof RuleContext ) { if (appendRule((RuleContext)payload)) return; } String s = Utils.escapeWhitespace(payload.toString(), false); buf.append("unknown payload <<"); buf.append(s); buf.append(">>"); }
/** Output the current symbol with line and character position. */ public void outputSymbol(Token symbol) { if ( outputStream != null) { outputStream.print(", LT(1)=["); String s = Utils.escapeWhitespace(symbol.getText(), false); outputStream.print(s); outputStream.print("] (line "); outputStream.print(String.valueOf(symbol.getLine())); outputStream.print(":"); outputStream.print(String.valueOf(symbol.getCharPositionInLine())); outputStream.println(")"); } }
@Override public String toString() { StringBuilder builder = new StringBuilder(); builder.append("func"); ParameterModelImpl receiver = getReceiverParameter(); if (receiver != null) { builder.append(" (").append(receiver).append(")"); } builder.append(" ").append(getName()); builder.append("("); builder.append(Utils.join(getParameters().iterator(), ", ")); builder.append(")"); @SuppressWarnings("LocalVariableHidesMemberVariable") Collection<? extends ParameterModel> returnValues = getReturnValues(); if (returnValues.size() == 1 && returnValues.iterator().next().getName().equals("_")) { builder.append(" ").append(returnValues.iterator().next().getVarType()); } else if (!returnValues.isEmpty()) { builder.append(" ("); boolean first = true; for (ParameterModel parameter : returnValues) { if (first) { first = false; } else { builder.append(", "); } if (!"_".equals(parameter.getName())) { builder.append(parameter.getName()).append(' '); } builder.append(parameter.getVarType()); } builder.append(')'); } return builder.toString(); }
@Override public boolean equals(Object obj) { if (!(obj instanceof TypeModel)) { return false; } TypeModel other = (TypeModel)obj; return getKind().equals(other.getKind()) && Utils.equals(getPackage(), other.getPackage()) && getName().equals(other.getName()); }
/** * @see org.antlr.v4.runtime.tree.Trees.toStringTree(Tree, List<String>) */ public static String toStringTree(final Tree t, @Nullable final List<String> ruleNames, final int depth) { String s = Utils.escapeWhitespace(Trees.getNodeText(t, ruleNames), false); if (t.getChildCount() == 0) { return s; } final StringBuilder buf = new StringBuilder(); if (depth > 0) { buf.append(NEWLINE); } buf.append(indent(depth)); buf.append("("); s = Utils.escapeWhitespace(Trees.getNodeText(t, ruleNames), false); buf.append(s); buf.append(' '); for (int i = 0; i < t.getChildCount(); i++) { if (i > 0) { buf.append(' '); } buf.append(toStringTree(t.getChild(i), ruleNames, depth + 1)); } buf.append(")"); return buf.toString(); }
@Override public String toString() { String symbol = ""; if (startIndex >= 0 && startIndex < getInputStream().size()) { symbol = getInputStream().getText(Interval.of(startIndex,startIndex)); symbol = Utils.escapeWhitespace(symbol, false); } return String.format(Locale.getDefault(), "%s('%s')", LexerNoViableAltException.class.getSimpleName(), symbol); }
@Override public void visitTerminal(TerminalNode node) { if (builder.length() > 0) { builder.append(' '); } append(Utils.escapeWhitespace(Trees.getNodeText(node, ruleNames), false)); }
@Override public void visitErrorNode(ErrorNode node) { if (builder.length() > 0) { builder.append(' '); } append(Utils.escapeWhitespace(Trees.getNodeText(node, ruleNames), false)); }
public static GrammarRootAST parseGrammar(Project project, Tool antlr, String grammarFileName) { try { String encoding = ConfigANTLRPerGrammar.getProp(project, grammarFileName, ConfigANTLRPerGrammar.PROP_ENCODING, "UTF-8"); char[] grammarText = Utils.readFile(grammarFileName, encoding); String grammarTextS = new String(grammarText).replaceAll("\\r", ""); ANTLRStringStream in = new ANTLRStringStream(grammarTextS); GrammarRootAST t = antlr.parse(grammarFileName, in); return t; } catch (IOException ioe) { antlr.errMgr.toolError(ErrorType.CANNOT_OPEN_FILE, ioe, grammarFileName); } return null; }
public void checkCGen(String filename) throws Exception { URL testFolderURL = TestCGen.class.getClassLoader().getResource(SAMPLES_DIR); String testFolder = testFolderURL.getPath(); String workingDir = getWorkingDir(); String J_pathToFile = testFolder+"/"+filename; String C_filename = basename(filename)+".c"; JTran jTran = new JTran(); String C_code = jTran.translate(J_pathToFile, C_filename, false, false); Utils.writeFile(workingDir+"/"+C_filename, C_code); String[] indent_result_cmd = { "indent", "-bap", "-bad", "-br", "-nce", "-ncs", "-nprs", "-npcs", "-sai", "-saw", "-di1", "-brs", "-blf", "--indent-level4", "-nut", "-sob", "-l200", C_filename, "-o", C_filename // write on top of itself }; // normalize generated code exec(indent_result_cmd, workingDir); // format the expected file as well String expected_C_CodeFilename = testFolder+"/"+C_filename; String[] indent_expected_cmd = { "indent", "-bap", "-bad", "-br", "-nce", "-ncs", "-nprs", "-npcs", "-sai", "-saw", "-di1", "-brs", "-blf", "--indent-level4", "-nut", "-sob", "-l200", expected_C_CodeFilename, "-o", "expected_"+C_filename }; exec(indent_expected_cmd, workingDir); // compare with expected c file String[] diff_cmd = { "diff", "expected_"+C_filename, C_filename }; Triple<Integer, String, String> result = exec(diff_cmd, workingDir); int execCode = result.a; String stdout = result.b; String stderr = result.c; assertEquals("", stdout); assertEquals("", stderr); assertEquals(0, execCode); }
public static void main(String[] args) throws Exception { LangDescriptor[] languages = new LangDescriptor[] { // QUORUM_DESCR, ANTLR4_DESCR, JAVA_DESCR, JAVA8_DESCR, JAVA_GUAVA_DESCR, JAVA8_GUAVA_DESCR, SQLITE_CLEAN_DESCR, TSQL_CLEAN_DESCR, }; int maxNumFiles = 30; int trials = 50; Map<String,float[]> results = new HashMap<>(); for (LangDescriptor language : languages) { float[] medians = getMedianErrorRates(language, maxNumFiles, trials); results.put(language.name, medians); } String python = "#\n"+ "# AUTO-GENERATED FILE. DO NOT EDIT\n" + "# CodeBuff <version> '<date>'\n" + "#\n"+ "import numpy as np\n"+ "import matplotlib.pyplot as plt\n\n" + "fig = plt.figure()\n"+ "ax = plt.subplot(111)\n"+ "N = <maxNumFiles>\n" + "sizes = range(1,N+1)\n" + "<results:{r |\n" + "<r> = [<rest(results.(r)); separator={,}>]\n"+ "ax.plot(range(1,len(<r>)+1), <r>, label=\"<r>\", marker='<markers.(r)>', color='<colors.(r)>')\n" + "}>\n" + "ax.yaxis.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)\n" + "ax.set_xlabel(\"Number of training files in sample corpus subset\", fontsize=14)\n"+ "ax.set_ylabel(\"Median Error rate for <trials> trials\", fontsize=14)\n" + "ax.set_title(\"Effect of Corpus size on Median Leave-one-out Validation Error Rate\")\n"+ "plt.legend()\n" + "plt.tight_layout()\n" + "fig.savefig('images/subset_validator.pdf', format='pdf')\n"+ "plt.show()\n"; ST pythonST = new ST(python); pythonST.add("results", results); pythonST.add("markers", LeaveOneOutValidator.nameToGraphMarker); pythonST.add("colors", LeaveOneOutValidator.nameToGraphColor); pythonST.add("version", version); pythonST.add("date", new Date()); pythonST.add("trials", trials); pythonST.add("maxNumFiles", maxNumFiles); List<String> corpusDirs = map(languages, l -> l.corpusDir); String[] dirs = corpusDirs.toArray(new String[languages.length]); String fileName = "python/src/subset_validator.py"; Utils.writeFile(fileName, pythonST.render()); System.out.println("wrote python code to "+fileName); }
protected String getText(Tree tree) { String s = treeTextProvider.getText(tree); s = Utils.escapeWhitespace(s, false); return s; }
public void text(Graphics g, String s, int x, int y) { // System.out.println("drawing '"+s+"' @ "+x+","+y); s = Utils.escapeWhitespace(s, true); g.drawString(s, x, y); }
protected String getText(Tree tree) { String s = treeTextProvider.getText(tree); s = Utils.escapeWhitespace(s, true); return s; }
@Override public String toString() { return Utils.join(syntaxErrors.iterator(), "\n"); }
/** Get a map from token names to token types. */ public static Map<String, Integer> createTokenTypeMap(String[] tokenNames) { return Utils.toMap(tokenNames); }
/** Get a map from rule names to rule indexes. */ public static Map<String, Integer> createRuleIndexMap(String[] ruleNames) { return Utils.toMap(ruleNames); }
@Override public void visitErrorNode(ErrorNode node) { stack.get(node.getParent()).add(Utils.escapeWhitespace(Trees.getNodeText(node, ruleNames), false)); }