org.antlr.v4.runtime.misc.Utils Java Examples

The following examples show how to use org.antlr.v4.runtime.misc.Utils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: LeaveOneOutValidator.java    From codebuff with BSD 2-Clause "Simplified" License 6 votes vote down vote up
public static void main(String[] args) throws Exception {
		LangDescriptor[] languages = new LangDescriptor[] {
			JAVA_DESCR,
			JAVA8_DESCR,
			JAVA_GUAVA_DESCR,
			JAVA8_GUAVA_DESCR,
			ANTLR4_DESCR,
			SQLITE_CLEAN_DESCR,
			TSQL_CLEAN_DESCR,
			SQLITE_NOISY_DESCR,
			TSQL_NOISY_DESCR,
//			QUORUM_DESCR,
		};
		List<String> corpusDirs = map(languages, l -> l.corpusDir);
		String[] dirs = corpusDirs.toArray(new String[languages.length]);
		String python = testAllLanguages(languages, dirs, "leave_one_out.pdf");
		String fileName = "python/src/leave_one_out.py";
		Utils.writeFile(fileName, python);
		System.out.println("wrote python code to "+fileName);
	}
 
Example #2
Source File: Tool.java    From codebuff with BSD 2-Clause "Simplified" License 6 votes vote down vote up
public static void format(LangDescriptor language,
                          String testFileName,
                          String outputFileName)
	throws Exception
{
	// load all files up front
	List<String> allFiles = getFilenames(new File(language.corpusDir), language.fileRegex);
	List<InputDocument> documents = load(allFiles, language);
	// if in corpus, don't include in corpus
	final String path = new File(testFileName).getAbsolutePath();
	List<InputDocument> others = filter(documents, d -> !d.fileName.equals(path));
	InputDocument testDoc = parse(testFileName, language);
	Corpus corpus = new Corpus(others, language);
	corpus.train();

	Formatter formatter = new Formatter(corpus, language.indentSize, Formatter.DEFAULT_K,
	                                    FEATURES_INJECT_WS, FEATURES_HPOS);
	String output = formatter.format(testDoc, false);

	if ( outputFileName!=null ) {
		Utils.writeFile(outputFileName, output);
	}
	else {
		System.out.print(output);
	}
}
 
Example #3
Source File: Trees.java    From codebuff with BSD 2-Clause "Simplified" License 6 votes vote down vote up
/** Print out a whole tree in LISP form. Arg nodeTextProvider is used on the
 *  node payloads to get the text for the nodes.
 *
 *  @since 4.5.1
 */
public static String toStringTree(Tree t, TreeTextProvider nodeTextProvider) {
	if ( t==null ) return "null";
	String s = Utils.escapeWhitespace(nodeTextProvider.getText(t), false);
	if ( t.getChildCount()==0 ) return s;
	StringBuilder buf = new StringBuilder();
	buf.append("(");
	s = Utils.escapeWhitespace(nodeTextProvider.getText(t), false);
	buf.append(s);
	buf.append(' ');
	for (int i = 0; i<t.getChildCount(); i++) {
		if ( i>0 ) buf.append(' ');
		buf.append(toStringTree(t.getChild(i), nodeTextProvider));
	}
	buf.append(")");
	return buf.toString();
}
 
Example #4
Source File: AllJavaLeaveOneOutValidation.java    From codebuff with BSD 2-Clause "Simplified" License 5 votes vote down vote up
public static void main(String[] args) throws Exception {
	LangDescriptor[] languages = new LangDescriptor[] {
		JAVA_DESCR,
		JAVA8_DESCR,
		JAVA_GUAVA_DESCR,
	};
	List<String> corpusDirs = map(languages, l -> l.corpusDir);
	String[] dirs = corpusDirs.toArray(new String[languages.length]);
	String python = testAllLanguages(languages, dirs, "all_java_leave_one_out.pdf");
	String fileName = "python/src/all_java_leave_one_out.py";
	Utils.writeFile(fileName, python);
	System.out.println("wrote python code to "+fileName);
}
 
Example #5
Source File: AllSQLLeaveOneOutValidation.java    From codebuff with BSD 2-Clause "Simplified" License 5 votes vote down vote up
public static void main(String[] args) throws Exception {
	LangDescriptor[] languages = new LangDescriptor[] {
		SQLITE_NOISY_DESCR,
		SQLITE_CLEAN_DESCR,
		TSQL_NOISY_DESCR,
		TSQL_CLEAN_DESCR,
	};
	List<String> corpusDirs = map(languages, l -> l.corpusDir);
	String[] dirs = corpusDirs.toArray(new String[languages.length]);
	String python = testAllLanguages(languages, dirs, "all_sql_leave_one_out.pdf");
	String fileName = "python/src/all_sql_leave_one_out.py";
	Utils.writeFile(fileName, python);
	System.out.println("wrote python code to "+fileName);
}
 
Example #6
Source File: TestK.java    From codebuff with BSD 2-Clause "Simplified" License 5 votes vote down vote up
public static void writePython(LangDescriptor[] languages, List<Integer> ks, Float[][] medians) throws IOException {
	StringBuilder data = new StringBuilder();
	StringBuilder plot = new StringBuilder();
	for (int i = 0; i<languages.length; i++) {
		LangDescriptor language = languages[i];
		List<Float> filteredMedians = BuffUtils.filter(Arrays.asList(medians[i]), m -> m!=null);
		data.append(language.name+'='+filteredMedians+'\n');
		plot.append(String.format("ax.plot(ks, %s, label=\"%s\", marker='%s', color='%s')\n",
		                          language.name, language.name,
		                          nameToGraphMarker.get(language.name),
								  nameToGraphColor.get(language.name)));
	}

	String python =
		"#\n"+
		"# AUTO-GENERATED FILE. DO NOT EDIT\n" +
		"# CodeBuff %s '%s'\n" +
		"#\n"+
		"import numpy as np\n"+
		"import matplotlib.pyplot as plt\n\n" +
		"%s\n" +
		"ks = %s\n"+
		"fig = plt.figure()\n"+
		"ax = plt.subplot(111)\n"+
		"%s"+
		"ax.tick_params(axis='both', which='major', labelsize=18)\n" +
		"ax.set_xlabel(\"$k$ nearest neighbors\", fontsize=20)\n"+
		"ax.set_ylabel(\"Median error rate\", fontsize=20)\n" +
		"#ax.set_title(\"k Nearest Neighbors vs\\nLeave-one-out Validation Error Rate\")\n"+
		"plt.legend(fontsize=18)\n\n" +
		"fig.savefig('images/vary_k.pdf', format='pdf')\n"+
		"plt.show()\n";
	String code = String.format(python, Tool.version, new Date(), data, ks, plot);

	String fileName = "python/src/vary_k.py";
	Utils.writeFile(fileName, code);
	System.out.println("wrote python code to "+fileName);
}
 
Example #7
Source File: TreeUtils.java    From proleap-vb6-parser with MIT License 5 votes vote down vote up
/**
 * @see org.antlr.v4.runtime.tree.Trees.toStringTree(Tree, List<String>)
 */
public static String toStringTree(final Tree t, final List<String> ruleNames, final int depth) {
	String s = Utils.escapeWhitespace(Trees.getNodeText(t, ruleNames), false);

	if (t.getChildCount() == 0) {
		return s;
	}

	final StringBuilder buf = new StringBuilder();

	if (depth > 0) {
		buf.append(NEWLINE);
	}

	buf.append(indent(depth));
	buf.append("(");
	s = Utils.escapeWhitespace(Trees.getNodeText(t, ruleNames), false);
	buf.append(s);
	buf.append(' ');

	for (int i = 0; i < t.getChildCount(); i++) {
		if (i > 0) {
			buf.append(' ');
		}

		buf.append(toStringTree(t.getChild(i), ruleNames, depth + 1));
	}

	buf.append(")");

	return buf.toString();
}
 
Example #8
Source File: InputPanel.java    From intellij-plugin-v4 with BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
/**
 * Display syntax errors, hints in tooltips if under the cursor
 */
public static void showTooltips(EditorMouseEvent event, Editor editor,
                                @NotNull PreviewState previewState, int offset) {
	if ( previewState.parsingResult==null ) return; // no results?

	// Turn off any tooltips if none under the cursor
	// find the highlighter associated with this offset
	List<RangeHighlighter> highlightersAtOffset = MyActionUtils.getRangeHighlightersAtOffset(editor, offset);
	if ( highlightersAtOffset.size()==0 ) {
		return;
	}

	List<String> msgList = new ArrayList<>();
	boolean foundDecisionEvent = false;
	for ( RangeHighlighter r : highlightersAtOffset ) {
		DecisionEventInfo eventInfo = r.getUserData(ProfilerPanel.DECISION_EVENT_INFO_KEY);
		String msg;
		if ( eventInfo!=null ) {
			// TODO: move decision event stuff to profiler?
			if ( eventInfo instanceof AmbiguityInfo ) {
				msg = "Ambiguous upon alts " + eventInfo.configs.getAlts().toString();
			} else if ( eventInfo instanceof ContextSensitivityInfo ) {
				msg = "Context-sensitive";
			} else if ( eventInfo instanceof LookaheadEventInfo ) {
				int k = eventInfo.stopIndex - eventInfo.startIndex + 1;
				msg = "Deepest lookahead k=" + k;
			} else if ( eventInfo instanceof PredicateEvalInfo ) {
				PredicateEvalInfo evalInfo = (PredicateEvalInfo) eventInfo;
				msg = ProfilerPanel.getSemanticContextDisplayString(evalInfo,
						previewState,
						evalInfo.semctx, evalInfo.predictedAlt,
						evalInfo.evalResult);
				msg = msg + (!evalInfo.fullCtx ? " (DFA)" : "");
			} else {
				msg = "Unknown decision event: " + eventInfo;
			}
			foundDecisionEvent = true;
		} else {
			// error tool tips
			SyntaxError errorUnderCursor = r.getUserData(SYNTAX_ERROR);
			msg = getErrorDisplayString(errorUnderCursor);
			if ( msg.length()>MAX_HINT_WIDTH ) {
				msg = msg.substring(0, MAX_HINT_WIDTH) + "...";
			}
			if ( msg.indexOf('<') >= 0 ) {
				msg = msg.replaceAll("<", "&lt;");
			}
		}
		msgList.add(msg);
	}
	String combinedMsg = Utils.join(msgList.iterator(), "\n");
	HintManagerImpl hintMgr = (HintManagerImpl) HintManager.getInstance();
	if ( foundDecisionEvent ) {
		showDecisionEventToolTip(editor, offset, hintMgr, combinedMsg);
	}
	else {
		showPreviewEditorErrorToolTip(editor, offset, hintMgr, combinedMsg);
	}
}
 
Example #9
Source File: InputPanel.java    From intellij-plugin-v4 with BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
/**
	 * Show tokens/region associated with parse tree parent of this token
	 * if the alt-key is down and mouse movement occurs.
	 */
	public void showParseRegion(EditorMouseEvent event, Editor editor,
	                            PreviewState previewState, int offset) {
		Token tokenUnderCursor = ParsingUtils.getTokenUnderCursor(previewState, offset);
		if ( tokenUnderCursor==null ) {
			return;
		}

		ParseTree tree = previewState.parsingResult.tree;
		TerminalNode nodeWithToken =
			(TerminalNode) ParsingUtils.getParseTreeNodeWithToken(tree, tokenUnderCursor);
		if ( nodeWithToken==null ) {
			// hidden token
			return;
		}

		PreviewParser parser = (PreviewParser) previewState.parsingResult.parser;
		CommonTokenStream tokenStream = (CommonTokenStream) parser.getInputStream();
		ParserRuleContext parent = (ParserRuleContext) nodeWithToken.getParent();
		Interval tokenInterval = parent.getSourceInterval();
		Token startToken = tokenStream.get(tokenInterval.a);
		Token stopToken = tokenStream.get(tokenInterval.b);
		Interval sourceInterval =
			Interval.of(startToken.getStartIndex(), stopToken.getStopIndex()+1);

		List<String> stack = parser.getRuleInvocationStack(parent);
		Collections.reverse(stack);

		if ( stack.size()>MAX_STACK_DISPLAY ) {
			// collapse contiguous dups to handle left-recursive stacks
			List<Pair<String, Integer>> smaller = new ArrayList<>();
			int last = 0;
			smaller.add(new Pair<>(stack.get(0), 1)); // init to having first element, count of 1
			for (int i = 1; i<stack.size(); i++) {
				String s = stack.get(i);
				if ( smaller.get(last).a.equals(s) ) {
					smaller.set(last, new Pair<>(s, smaller.get(last).b + 1));
				}
				else {
					smaller.add(new Pair<>(s, 1));
					last++;
				}
			}
			stack = new ArrayList<>();
			for ( Pair<String, Integer> pair : smaller ) {
				if ( pair.b>1 ) {
					stack.add(pair.a + "^" + pair.b);
				} else {
					stack.add(pair.a);
				}
			}
		}
		String stackS = Utils.join(stack.toArray(), "\n");
		highlightAndOfferHint(editor, offset, sourceInterval,
		                      JBColor.BLUE, EffectType.ROUNDED_BOX, stackS);


		// Code for a balloon.

//		JBPopupFactory popupFactory = JBPopupFactory.getInstance();
//		BalloonBuilder builder =
//		    popupFactory.createHtmlTextBalloonBuilder(Utils.join(stack.toArray(), "<br>"),
//												  MessageType.INFO, null);
//		builder.setHideOnClickOutside(true);
//		Balloon balloon = builder.createBalloon();
//		MouseEvent mouseEvent = event.getMouseEvent();
//		Point point = mouseEvent.getPoint();
//		point.translate(10, -15);
//		RelativePoint where = new RelativePoint(mouseEvent.getComponent(), point);
//		balloon.show(where, Balloon.Position.above);
	}
 
Example #10
Source File: GenerateLexerRulesForLiteralsAction.java    From intellij-plugin-v4 with BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
@Override
	public void actionPerformed(AnActionEvent e) {
		LOG.info("actionPerformed GenerateLexerRulesForLiteralsAction");
		final Project project = e.getProject();

		final PsiFile psiFile = e.getData(LangDataKeys.PSI_FILE);
		if (psiFile == null) {
			return;
		}
		String inputText = psiFile.getText();
		ParsingResult results = ParsingUtils.parseANTLRGrammar(inputText);

		final Parser parser = results.parser;
		final ParseTree tree = results.tree;
		Collection<ParseTree> literalNodes = XPath.findAll(tree, "//ruleBlock//STRING_LITERAL", parser);
		LinkedHashMap<String, String> lexerRules = new LinkedHashMap<String, String>();
		for (ParseTree node : literalNodes) {
			String literal = node.getText();
			String ruleText = String.format("%s : %s ;",
											RefactorUtils.getLexerRuleNameFromLiteral(literal), literal);
			lexerRules.put(literal, ruleText);
		}

		// remove those already defined
		String lexerRulesXPath = "//lexerRule";
		String treePattern = "<TOKEN_REF> : <STRING_LITERAL>;";
		ParseTreePattern p = parser.compileParseTreePattern(treePattern, ANTLRv4Parser.RULE_lexerRule);
		List<ParseTreeMatch> matches = p.findAll(tree, lexerRulesXPath);

		for (ParseTreeMatch match : matches) {
			ParseTree lit = match.get("STRING_LITERAL");
			if (lexerRules.containsKey(lit.getText())) { // we have rule for this literal already
				lexerRules.remove(lit.getText());
			}
		}

		final LiteralChooser chooser =
			new LiteralChooser(project, new ArrayList<String>(lexerRules.values()));
		chooser.show();
		List<String> selectedElements = chooser.getSelectedElements();
		// chooser disposed automatically.

		final Editor editor = e.getData(PlatformDataKeys.EDITOR);
		final Document doc = editor.getDocument();
		final CommonTokenStream tokens = (CommonTokenStream) parser.getTokenStream();
//		System.out.println(selectedElements);
		if (selectedElements != null) {
			String text = doc.getText();
			int cursorOffset = editor.getCaretModel().getOffset();
			// make sure it's not in middle of rule; put between.
//					System.out.println("offset "+cursorOffset);
			Collection<ParseTree> allRuleNodes = XPath.findAll(tree, "//ruleSpec", parser);
			for (ParseTree r : allRuleNodes) {
				Interval extent = r.getSourceInterval(); // token indexes
				int start = tokens.get(extent.a).getStartIndex();
				int stop = tokens.get(extent.b).getStopIndex();
//						System.out.println("rule "+r.getChild(0).getText()+": "+start+".."+stop);
				if (cursorOffset < start) {
					// before this rule, so must be between previous and this one
					cursorOffset = start; // put right before this rule
					break;
				}
				else if (cursorOffset >= start && cursorOffset <= stop) {
					// cursor in this rule
					cursorOffset = stop + 2; // put right before this rule (after newline)
					if (cursorOffset >= text.length()) {
						cursorOffset = text.length();
					}
					break;
				}
			}

			String allRules = Utils.join(selectedElements.iterator(), "\n");
			text =
				text.substring(0, cursorOffset) +
					"\n" + allRules + "\n" +
					text.substring(cursorOffset, text.length());
			MyPsiUtils.replacePsiFileFromText(project, psiFile, text);
		}
	}
 
Example #11
Source File: PSIElementTypeFactory.java    From antlr4-intellij-adaptor with BSD 2-Clause "Simplified" License 4 votes vote down vote up
/** Get a map from rule names to rule indexes. */
public static Map<String, Integer> createRuleIndexMap(String[] ruleNames) {
	return Utils.toMap(ruleNames);
}
 
Example #12
Source File: PSIElementTypeFactory.java    From antlr4-intellij-adaptor with BSD 2-Clause "Simplified" License 4 votes vote down vote up
/** Get a map from token names to token types. */
public static Map<String, Integer> createTokenTypeMap(String[] tokenNames) {
	return Utils.toMap(tokenNames);
}
 
Example #13
Source File: SyntaxErrorListener.java    From antlr4-intellij-adaptor with BSD 2-Clause "Simplified" License 4 votes vote down vote up
@Override
public String toString() {
	return Utils.join(syntaxErrors.iterator(), "\n");
}
 
Example #14
Source File: TreeViewer.java    From codebuff with BSD 2-Clause "Simplified" License 4 votes vote down vote up
protected String getText(Tree tree) {
	String s = treeTextProvider.getText(tree);
	s = Utils.escapeWhitespace(s, true);
	return s;
}
 
Example #15
Source File: TreeViewer.java    From codebuff with BSD 2-Clause "Simplified" License 4 votes vote down vote up
public void text(Graphics g, String s, int x, int y) {
//		System.out.println("drawing '"+s+"' @ "+x+","+y);
		s = Utils.escapeWhitespace(s, true);
		g.drawString(s, x, y);
	}
 
Example #16
Source File: TreePostScriptGenerator.java    From codebuff with BSD 2-Clause "Simplified" License 4 votes vote down vote up
protected String getText(Tree tree) {
	String s = treeTextProvider.getText(tree);
	s = Utils.escapeWhitespace(s, false);
	return s;
}
 
Example #17
Source File: SubsetValidator.java    From codebuff with BSD 2-Clause "Simplified" License 4 votes vote down vote up
public static void main(String[] args) throws Exception {
		LangDescriptor[] languages = new LangDescriptor[] {
//			QUORUM_DESCR,
			ANTLR4_DESCR,
			JAVA_DESCR,
			JAVA8_DESCR,
			JAVA_GUAVA_DESCR,
			JAVA8_GUAVA_DESCR,
			SQLITE_CLEAN_DESCR,
			TSQL_CLEAN_DESCR,
		};

		int maxNumFiles = 30;
		int trials = 50;
		Map<String,float[]> results = new HashMap<>();
		for (LangDescriptor language : languages) {
			float[] medians = getMedianErrorRates(language, maxNumFiles, trials);
			results.put(language.name, medians);
		}
		String python =
			"#\n"+
			"# AUTO-GENERATED FILE. DO NOT EDIT\n" +
			"# CodeBuff <version> '<date>'\n" +
			"#\n"+
			"import numpy as np\n"+
			"import matplotlib.pyplot as plt\n\n" +
			"fig = plt.figure()\n"+
			"ax = plt.subplot(111)\n"+
			"N = <maxNumFiles>\n" +
			"sizes = range(1,N+1)\n" +
			"<results:{r |\n" +
			"<r> = [<rest(results.(r)); separator={,}>]\n"+
			"ax.plot(range(1,len(<r>)+1), <r>, label=\"<r>\", marker='<markers.(r)>', color='<colors.(r)>')\n" +
			"}>\n" +
			"ax.yaxis.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)\n" +
			"ax.set_xlabel(\"Number of training files in sample corpus subset\", fontsize=14)\n"+
			"ax.set_ylabel(\"Median Error rate for <trials> trials\", fontsize=14)\n" +
			"ax.set_title(\"Effect of Corpus size on Median Leave-one-out Validation Error Rate\")\n"+
			"plt.legend()\n" +
			"plt.tight_layout()\n" +
			"fig.savefig('images/subset_validator.pdf', format='pdf')\n"+
			"plt.show()\n";
		ST pythonST = new ST(python);
		pythonST.add("results", results);
		pythonST.add("markers", LeaveOneOutValidator.nameToGraphMarker);
		pythonST.add("colors", LeaveOneOutValidator.nameToGraphColor);
		pythonST.add("version", version);
		pythonST.add("date", new Date());
		pythonST.add("trials", trials);
		pythonST.add("maxNumFiles", maxNumFiles);
		List<String> corpusDirs = map(languages, l -> l.corpusDir);
		String[] dirs = corpusDirs.toArray(new String[languages.length]);
		String fileName = "python/src/subset_validator.py";
		Utils.writeFile(fileName, pythonST.render());
		System.out.println("wrote python code to "+fileName);
	}