org.antlr.v4.tool.Grammar Java Examples

public void processGrammarsOnCommandLine() {
	List<GrammarRootAST> sortedGrammars = sortGrammarByTokenVocab(grammarFiles);

	for (GrammarRootAST t : sortedGrammars) {
		final Grammar g = createGrammar(t);
		g.fileName = t.fileName;
		if ( gen_dependencies ) {
			BuildDependencyGenerator dep =
				new BuildDependencyGenerator(this, g);
			/*
				List outputFiles = dep.getGeneratedFileList();
				List dependents = dep.getDependenciesFileList();
				System.out.println("output: "+outputFiles);
				System.out.println("dependents: "+dependents);
				 */
			System.out.println(dep.getDependencies().render());

		}
		else if (errMgr.getNumErrors() == 0) {
			process(g, true);
		}
	}
}
 

public void generateATNs(Grammar g) {
	DOTGenerator dotGenerator = new DOTGenerator(g);
	List<Grammar> grammars = new ArrayList<Grammar>();
	grammars.add(g);
	List<Grammar> imported = g.getAllImportedGrammars();
	if ( imported!=null ) grammars.addAll(imported);
	for (Grammar ig : grammars) {
		for (Rule r : ig.rules.values()) {
			try {
				String dot = dotGenerator.getDOT(g.atn.ruleToStartState[r.index], g.isLexer());
				if (dot != null) {
					writeDOTFile(g, r, dot);
				}
			}
               catch (IOException ioe) {
				errMgr.toolError(ErrorType.CANNOT_WRITE_FILE, ioe);
			}
		}
	}
}
 

public static TerminalNode getRuleDefNameNode(Parser parser, ParseTree tree, String ruleName) {
	Collection<ParseTree> ruleDefRuleNodes;
	if ( Grammar.isTokenName(ruleName) ) {
		ruleDefRuleNodes = XPath.findAll(tree, "//lexerRule/TOKEN_REF", parser);
	}
	else {
		ruleDefRuleNodes = XPath.findAll(tree, "//parserRuleSpec/RULE_REF", parser);
	}
	for (ParseTree node : ruleDefRuleNodes) {
		String r = node.getText(); // always a TerminalNode; just get rule name of this def
		if ( r.equals(ruleName) ) {
			return (TerminalNode)node;
		}
	}
	return null;
}
 

public static List<TerminalNode> getAllRuleRefNodes(Parser parser, ParseTree tree, String ruleName) {
	List<TerminalNode> nodes = new ArrayList<TerminalNode>();
	Collection<ParseTree> ruleRefs;
	if ( Grammar.isTokenName(ruleName) ) {
		ruleRefs = XPath.findAll(tree, "//lexerRuleBlock//TOKEN_REF", parser);
	}
	else {
		ruleRefs = XPath.findAll(tree, "//ruleBlock//RULE_REF", parser);
	}
	for (ParseTree node : ruleRefs) {
		TerminalNode terminal = (TerminalNode)node;
		Token rrefToken = terminal.getSymbol();
		String r = rrefToken.getText();
		if ( r.equals(ruleName) ) {
			nodes.add(terminal);
		}
	}
	if ( nodes.size()==0 ) return null;
	return nodes;
}
 

protected void handleOptionSetArg(String arg) {
	int eq = arg.indexOf('=');
	if ( eq>0 && arg.length()>3 ) {
		String option = arg.substring("-D".length(), eq);
		String value = arg.substring(eq+1);
		if ( value.length()==0 ) {
			errMgr.toolError(ErrorType.BAD_OPTION_SET_SYNTAX, arg);
			return;
		}
		if ( Grammar.parserOptions.contains(option) ||
			 Grammar.lexerOptions.contains(option) )
		{
			if ( grammarOptions==null ) grammarOptions = new HashMap<String, String>();
			grammarOptions.put(option, value);
		}
		else {
			errMgr.grammarError(ErrorType.ILLEGAL_OPTION,
								null,
								null,
								option);
		}
	}
	else {
		errMgr.toolError(ErrorType.BAD_OPTION_SET_SYNTAX, arg);
	}
}
 

public static ParsingResult parseText(Grammar g,
									  LexerGrammar lg,
									  String startRuleName,
									  final VirtualFile grammarFile,
									  String inputText,
									  Project project) {
	ANTLRv4GrammarProperties grammarProperties = getGrammarProperties(project, grammarFile);
	CharStream input = grammarProperties.getCaseChangingStrategy()
			.applyTo(CharStreams.fromString(inputText, grammarFile.getPath()));
	LexerInterpreter lexEngine;
	lexEngine = lg.createLexerInterpreter(input);
	SyntaxErrorListener syntaxErrorListener = new SyntaxErrorListener();
	lexEngine.removeErrorListeners();
	lexEngine.addErrorListener(syntaxErrorListener);
	CommonTokenStream tokens = new TokenStreamSubset(lexEngine);
	return parseText(g, lg, startRuleName, grammarFile, syntaxErrorListener, tokens, 0);
}
 

public static String getSemanticContextDisplayString(PredicateEvalInfo pred,
                                                     PreviewState previewState,
                                                     SemanticContext semctx,
                                                     int alt,
                                                     boolean result) {
	Grammar g = previewState.g;
	String semanticContextDisplayString = g.getSemanticContextDisplayString(semctx);
	if ( semctx instanceof SemanticContext.PrecedencePredicate ) {
		int ruleIndex = previewState.parsingResult.parser.getATN().decisionToState.get(pred.decision).ruleIndex;
		Rule rule = g.getRule(ruleIndex);
		int precedence = ((SemanticContext.PrecedencePredicate) semctx).precedence;
		// precedence = n - originalAlt + 1, So:
		int originalAlt = rule.getOriginalNumberOfAlts()-precedence+1;
		alt = originalAlt;
	}
	return semanticContextDisplayString+" => alt "+alt+" is "+result;
}
 

public PreviewState getAssociatedParserIfLexer(String grammarFileName) {
		for (PreviewState s : grammarToPreviewState.values()) {
			if ( s!=null && s.lg!=null &&
					(sameFile(grammarFileName, s.lg.fileName)||s.lg==ParsingUtils.BAD_LEXER_GRAMMAR) )
			{
				// s has a lexer with same filename, see if there is a parser grammar
				// (not a combined grammar)
				if ( s.g!=null && s.g.getType()==ANTLRParser.PARSER ) {
//					System.out.println(s.lg.fileName+" vs "+grammarFileName+", g="+s.g.name+", type="+s.g.getTypeString());
					return s;
				}
			}

			if ( s!=null && s.g!=null && s.g.importedGrammars!=null ) {
				for ( Grammar importedGrammar : s.g.importedGrammars ) {
					if (grammarFileName.equals(importedGrammar.fileName)) {
						return s;
					}
				}
			}
		}
		return null;
	}
 

public Rule isolatedRuleRef(String x) {
if ( node.resolver instanceof Grammar ) return null;

if ( x.equals(r.name) ) return r;
List<LabelElementPair> labels = null;
if ( node.resolver instanceof Rule ) {
	labels = r.getElementLabelDefs().get(x);
}
else if ( node.resolver instanceof Alternative ) {
	labels = ((Alternative)node.resolver).labelDefs.get(x);
}
if ( labels!=null ) {  // it's a label ref. is it a rule label?
	LabelElementPair anyLabelDef = labels.get(0);
	if ( anyLabelDef.type==LabelType.RULE_LABEL ) {
		return g.getRule(anyLabelDef.element.getText());
	}
}
if ( node.resolver instanceof Alternative ) {
	if ( ((Alternative)node.resolver).ruleRefs.get(x)!=null ) {
		return g.getRule(x);
	}
}
      return null;
  }
 

public void checkForQualifiedRuleIssues(Grammar g, List<GrammarAST> qualifiedRuleRefs) {
	for (GrammarAST dot : qualifiedRuleRefs) {
		GrammarAST grammar = (GrammarAST)dot.getChild(0);
		GrammarAST rule = (GrammarAST)dot.getChild(1);
           g.tool.log("semantics", grammar.getText()+"."+rule.getText());
		Grammar delegate = g.getImportedGrammar(grammar.getText());
		if ( delegate==null ) {
			errMgr.grammarError(ErrorType.NO_SUCH_GRAMMAR_SCOPE,
									  g.fileName, grammar.token, grammar.getText(),
									  rule.getText());
		}
		else {
			if ( g.getRule(grammar.getText(), rule.getText())==null ) {
				errMgr.grammarError(ErrorType.NO_SUCH_RULE_IN_SCOPE,
										  g.fileName, rule.token, grammar.getText(),
										  rule.getText());
			}
		}
	}
}
 

public void checkRuleArgs(Grammar g, List<GrammarAST> rulerefs) {
	if ( rulerefs==null ) return;
	for (GrammarAST ref : rulerefs) {
		String ruleName = ref.getText();
		Rule r = g.getRule(ruleName);
		GrammarAST arg = (GrammarAST)ref.getFirstChildWithType(ANTLRParser.ARG_ACTION);
		if ( arg!=null && (r==null || r.args==null) ) {
			errMgr.grammarError(ErrorType.RULE_HAS_NO_ARGS,
									  g.fileName, ref.token, ruleName);

		}
		else if ( arg==null && (r!=null&&r.args!=null) ) {
			errMgr.grammarError(ErrorType.MISSING_RULE_ARGS,
									  g.fileName, ref.token, ruleName);
		}
	}
}
 

public SymbolChecks(Grammar g, SymbolCollector collector) {
      this.g = g;
      this.collector = collector;
this.errMgr = g.tool.errMgr;

      for (GrammarAST tokenId : collector.tokenIDRefs) {
          tokenIDs.add(tokenId.getText());
      }
      /*
      System.out.println("rules="+collector.rules);
      System.out.println("rulerefs="+collector.rulerefs);
      System.out.println("tokenIDRefs="+collector.tokenIDRefs);
      System.out.println("terminals="+collector.terminals);
      System.out.println("strings="+collector.strings);
      System.out.println("tokensDef="+collector.tokensDefs);
      System.out.println("actions="+collector.actions);
      System.out.println("scopes="+collector.scopes);
       */
  }
 

void checkImport(Token importID) {
	Grammar delegate = g.getImportedGrammar(importID.getText());
	if ( delegate==null ) return;
	List<Integer> validDelegators = validImportTypes.get(delegate.getType());
	if ( validDelegators!=null && !validDelegators.contains(g.getType()) ) {
		g.tool.errMgr.grammarError(ErrorType.INVALID_IMPORT,
								   g.fileName,
								   importID,
								   g, delegate);
	}
	if ( g.isCombined() &&
		 (delegate.name.equals(g.name+Grammar.getGrammarTypeToFileNameSuffix(ANTLRParser.LEXER))||
		  delegate.name.equals(g.name+Grammar.getGrammarTypeToFileNameSuffix(ANTLRParser.PARSER))) )
	{
		g.tool.errMgr.grammarError(ErrorType.IMPORT_NAME_CLASH,
								   g.fileName,
								   importID,
								   g, delegate);
	}
}
 

void checkInvalidRuleDef(Token ruleID) {
	String fileName = null;
	if ( ruleID.getInputStream()!=null ) {
		fileName = ruleID.getInputStream().getSourceName();
	}
	if ( g.isLexer() && Character.isLowerCase(ruleID.getText().charAt(0)) ) {
		g.tool.errMgr.grammarError(ErrorType.PARSER_RULES_NOT_ALLOWED,
								   fileName, ruleID, ruleID.getText());
	}
	if ( g.isParser() &&
		Grammar.isTokenName(ruleID.getText()) )
	{
		g.tool.errMgr.grammarError(ErrorType.LEXER_RULES_NOT_ALLOWED,
								   fileName, ruleID, ruleID.getText());
	}
}
 

void checkGrammarName(Token nameToken) {
	String fullyQualifiedName = nameToken.getInputStream().getSourceName();
	if (fullyQualifiedName == null) {
		// This wasn't read from a file.
		return;
	}

	File f = new File(fullyQualifiedName);
	String fileName = f.getName();
	if ( g.originalGrammar!=null ) return; // don't warn about diff if this is implicit lexer
	if ( !Utils.stripFileExtension(fileName).equals(nameToken.getText()) &&
	     !fileName.equals(Grammar.GRAMMAR_FROM_STRING_NAME)) {
		g.tool.errMgr.grammarError(ErrorType.FILE_AND_GRAMMAR_NAME_DIFFER,
								   fileName, nameToken, nameToken.getText(), fileName);
	}
}
 

public ParserFile(OutputModelFactory factory, String fileName) {
	super(factory, fileName);
	Grammar g = factory.getGrammar();
	namedActions = new HashMap<String, Action>();
	for (String name : g.namedActions.keySet()) {
		ActionAST ast = g.namedActions.get(name);
		namedActions.put(name, new Action(factory, ast));
	}
	genPackage = g.tool.genPackage;
	// need the below members in the ST for Python
	genListener = g.tool.gen_listener;
	genVisitor = g.tool.gen_visitor;
	grammarName = g.name;

	if (g.getOptionString("contextSuperClass") != null) {
		contextSuperClass = new ActionText(null, g.getOptionString("contextSuperClass"));
	}
}
 

public ListenerFile(OutputModelFactory factory, String fileName) {
	super(factory, fileName);
	Grammar g = factory.getGrammar();
	parserName = g.getRecognizerName();
	grammarName = g.name;
	for (Rule r : g.rules.values()) {
		Map<String, List<Pair<Integer,AltAST>>> labels = r.getAltLabels();
		if ( labels!=null ) {
			for (Map.Entry<String, List<Pair<Integer, AltAST>>> pair : labels.entrySet()) {
				listenerNames.add(pair.getKey());
				listenerLabelRuleNames.put(pair.getKey(), r.name);
			}
		}
		else {
			// only add rule context if no labels
			listenerNames.add(r.name);
		}
	}
	ActionAST ast = g.namedActions.get("header");
	if ( ast!=null ) header = new Action(factory, ast);
	genPackage = factory.getGrammar().tool.genPackage;
}
 

public static Map<Rule, Set<Rule>> getRuleDependencies(Grammar g, Collection<Rule> rules) {
	Map<Rule, Set<Rule>> dependencies = new HashMap<Rule, Set<Rule>>();

	for (Rule r : rules) {
		List<GrammarAST> tokenRefs = r.ast.getNodesWithType(ANTLRParser.TOKEN_REF);
		for (GrammarAST tref : tokenRefs) {
			Set<Rule> calls = dependencies.get(r);
			if ( calls==null ) {
				calls = new HashSet<Rule>();
				dependencies.put(r, calls);
			}
			calls.add(g.getRule(tref.getText()));
		}
	}

	return dependencies;
}
 

public VisitorFile(OutputModelFactory factory, String fileName) {
	super(factory, fileName);
	Grammar g = factory.getGrammar();
	parserName = g.getRecognizerName();
	grammarName = g.name;
	for (Rule r : g.rules.values()) {
		Map<String, List<Pair<Integer, AltAST>>> labels = r.getAltLabels();
		if ( labels!=null ) {
			for (Map.Entry<String, List<Pair<Integer, AltAST>>> pair : labels.entrySet()) {
				visitorNames.add(pair.getKey());
				visitorLabelRuleNames.put(pair.getKey(), r.name);
			}
		}
		else {
			// if labels, must label all. no need for generic rule visitor then
			visitorNames.add(r.name);
		}
	}
	ActionAST ast = g.namedActions.get("header");
	if ( ast!=null ) header = new Action(factory, ast);
	genPackage = factory.getGrammar().tool.genPackage;
}
 

void assignTokenTypes(Grammar g, List<GrammarAST> tokensDefs,
					  List<GrammarAST> tokenIDs, List<GrammarAST> terminals)
{
	//Grammar G = g.getOutermostGrammar(); // put in root, even if imported

	// create token types for tokens { A, B, C } ALIASES
	for (GrammarAST alias : tokensDefs) {
		if (g.getTokenType(alias.getText()) != Token.INVALID_TYPE) {
			g.tool.errMgr.grammarError(ErrorType.TOKEN_NAME_REASSIGNMENT, g.fileName, alias.token, alias.getText());
		}

		g.defineTokenName(alias.getText());
	}

	// DEFINE TOKEN TYPES FOR TOKEN REFS LIKE ID, INT
	for (GrammarAST idAST : tokenIDs) {
		if (g.getTokenType(idAST.getText()) == Token.INVALID_TYPE) {
			g.tool.errMgr.grammarError(ErrorType.IMPLICIT_TOKEN_DEFINITION, g.fileName, idAST.token, idAST.getText());
		}

		g.defineTokenName(idAST.getText());
	}

	// VERIFY TOKEN TYPES FOR STRING LITERAL REFS LIKE 'while', ';'
	for (GrammarAST termAST : terminals) {
		if (termAST.getType() != ANTLRParser.STRING_LITERAL) {
			continue;
		}

		if (g.getTokenType(termAST.getText()) == Token.INVALID_TYPE) {
			g.tool.errMgr.grammarError(ErrorType.IMPLICIT_STRING_DEFINITION, g.fileName, termAST.token, termAST.getText());
		}
	}

	g.tool.log("semantics", "tokens="+g.tokenNameToTypeMap);
       g.tool.log("semantics", "strings="+g.stringLiteralToTypeMap);
}
 

@Nullable
private static Grammar loadGrammar(VirtualFile grammarFile, Project project, Tool antlr) {
	// basically here I am mimicking the loadGrammar() method from Tool
	// so that I can check for an empty AST coming back.
	GrammarRootAST grammarRootAST = parseGrammar(project, antlr, grammarFile);
	if ( grammarRootAST==null ) {
		return null;
	}

	// Create a grammar from the AST so we can figure out what type it is
	Grammar g = antlr.createGrammar(grammarRootAST);
	g.fileName = grammarFile.getPath();

	return g;
}
 

public AttributeChecks(Grammar g, Rule r, Alternative alt, ActionAST node, Token actionToken) {
      this.g = g;
      this.r = r;
      this.alt = alt;
      this.node = node;
      this.actionToken = actionToken;
this.errMgr = g.tool.errMgr;
  }
 

public ActionSniffer(Grammar g, Rule r, Alternative alt, ActionAST node, Token actionToken) {
	this.g = g;
	this.r = r;
	this.alt = alt;
	this.node = node;
	this.actionToken = actionToken;
	this.errMgr = g.tool.errMgr;
}
 

private static void parseWithListeners( Grammar grammar, String query, ANTLRErrorListener lexerListener, ANTLRErrorListener parserListener )
{
    LexerInterpreter lexer = grammar.createLexerInterpreter( new ANTLRInputStream( query ) );
    ParserInterpreter parser = grammar.createParserInterpreter( new CommonTokenStream( lexer ) );
    lexer.removeErrorListeners();
    parser.removeErrorListeners();
    lexer.addErrorListener( lexerListener );
    parser.addErrorListener( parserListener );
    parser.parse( grammar.getRule( Antlr4.PREFIX + "Cypher" ).index );
}
 

private boolean isGrammarStale(ANTLRv4GrammarProperties grammarProperties) {
	String sourcePath = grammarProperties.resolveLibDir(project, getParentDir(grammarFile));
	String fullyQualifiedInputFileName = sourcePath+File.separator+grammarFile.getName();

	ANTLRv4PluginController controller = ANTLRv4PluginController.getInstance(project);
	final PreviewState previewState = controller.getPreviewState(grammarFile);
	Grammar g = previewState.getMainGrammar();
	// Grammar should be updated in the preview state before calling this function
	if ( g==null ) {
		return false;
	}

	String language = g.getOptionString(ANTLRv4GrammarProperties.PROP_LANGUAGE);
	CodeGenerator generator = new CodeGenerator(null, g, language);
	String recognizerFileName = generator.getRecognizerFileName();

	VirtualFile contentRoot = getContentRoot(project, grammarFile);
	String package_ = grammarProperties.getPackage();
	String outputDirName = grammarProperties.resolveOutputDirName(project, contentRoot, package_);
	String fullyQualifiedOutputFileName = outputDirName+File.separator+recognizerFileName;

	File inF = new File(fullyQualifiedInputFileName);
	File outF = new File(fullyQualifiedOutputFileName);
	boolean stale = inF.lastModified()>outF.lastModified();
	LOG.info((!stale ? "not" : "") + "stale: " + fullyQualifiedInputFileName + " -> " + fullyQualifiedOutputFileName);
	return stale;
}
 

public static void trackTokenRuleRefsInActions(Grammar g) {
	for (Rule r : g.rules.values()) {
		for (int i=1; i<=r.numberOfAlts; i++) {
			Alternative alt = r.alt[i];
			for (ActionAST a : alt.actions) {
				ActionSniffer sniffer =	new ActionSniffer(g, r, alt, a, a.token);
				sniffer.examineAction();
			}
		}
	}
}
 

private String updateGrammarObjectsFromFile_(VirtualFile grammarFile) {
	String grammarFileName = grammarFile.getPath();
	PreviewState previewState = getPreviewState(grammarFile);
	Grammar[] grammars = ParsingUtils.loadGrammars(grammarFile, project);
	if (grammars != null) {
		synchronized (previewState) { // build atomically
			previewState.lg = (LexerGrammar)grammars[0];
			previewState.g = grammars[1];
		}
	}
	return grammarFileName;
}
 

RulePropertyRef getRulePropertyRef(Token x, Token prop) {
	Grammar g = factory.getGrammar();
	try {
		Class<? extends RulePropertyRef> c = rulePropToModelMap.get(prop.getText());
		Constructor<? extends RulePropertyRef> ctor = c.getConstructor(StructDecl.class, String.class);
		RulePropertyRef ref =
			ctor.newInstance(nodeContext, getRuleLabel(x.getText()));
		return ref;
	}
	catch (Exception e) {
		g.tool.errMgr.toolError(ErrorType.INTERNAL_ERROR, e, prop.getText());
	}
	return null;
}
 

public void buildLexerRuleActions(Lexer lexer, final Rule r) {
	if (r.actions.isEmpty()) {
		return;
	}

	CodeGenerator gen = delegate.getGenerator();
	Grammar g = delegate.getGrammar();
	String ctxType = gen.getTarget().getRuleFunctionContextStructName(r);
	RuleActionFunction raf = lexer.actionFuncs.get(r);
	if ( raf==null ) {
		raf = new RuleActionFunction(delegate, r, ctxType);
	}

	for (ActionAST a : r.actions) {
		if ( a instanceof PredAST ) {
			PredAST p = (PredAST)a;
			RuleSempredFunction rsf = lexer.sempredFuncs.get(r);
			if ( rsf==null ) {
				rsf = new RuleSempredFunction(delegate, r, ctxType);
				lexer.sempredFuncs.put(r, rsf);
			}
			rsf.actions.put(g.sempreds.get(p), new Action(delegate, p));
		}
		else if ( a.getType()== ANTLRParser.ACTION ) {
			raf.actions.put(g.lexerActions.get(a), new Action(delegate, a));
		}
	}

	if (!raf.actions.isEmpty() && !lexer.actionFuncs.containsKey(r)) {
		// only add to lexer if the function actually contains actions
		lexer.actionFuncs.put(r, raf);
	}
}
 

boolean checkRuleRefOptions(RuleRefAST elem, GrammarAST ID, GrammarAST valueAST) {
	Token optionID = ID.token;
	String fileName = optionID.getInputStream().getSourceName();
	// don't care about id<SimpleValue> options
	if ( valueAST!=null && !Grammar.ruleRefOptions.contains(optionID.getText()) ) {
		g.tool.errMgr.grammarError(ErrorType.ILLEGAL_OPTION,
								   fileName,
								   optionID,
								   optionID.getText());
		return false;
	}
	// TODO: extra checks depending on rule kind?
	return true;
}