Java Code Examples for org.antlr.v4.runtime.Token#INVALID_TYPE
The following examples show how to use
org.antlr.v4.runtime.Token#INVALID_TYPE .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: LexerAdaptor.java From ast with Apache License 2.0 | 6 votes |
@Override public Token emit() { if (_type == ANTLRv4Lexer.ID) { String firstChar = _input.getText(Interval.of(_tokenStartCharIndex, _tokenStartCharIndex)); if (Character.isUpperCase(firstChar.charAt(0))) { _type = ANTLRv4Lexer.TOKEN_REF; } else { _type = ANTLRv4Lexer.RULE_REF; } if (_currentRuleType == Token.INVALID_TYPE) { // if outside of rule def _currentRuleType = _type; // set to inside lexer or parser rule } } else if (_type == ANTLRv4Lexer.SEMI) { // exit rule def _currentRuleType = Token.INVALID_TYPE; } return super.emit(); }
Example 2
Source File: Formatter.java From codebuff with BSD 2-Clause "Simplified" License | 6 votes |
public static void wipeCharPositionInfoAndWhitespaceTokens(CodeBuffTokenStream tokens) { tokens.fill(); CommonToken dummy = new CommonToken(Token.INVALID_TYPE, ""); dummy.setChannel(Token.HIDDEN_CHANNEL); Token firstRealToken = tokens.getNextRealToken(-1); for (int i = 0; i<tokens.size(); i++) { if ( i==firstRealToken.getTokenIndex() ) continue; // don't wack first token CommonToken t = (CommonToken)tokens.get(i); if ( t.getText().matches("\\s+") ) { tokens.getTokens().set(i, dummy); // wack whitespace token so we can't use it during prediction } else { t.setLine(0); t.setCharPositionInLine(-1); } } }
Example 3
Source File: ANTLRv4GrammarParser.java From intellij-plugin-v4 with BSD 3-Clause "New" or "Revised" License | 6 votes |
@Override protected ParseTree parse(Parser parser, IElementType root) { int startRule; if (root instanceof IFileElementType) { startRule = ANTLRv4Parser.RULE_grammarSpec; } else if (root == ANTLRv4TokenTypes.TOKEN_ELEMENT_TYPES.get(ANTLRv4Lexer.TOKEN_REF) || root == ANTLRv4TokenTypes.TOKEN_ELEMENT_TYPES.get(ANTLRv4Lexer.RULE_REF)) { startRule = ANTLRv4Parser.RULE_atom; } else { startRule = Token.INVALID_TYPE; } switch (startRule) { case ANTLRv4Parser.RULE_grammarSpec: return ((ANTLRv4Parser) parser).grammarSpec(); case ANTLRv4Parser.RULE_atom: return ((ANTLRv4Parser) parser).atom(); default: String ruleName = ANTLRv4Parser.ruleNames[startRule]; throw new UnsupportedOperationException(String.format("cannot start parsing using root element %s", root)); } }
Example 4
Source File: Grammar.java From codebuff with BSD 2-Clause "Simplified" License | 5 votes |
public int getTokenType(String token) { Integer I; if ( token.charAt(0)=='\'') { I = stringLiteralToTypeMap.get(token); } else { // must be a label like ID I = tokenNameToTypeMap.get(token); } int i = (I!=null)? I : Token.INVALID_TYPE; //tool.log("grammar", "grammar type "+type+" "+tokenName+"->"+i); return i; }
Example 5
Source File: Grammar.java From codebuff with BSD 2-Clause "Simplified" License | 5 votes |
/** Given a token type, get a meaningful name for it such as the ID * or string literal. If this is a lexer and the ttype is in the * char vocabulary, compute an ANTLR-valid (possibly escaped) char literal. */ public String getTokenDisplayName(int ttype) { // inside any target's char range and is lexer grammar? if ( isLexer() && ttype >= Lexer.MIN_CHAR_VALUE && ttype <= Lexer.MAX_CHAR_VALUE ) { return CharSupport.getANTLRCharLiteralForChar(ttype); } if ( ttype==Token.EOF ) { return "EOF"; } if ( ttype==Token.INVALID_TYPE ) { return INVALID_TOKEN_NAME; } if (ttype >= 0 && ttype < typeToStringLiteralList.size() && typeToStringLiteralList.get(ttype) != null) { return typeToStringLiteralList.get(ttype); } if (ttype >= 0 && ttype < typeToTokenList.size() && typeToTokenList.get(ttype) != null) { return typeToTokenList.get(ttype); } return String.valueOf(ttype); }
Example 6
Source File: Grammar.java From codebuff with BSD 2-Clause "Simplified" License | 5 votes |
public int defineStringLiteral(String lit, int ttype) { if ( !stringLiteralToTypeMap.containsKey(lit) ) { stringLiteralToTypeMap.put(lit, ttype); // track in reverse index too if ( ttype>=typeToStringLiteralList.size() ) { Utils.setSize(typeToStringLiteralList, ttype+1); } typeToStringLiteralList.set(ttype, lit); setTokenForType(ttype, lit); return ttype; } return Token.INVALID_TYPE; }
Example 7
Source File: Target.java From codebuff with BSD 2-Clause "Simplified" License | 5 votes |
public String getElementName(String name) { if (".".equals(name)) { return "_wild"; } if ( getCodeGenerator().g.getRule(name)!=null ) return name; int ttype = getCodeGenerator().g.getTokenType(name); if ( ttype==Token.INVALID_TYPE ) return name; return getTokenTypeAsTargetLabel(getCodeGenerator().g, ttype); }
Example 8
Source File: SemanticPipeline.java From codebuff with BSD 2-Clause "Simplified" License | 5 votes |
void assignTokenTypes(Grammar g, List<GrammarAST> tokensDefs, List<GrammarAST> tokenIDs, List<GrammarAST> terminals) { //Grammar G = g.getOutermostGrammar(); // put in root, even if imported // create token types for tokens { A, B, C } ALIASES for (GrammarAST alias : tokensDefs) { if (g.getTokenType(alias.getText()) != Token.INVALID_TYPE) { g.tool.errMgr.grammarError(ErrorType.TOKEN_NAME_REASSIGNMENT, g.fileName, alias.token, alias.getText()); } g.defineTokenName(alias.getText()); } // DEFINE TOKEN TYPES FOR TOKEN REFS LIKE ID, INT for (GrammarAST idAST : tokenIDs) { if (g.getTokenType(idAST.getText()) == Token.INVALID_TYPE) { g.tool.errMgr.grammarError(ErrorType.IMPLICIT_TOKEN_DEFINITION, g.fileName, idAST.token, idAST.getText()); } g.defineTokenName(idAST.getText()); } // VERIFY TOKEN TYPES FOR STRING LITERAL REFS LIKE 'while', ';' for (GrammarAST termAST : terminals) { if (termAST.getType() != ANTLRParser.STRING_LITERAL) { continue; } if (g.getTokenType(termAST.getText()) == Token.INVALID_TYPE) { g.tool.errMgr.grammarError(ErrorType.IMPLICIT_STRING_DEFINITION, g.fileName, termAST.token, termAST.getText()); } } g.tool.log("semantics", "tokens="+g.tokenNameToTypeMap); g.tool.log("semantics", "strings="+g.stringLiteralToTypeMap); }
Example 9
Source File: SemanticPipeline.java From codebuff with BSD 2-Clause "Simplified" License | 5 votes |
/** * Assign constant values to custom channels defined in a grammar. * * @param g The grammar. * @param channelDefs A collection of AST nodes defining individual channels * within a {@code channels{}} block in the grammar. */ void assignChannelTypes(Grammar g, List<GrammarAST> channelDefs) { Grammar outermost = g.getOutermostGrammar(); for (GrammarAST channel : channelDefs) { String channelName = channel.getText(); // Channel names can't alias tokens or modes, because constant // values are also assigned to them and the ->channel(NAME) lexer // command does not distinguish between the various ways a constant // can be declared. This method does not verify that channels do not // alias rules, because rule names are not associated with constant // values in ANTLR grammar semantics. if (g.getTokenType(channelName) != Token.INVALID_TYPE) { g.tool.errMgr.grammarError(ErrorType.CHANNEL_CONFLICTS_WITH_TOKEN, g.fileName, channel.token, channelName); } if (LexerATNFactory.COMMON_CONSTANTS.containsKey(channelName)) { g.tool.errMgr.grammarError(ErrorType.CHANNEL_CONFLICTS_WITH_COMMON_CONSTANTS, g.fileName, channel.token, channelName); } if (outermost instanceof LexerGrammar) { LexerGrammar lexerGrammar = (LexerGrammar)outermost; if (lexerGrammar.modes.containsKey(channelName)) { g.tool.errMgr.grammarError(ErrorType.CHANNEL_CONFLICTS_WITH_MODE, g.fileName, channel.token, channelName); } } outermost.defineChannelName(channel.getText()); } }
Example 10
Source File: XPath.java From antlr4-intellij-adaptor with BSD 2-Clause "Simplified" License | 5 votes |
/** * Convert word like {@code *} or {@code ID} or {@code expr} to a path * element. {@code anywhere} is {@code true} if {@code //} precedes the * word. */ protected XPathElement getXPathElement(Token wordToken, boolean anywhere) { if ( wordToken.getType()==Token.EOF ) { throw new IllegalArgumentException("Missing path element at end of path"); } String word = wordToken.getText(); Integer ttype = tokenTypes.get(word); Integer ruleIndex = ruleIndexes.get(word); switch ( wordToken.getType() ) { case XPathLexer.WILDCARD : return anywhere ? new XPathWildcardAnywhereElement() : new XPathWildcardElement(); case XPathLexer.TOKEN_REF : case XPathLexer.STRING : if ( ttype==null || ttype==Token.INVALID_TYPE ) { throw new IllegalArgumentException(word+ " at index "+ wordToken.getStartIndex()+ " isn't a valid token name"); } return anywhere ? new XPathTokenAnywhereElement(word, ttype) : new XPathTokenElement(word, ttype); default : if ( ruleIndex==null || ruleIndex==-1 ) { throw new IllegalArgumentException(word+ " at index "+ wordToken.getStartIndex()+ " isn't a valid rule name"); } return anywhere ? new XPathRuleAnywhereElement(word, ruleIndex) : new XPathRuleElement(word, ruleIndex); } }
Example 11
Source File: DelimiterLexer.java From presto with Apache License 2.0 | 4 votes |
@Override public Token nextToken() { if (_input == null) { throw new IllegalStateException("nextToken requires a non-null input stream."); } // Mark start location in char stream so unbuffered streams are // guaranteed at least have text of current token int tokenStartMarker = _input.mark(); try { outer: while (true) { if (_hitEOF) { emitEOF(); return _token; } _token = null; _channel = Token.DEFAULT_CHANNEL; _tokenStartCharIndex = _input.index(); _tokenStartCharPositionInLine = getInterpreter().getCharPositionInLine(); _tokenStartLine = getInterpreter().getLine(); _text = null; do { _type = Token.INVALID_TYPE; int ttype = -1; // This entire method is copied from org.antlr.v4.runtime.Lexer, with the following bit // added to match the delimiters before we attempt to match the token boolean found = false; for (String terminator : delimiters) { if (match(terminator)) { ttype = SqlBaseParser.DELIMITER; found = true; break; } } if (!found) { try { ttype = getInterpreter().match(_input, _mode); } catch (LexerNoViableAltException e) { notifyListeners(e); // report error recover(e); ttype = SKIP; } } if (_input.LA(1) == IntStream.EOF) { _hitEOF = true; } if (_type == Token.INVALID_TYPE) { _type = ttype; } if (_type == SKIP) { continue outer; } } while (_type == MORE); if (_token == null) { emit(); } return _token; } } finally { // make sure we release marker after match or // unbuffered char stream will keep buffering _input.release(tokenStartMarker); } }
Example 12
Source File: DelimiterLexer.java From rainbow with Apache License 2.0 | 4 votes |
@Override public Token nextToken() { if (_input == null) { throw new IllegalStateException("nextToken requires a non-null input stream."); } // Mark start location in char stream so unbuffered streams are // guaranteed at least have text of current token int tokenStartMarker = _input.mark(); try { outer: while (true) { if (_hitEOF) { emitEOF(); return _token; } _token = null; _channel = Token.DEFAULT_CHANNEL; _tokenStartCharIndex = _input.index(); _tokenStartCharPositionInLine = getInterpreter().getCharPositionInLine(); _tokenStartLine = getInterpreter().getLine(); _text = null; do { _type = Token.INVALID_TYPE; int ttype = -1; // This entire method is copied from org.antlr.v4.runtime.Lexer, with the following bit // added to match the delimiters before we attempt to match the token boolean found = false; for (String terminator : delimiters) { if (match(terminator)) { ttype = SqlBaseParser.DELIMITER; found = true; break; } } if (!found) { try { ttype = getInterpreter().match(_input, _mode); } catch (LexerNoViableAltException e) { notifyListeners(e); // report error recover(e); ttype = SKIP; } } if (_input.LA(1) == IntStream.EOF) { _hitEOF = true; } if (_type == Token.INVALID_TYPE) { _type = ttype; } if (_type == SKIP) { continue outer; } } while (_type == MORE); if (_token == null) { emit(); } return _token; } } finally { // make sure we release marker after match or // unbuffered char stream will keep buffering _input.release(tokenStartMarker); } }
Example 13
Source File: DelimiterLexer.java From macrobase with Apache License 2.0 | 4 votes |
@Override public Token nextToken() { if (_input == null) { throw new IllegalStateException("nextToken requires a non-null input stream."); } // Mark start location in char stream so unbuffered streams are // guaranteed at least have text of current token int tokenStartMarker = _input.mark(); try { outer: while (true) { if (_hitEOF) { emitEOF(); return _token; } _token = null; _channel = Token.DEFAULT_CHANNEL; _tokenStartCharIndex = _input.index(); _tokenStartCharPositionInLine = getInterpreter().getCharPositionInLine(); _tokenStartLine = getInterpreter().getLine(); _text = null; do { _type = Token.INVALID_TYPE; int ttype = -1; // This entire method is copied from org.antlr.v4.runtime.Lexer, with the following bit // added to match the delimiters before we attempt to match the token boolean found = false; for (String terminator : delimiters) { if (match(terminator)) { ttype = SqlBaseParser.DELIMITER; found = true; break; } } if (!found) { try { ttype = getInterpreter().match(_input, _mode); } catch (LexerNoViableAltException e) { notifyListeners(e); // report error recover(e); ttype = SKIP; } } if (_input.LA(1) == IntStream.EOF) { _hitEOF = true; } if (_type == Token.INVALID_TYPE) { _type = ttype; } if (_type == SKIP) { continue outer; } } while (_type == MORE); if (_token == null) { emit(); } return _token; } } finally { // make sure we release marker after match or // unbuffered char stream will keep buffering _input.release(tokenStartMarker); } }