Java Code Examples for org.antlr.runtime.Token#INVALID_TOKEN_TYPE

The following examples show how to use org.antlr.runtime.Token#INVALID_TOKEN_TYPE . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: NbParseTreeBuilder.java    From netbeans with Apache License 2.0 5 votes vote down vote up 
@Override
public void consumeToken(Token token) {
    if (backtracking > 0 || resync) {
        return;
    }

    if (debug_tokens) {
        CommonToken ct = (CommonToken) token;
        int[] ctr = CommonTokenUtil.getCommonTokenOffsetRange(ct);
        System.out.println(token + "(" + ctr[0] + "-" + ctr[1] + ")");
    }

    //ignore the closing EOF token, we do not want it
    //it the parse tree
    if (token.getType() == Css3Lexer.EOF) {
        return;
    }

    //also ignore error tokens - they are added as children of ErrorNode-s in the recognitionException(...) method
    if (token.getType() == Token.INVALID_TOKEN_TYPE) {
        return;
    }

    lastConsumedToken = (CommonToken) token;

    RuleNode ruleNode = callStack.peek();
    TokenNode elementNode = new TokenNode(source, (CommonToken) token);
    elementNode.hiddenTokens = this.hiddenTokens;
    hiddenTokens.clear();
    ruleNode.addChild(elementNode);

    updateFirstTokens(ruleNode, lastConsumedToken);
}
Example 2
Source File: TerminalsTokenTypeToPartitionMapper.java    From xtext-eclipse with Eclipse Public License 2.0 5 votes vote down vote up 
@Override
public String getPartitionType(int antlrTokenType) {
	// on lexer error return default content type
	if (antlrTokenType == Token.INVALID_TOKEN_TYPE) {
		return IDocument.DEFAULT_CONTENT_TYPE;
	}
	return getMappedValue(antlrTokenType);
}
Example 3
Source File: Lexer.java    From gef with Eclipse Public License 2.0 5 votes vote down vote up 
@Override
public Token nextToken() {
	while (true) {
		this.state.token = null;
		this.state.channel = Token.DEFAULT_CHANNEL;
		this.state.tokenStartCharIndex = input.index();
		this.state.tokenStartCharPositionInLine = input
				.getCharPositionInLine();
		this.state.tokenStartLine = input.getLine();
		this.state.text = null;
		if (input.LA(1) == CharStream.EOF) {
			return Token.EOF_TOKEN;
		}
		try {
			mTokens();
			if (this.state.token == null) {
				emit();
			} else if (this.state.token == Token.SKIP_TOKEN) {
				continue;
			}
			return this.state.token;
		} catch (RecognitionException re) {
			reportError(re);
			if (re instanceof NoViableAltException
					|| re instanceof FailedPredicateException) {
				recover(re);
			}
			// create token that holds mismatched char
			Token t = new CommonToken(input, Token.INVALID_TOKEN_TYPE,
					Token.HIDDEN_CHANNEL, this.state.tokenStartCharIndex,
					getCharIndex() - 1);
			t.setLine(this.state.tokenStartLine);
			t.setCharPositionInLine(
					this.state.tokenStartCharPositionInLine);
			tokenErrorMap.put(t, getErrorMessage(re, this.getTokenNames()));
			emit(t);
			return this.state.token;
		}
	}
}
Example 4
Source File: AbstractInternalAntlrParser.java    From xtext-core with Eclipse Public License 2.0 5 votes vote down vote up 
private ILeafNode createLeafNode(Token token, EObject grammarElement) {
	boolean isHidden = token.getChannel() == HIDDEN;
	SyntaxErrorMessage error = null;
	if (!isHidden) {
		if (currentError != null) {
			error = currentError;
			currentError = null;
		}
	}
	if (token.getType() == Token.INVALID_TOKEN_TYPE) {
		if (error == null) {
			String lexerErrorMessage = ((XtextTokenStream) input).getLexerErrorMessage(token);
			LexerErrorContext errorContext = new LexerErrorContext(lexerErrorMessage);
			error = syntaxErrorProvider.getSyntaxErrorMessage(errorContext);
		}
	}
	if (grammarElement == null) {
		String ruleName = antlrTypeToLexerName.get(token.getType());
		grammarElement = allRules.get(ruleName);
	}
	CommonToken commonToken = (CommonToken) token;
	if (error != null)
		hadErrors = true;
	return nodeBuilder.newLeafNode(
			commonToken.getStartIndex(), 
			commonToken.getStopIndex() - commonToken.getStartIndex() + 1, 
			grammarElement, 
			isHidden, 
			error, 
			currentNode);
}
Example 5
Source File: Lexer.java    From xtext-core with Eclipse Public License 2.0 5 votes vote down vote up 
@Override
public Token nextToken() {
	while (true) {
		this.state.token = null;
		this.state.channel = Token.DEFAULT_CHANNEL;
		this.state.tokenStartCharIndex = input.index();
		this.state.tokenStartCharPositionInLine = input.getCharPositionInLine();
		this.state.tokenStartLine = input.getLine();
		this.state.text = null;
		if (input.LA(1) == CharStream.EOF) {
			return Token.EOF_TOKEN;
		}
		try {
			mTokens();
			if (this.state.token == null) {
				emit();
			}
			else if (this.state.token == Token.SKIP_TOKEN) {
				continue;
			}
			return this.state.token;
		}
		catch (RecognitionException re) {
			reportError(re);
			if (re instanceof NoViableAltException ||
				re instanceof FailedPredicateException) {
				recover(re);
			}
			// create token that holds mismatched char
			Token t = new CommonToken(input, Token.INVALID_TOKEN_TYPE, Token.HIDDEN_CHANNEL,
					this.state.tokenStartCharIndex, getCharIndex() - 1);
			t.setLine(this.state.tokenStartLine);
			t.setCharPositionInLine(this.state.tokenStartCharPositionInLine);
			tokenErrorMap.put(t, getErrorMessage(re, this.getTokenNames()));
			emit(t);
			return this.state.token;
		}
	}
}
Example 6
Source File: TokenTypeToStringMapper.java    From xtext-core with Eclipse Public License 2.0 5 votes vote down vote up 
protected String getMappedValue(int tokenType) {
	if (tokenType == Token.INVALID_TOKEN_TYPE) {
		return HighlightingStyles.INVALID_TOKEN_ID;
	} else {
		return mappedValues[tokenType - Token.MIN_TOKEN_TYPE];
	}
}
Example 7
Source File: FixedTerminalsTokenTypeToPartitionMapper.java    From dsl-devkit with Eclipse Public License 1.0 5 votes vote down vote up 
@Override
public String getPartitionType(final int antlrTokenType) {
  // on lexer error return default content type
  if (antlrTokenType == Token.INVALID_TOKEN_TYPE) {
    return IDocument.DEFAULT_CONTENT_TYPE;
  }
  return getMappedValue(antlrTokenType);
}
Example 8
Source File: TokenTypeToStringMapper.java    From xtext-eclipse with Eclipse Public License 2.0 4 votes vote down vote up 
protected String getMappedValue(int tokenType) {
	if (tokenType == Token.INVALID_TOKEN_TYPE)
		return HighlightingStyles.INVALID_TOKEN_ID;
	return mappedValues[tokenType-Token.MIN_TOKEN_TYPE];
}
Example 9
Source File: XtextTokenStream.java    From xtext-core with Eclipse Public License 2.0 4 votes vote down vote up 
public String getLexerErrorMessage(Token invalidToken) {
	if (tokenSource instanceof org.eclipse.xtext.parser.antlr.Lexer) {
		return ((org.eclipse.xtext.parser.antlr.Lexer) tokenSource).getErrorMessage(invalidToken);
	}
	return (invalidToken.getType() == Token.INVALID_TOKEN_TYPE) ? "Invalid token " + invalidToken.getText() : null;
}