org.antlr.v4.runtime.Token Java Examples

/**
 * Consume all tokens a whole line at a time until the next token is one expected by the current
 * rule. Each line (as delimited by supplied separator token) starting from the current line up to
 * the last line consumed is placed in an {@link ErrorNode} and inserted as a child of the current
 * rule.
 *
 * @param recognizer The {@link Parser} to whom to delegate creation of each {@link ErrorNode}
 */
private void consumeBlocksUntilWanted(Parser recognizer) {
  IntervalSet expecting = recognizer.getExpectedTokens();
  IntervalSet whatFollowsLoopIterationOrRule = expecting.or(getErrorRecoverySet(recognizer));

  int nextToken;
  do {
    // Eat tokens until we are at the end of the line
    consumeUntilEndOfLine(recognizer);

    // Get the line number and separator text from the separator token
    Token separatorToken = recognizer.getCurrentToken();

    // Insert the current line as an {@link ErrorNode} as a child of the current rule
    createErrorNode(recognizer, recognizer.getContext(), separatorToken);

    // Eat the separator token
    recognizer.consume();

    nextToken = recognizer.getInputStream().LA(1);
  } while (!whatFollowsLoopIterationOrRule.contains(nextToken) && nextToken != Lexer.EOF);
}
 

private Expr<?> createMutliplicativeExpr(final Expr<?> opsHead, final List<? extends Expr<?>> opsTail,
										 final List<? extends Token> opers) {
	checkArgument(opsTail.size() == opers.size());

	if (opsTail.isEmpty()) {
		return opsHead;
	} else {
		final Expr<?> newOpsHead = opsTail.get(0);
		final List<? extends Expr<?>> newOpsTail = opsTail.subList(1, opsTail.size());

		final Token operHead = opers.get(0);
		final List<? extends Token> opersTail = opers.subList(1, opers.size());

		final Expr<?> subExpr = createMultiplicativeSubExpr(opsHead, newOpsHead, operHead);

		return createMutliplicativeExpr(subExpr, newOpsTail, opersTail);
	}
}
 

private Optional<Token> findTrailingComment(@Nullable List<Token> tokensAfter) {
    if (tokensAfter == null) {
        return Optional.empty();
    }
    Optional<Token> trailingComment = Optional.empty();
    for (Token token : tokensAfter) {
        if (token.getType() == LINE_COMMENT) {
            trailingComment = Optional.of(token);
        }
        // Try to find single trailing comment (on the same line).
        // If we hit newline, then we can stop right away.
        if (trailingComment.isPresent() || token.getType() == NL) {
            break;
        }
    }
    return trailingComment;
}
 

private Expr<?> createAdditiveExpr(final Expr<?> opsHead, final List<? extends Expr<?>> opsTail,
								   final List<? extends Token> opers) {
	checkArgument(opsTail.size() == opers.size());

	if (opsTail.isEmpty()) {
		return opsHead;
	} else {
		final Expr<?> newOpsHead = opsTail.get(0);
		final List<? extends Expr<?>> newOpsTail = opsTail.subList(1, opsTail.size());

		final Token operHead = opers.get(0);
		final List<? extends Token> opersTail = opers.subList(1, opers.size());

		final Expr<?> subExpr = createAdditiveSubExpr(opsHead, newOpsHead, operHead);

		return createAdditiveExpr(subExpr, newOpsTail, opersTail);
	}
}
 

protected void reportUnwantedToken(@NotNull Parser recognizer)
{
	if (inErrorRecoveryMode(recognizer))
	{
		return;
	}

	beginErrorCondition(recognizer);

	Token t = recognizer.getCurrentToken();
	String tokenName = getTokenErrorDisplay(t);
	IntervalSet expecting = getExpectedTokens(recognizer);
	String msg = "多余输入 " + tokenName + " 期望 " + expecting.toString(recognizer.getTokenNames());
	BeetlException exception = new BeetlParserException(BeetlException.PARSER_MISS_ERROR, msg);
	//		exception.token = this.getGrammarToken(t);
	exception.pushToken(this.getGrammarToken(t));
	throw exception;
}
 

public static List<TerminalNode> getAllRuleRefNodes(Parser parser, ParseTree tree, String ruleName) {
	List<TerminalNode> nodes = new ArrayList<TerminalNode>();
	Collection<ParseTree> ruleRefs;
	if ( Grammar.isTokenName(ruleName) ) {
		ruleRefs = XPath.findAll(tree, "//lexerRuleBlock//TOKEN_REF", parser);
	}
	else {
		ruleRefs = XPath.findAll(tree, "//ruleBlock//RULE_REF", parser);
	}
	for (ParseTree node : ruleRefs) {
		TerminalNode terminal = (TerminalNode)node;
		Token rrefToken = terminal.getSymbol();
		String r = rrefToken.getText();
		if ( r.equals(ruleName) ) {
			nodes.add(terminal);
		}
	}
	if ( nodes.size()==0 ) return null;
	return nodes;
}
 

public void error(final String kind, final TokenSource tokens, final Object offendingSymbol, final int line, final int charPositionInLine, final int length, final String msg, final Exception e) {
	hasError = true;

	final String filename = tokens.getSourceName();

	System.err.print(filename.substring(filename.lastIndexOf(File.separator) + 1) + ": compilation failed: ");
	System.err.print("Encountered " + kind + " error ");
	if (offendingSymbol != null)
		System.err.print("\"" + offendingSymbol + "\" ");
	System.err.print("at line " + line + ", ");
	if (length > 0)
		System.err.print("columns " + charPositionInLine + "-" + (charPositionInLine + length - 1));
	else
		System.err.print("column " + charPositionInLine);
	System.err.println(". " + msg);

	underlineError(tokens, (Token)offendingSymbol, line, charPositionInLine, length);

	if (e != null)
		for (final StackTraceElement st : e.getStackTrace())
			System.err.println("\tat " + st);
	else
		System.err.println("\tat unknown stack");
}
 

/** Get start/stop of an entire rule including semi and then clean up
 *  WS at end.
 */
public static String getRuleText(CommonTokenStream tokens, ParserRuleContext ruleDefNode) {
	Token stop = ruleDefNode.getStop();
	Token semi = stop;
	TerminalNode colonNode = ruleDefNode.getToken(ANTLRv4Parser.COLON, 0);
	Token colon = colonNode.getSymbol();
	Token beforeSemi = tokens.get(stop.getTokenIndex()-1);
	Token afterColon = tokens.get(colon.getTokenIndex()+1);

	// trim whitespace/comments before / after rule text
	List<Token> ignoreBefore = tokens.getHiddenTokensToRight(colon.getTokenIndex());
	List<Token> ignoreAfter = tokens.getHiddenTokensToLeft(semi.getTokenIndex());
	Token textStart = afterColon;
	Token textStop = beforeSemi;
	if ( ignoreBefore!=null ) {
		Token lastWSAfterColon = ignoreBefore.get(ignoreBefore.size()-1);
		textStart = tokens.get(lastWSAfterColon.getTokenIndex()+1);
	}
	if ( ignoreAfter!=null ) {
		int firstWSAtEndOfRule = ignoreAfter.get(0).getTokenIndex()-1;
		textStop = tokens.get(firstWSAtEndOfRule); // stop before 1st ignore token at end
	}

	return tokens.getText(textStart, textStop);
}
 

private static IntervalLiteral.IntervalField getIntervalFieldType(Token token)
{
    switch (token.getType()) {
        case SqlBaseLexer.YEAR:
            return IntervalLiteral.IntervalField.YEAR;
        case SqlBaseLexer.MONTH:
            return IntervalLiteral.IntervalField.MONTH;
        case SqlBaseLexer.DAY:
            return IntervalLiteral.IntervalField.DAY;
        case SqlBaseLexer.HOUR:
            return IntervalLiteral.IntervalField.HOUR;
        case SqlBaseLexer.MINUTE:
            return IntervalLiteral.IntervalField.MINUTE;
        case SqlBaseLexer.SECOND:
            return IntervalLiteral.IntervalField.SECOND;
    }

    throw new IllegalArgumentException("Unsupported interval field: " + token.getText());
}
 

public ArithmeticNode createBinary(Token opToken, ExpressionNode leftNode, ExpressionNode rightNode) {
    final ArithmeticNode result;
    switch (opToken.getText()) {
        case "+":
            result = new ArithmeticNode(ArithmeticNode.Operation.ADD, leftNode, rightNode);
            break;
        case "-":
            result = new ArithmeticNode(ArithmeticNode.Operation.SUBTRACT, leftNode, rightNode);
            break;
        case "*":
            result = new ArithmeticNode(ArithmeticNode.Operation.MULTIPLY, leftNode, rightNode);
            break;
        case "/":
            result = new ArithmeticNode(ArithmeticNode.Operation.DIVIDE, leftNode, rightNode);
            break;
        case "%":
            result = new ArithmeticNode(ArithmeticNode.Operation.MODULO, leftNode, rightNode);
            break;
        default:
            // should not happen due to lexer
            throw new GrCUDAInternalException("unexpected operation: " + opToken.getText());
    }
    return result;
}
 

/**
 * Create an error node with the text of the current line and insert it into parse tree
 *
 * @param recognizer The recognizer with which to create the error node
 * @param separator The token that ends the unrecognized link. This is also used to determine the
 *     index of the line to return in error messages.
 * @return The token contained in the error node
 */
private Token createErrorNode(Parser recognizer, ParserRuleContext ctx, Token separator) {
  if (_recoveredAtEof) {
    _recoveredAtEof = false;
    throw new BatfishRecognitionException(recognizer, recognizer.getInputStream(), ctx);
  }
  if (separator.getType() == Lexer.EOF) {
    _recoveredAtEof = true;
  }
  String lineText = _lines[separator.getLine() - 1] + separator.getText();
  Token lineToken =
      new UnrecognizedLineToken(lineText, separator.getLine(), _parserStateAtRecovery);
  ErrorNode errorNode = recognizer.createErrorNode(ctx, lineToken);
  ctx.addErrorNode(errorNode);
  return lineToken;
}
 

public static boolean isClassName(TokenStream _input) {
  try {
    int i=1;
    Token token = _input.LT(i);
    while (token!=null && i < _input.size() && _input.LT(i+1).getType() == GroovyParser.DOT) {
      i = i + 2;
      token = _input.LT(i);
    }
    if(token==null)
      return false;
    // TODO here
    return Character.isUpperCase(Character.codePointAt(token.getText(), 0));
  } catch(Exception e) {
    e.printStackTrace();
  }
  
  return false;
}
 

public ExecutableCodeDef(ParserRuleContext tree,
                         String inputFilename,
                         int index,
                         Token startOrRefToken,
                         Map<String,String> attributes,
                         String code)
{
	super(index, startOrRefToken);
	this.tree = tree;
	this.inputFilename = inputFilename;
	this.code = expandTabs(code, 4);
	this.attributes = attributes;
	if ( attributes!=null ) {
		this.label = attributes.get("label");
		if ( attributes.containsKey("hide") ) {
			isCodeVisible = !attributes.get("hide").equals("true");
		}
		if ( attributes.containsKey("output") ) {
			displayExpr = attributes.get("output");
		}
	}
}
 

/**
 * Finds appropriate documentation comment which belongs to the given context.
 * Overloaded version for structure field.
 *
 * @param ctx Parser context.
 *
 * @return Parsed documentation comment or null.
 */
public DocComment findDocComment(ZserioParser.StructureFieldDefinitionContext ctx)
{
    // before field alignment
    Token docToken = findDocTokenBefore(ctx.fieldAlignment());
    if (docToken != null)
        return parseDocComment(docToken);

    // before field offset
    docToken = findDocTokenBefore(ctx.fieldOffset());
    if (docToken != null)
        return parseDocComment(docToken);

    // before optional keyword
    docToken = findDocTokenBefore(ctx.OPTIONAL());
    if (docToken != null)
        return parseDocComment(docToken);

    // before field type
    docToken = findDocTokenBefore(ctx.fieldTypeId());
    if (docToken != null)
        return parseDocComment(docToken);

    return null;
}
 

public CodeBuffTokenStream(CommonTokenStream stream) {
	super(stream.getTokenSource());
	this.fetchedEOF = false;
	for (Token t : stream.getTokens()) {
		tokens.add(new CommonToken(t));
	}
	reset();
}
 

@Override
public ExternType visitExternType(ZserioParser.ExternTypeContext ctx)
{
    final Token token = ctx.getStart();
    final AstLocation location = new AstLocation(token);

    return new ExternType(location, token.getText());
}
 

public MethodDeclaration(final String name, final StaticScope myEnclosedScope,
		final Type returnType, final AccessQualifier accessQualifier, final Token token,
		final boolean isStatic) {
	super(name);
	signature_ = new MethodSignature(name(), returnType, accessQualifier, token, isStatic);
	isAConstructor_ = calculateConstructorStatus(myEnclosedScope);
	this.commonInit(myEnclosedScope, returnType, accessQualifier, token);
}
 

private Token getAddedToken(String suggestedCompletion) {
    String completedText = this.input + suggestedCompletion;
    List<? extends Token> completedTextTokens = this.lexerWrapper.tokenizeNonDefaultChannel(completedText).tokens;
    if (completedTextTokens.size() <= inputTokens.size()) {
        return null; // Completion didn't yield whole token, could be just a token fragment
    }
    logger.debug("TOKENS IN COMPLETED TEXT: " + completedTextTokens);
    Token newToken = completedTextTokens.get(completedTextTokens.size() - 1);
    return newToken;
}
 

@Override
public FloatType visitFloatType(ZserioParser.FloatTypeContext ctx)
{
    final Token token = ctx.getStart();
    final AstLocation location = new AstLocation(token);

    return new FloatType(location, token.getText(), token.getType());
}
 

static Token verifyToken(final ParseTree parent, final int offset, final int expected) {
    final TerminalNode node = verifyTerminal(parent.getChild(offset));
    final Token ret = node.getSymbol();
    final int type = ret.getType();
    verify(type == expected, "Item %s has type %s, expected %s", node, type, expected);
    return ret;
}
 

public static TerminalNode getMatchingLeftSymbol(Corpus corpus,
                                                 InputDocument doc,
                                                 TerminalNode node)
{
	ParserRuleContext parent = (ParserRuleContext)node.getParent();
	int curTokensParentRuleIndex = parent.getRuleIndex();
	Token curToken = node.getSymbol();
	if (corpus.ruleToPairsBag != null) {
		String ruleName = doc.parser.getRuleNames()[curTokensParentRuleIndex];
		RuleAltKey ruleAltKey = new RuleAltKey(ruleName, parent.getAltNumber());
		List<Pair<Integer, Integer>> pairs = corpus.ruleToPairsBag.get(ruleAltKey);
		if ( pairs!=null ) {
			// Find appropriate pair given current token
			// If more than one pair (a,b) with b=current token pick first one
			// or if a common pair like ({,}), then give that one preference.
			// or if b is punctuation, prefer a that is punct
			List<Integer> viableMatchingLeftTokenTypes = viableLeftTokenTypes(parent, curToken, pairs);
			Vocabulary vocab = doc.parser.getVocabulary();
			if ( !viableMatchingLeftTokenTypes.isEmpty() ) {
				int matchingLeftTokenType =
					CollectTokenPairs.getMatchingLeftTokenType(curToken, viableMatchingLeftTokenTypes, vocab);
				List<TerminalNode> matchingLeftNodes = parent.getTokens(matchingLeftTokenType);
				// get matching left node by getting last node to left of current token
				List<TerminalNode> nodesToLeftOfCurrentToken =
					filter(matchingLeftNodes, n -> n.getSymbol().getTokenIndex()<curToken.getTokenIndex());
				TerminalNode matchingLeftNode = nodesToLeftOfCurrentToken.get(nodesToLeftOfCurrentToken.size()-1);
				if (matchingLeftNode == null) {
					System.err.println("can't find matching node for "+node.getSymbol());
				}
				return matchingLeftNode;
			}
		}
	}
	return null;
}
 

@Override protected <ExprType> Expression expressionFromReturnStatement(final ExprType ctxExpr, final RuleContext ctxParent, final Token ctxGetStart)
{
	Expression expressionReturned = null;
	if (null != ctxExpr) {
		expressionReturned = parsingData_.popExpression();
	}
	final MethodDeclaration methodDecl = (MethodDeclaration)findProperMethodScopeAround(ctxExpr, ctxParent, ctxGetStart);
	assert null == methodDecl || methodDecl instanceof MethodDeclaration;
	
	if (null == methodDecl) {
		expressionReturned = new ErrorExpression(null);
		ErrorLogger.error(ErrorIncidenceType.Fatal, ctxGetStart, "Return statement must be within a method scope ", "", "", "");
	} else {
		if (methodDecl.returnType() == null || methodDecl.returnType().name().equals("void")) {
			if (null == expressionReturned || expressionReturned.type().name().equals("void")) {
				;
			} else {
				ErrorLogger.error(ErrorIncidenceType.Fatal, ctxGetStart, "Return expression `", expressionReturned.getText(), "' of type ",
						expressionReturned.type().getText(), " is incompatible with method that returns no value.", "");
				expressionReturned = new ErrorExpression(null);
			}
		} else if (methodDecl.returnType().canBeConvertedFrom(expressionReturned.type())) {
			;
		} else {
			if (expressionReturned.isntError()) {
				ErrorLogger.error(ErrorIncidenceType.Fatal, ctxGetStart,
						"Return expression `" + expressionReturned.getText(),
						"`' of type `", expressionReturned.type().getText(),
						"' is not compatible with declared return type `",
						methodDecl.returnType().getText(), "'.");
				expressionReturned = new ErrorExpression(expressionReturned);
			}
		}
	}
	return expressionReturned;
}
 

/** Returns {@code true} iff {@code t}'s text represents a valid vlan ID (1-4094) in base 10. */
protected static boolean isVlanId(Token t) {
  try {
    int val = Integer.parseInt(t.getText());
    checkArgument(1 <= val && val <= 4094);
  } catch (IllegalArgumentException e) {
    return false;
  }
  return true;
}
 

private void setRectangularShape(final RectangularShape sh, final double x, final double y, final double width, final double height,
								final PSTContext ctx, final Token cmd) {
	sh.setPosition(x, y);
	sh.setWidth(Math.max(0.1, width));
	sh.setHeight(Math.max(0.1, height));
	setShapeParameters(sh, ctx);

	if(ctx.starredCmd(cmd)) {
		setShapeForStar(sh);
	}
}
 

public ObjectDeclaration(final String name, final Type type, final Token token) {
	super(name);
	token_ = token;
	lineNumber_ = (null == token_)? 0: token_.getLine();
	type_ = type;
	accessQualifier_ = AccessQualifier.DefaultAccess;
}
 

public MethodDeclaration(final MethodSignature signature, final StaticScope myEnclosedScope, 
		final Token token) {
	super(signature.name());
	signature_ = signature;
	isAConstructor_ = calculateConstructorStatus(myEnclosedScope);
	this.commonInit(myEnclosedScope, signature.returnType(),
			signature.accessQualifier(), token);
}
 

public SQLParseError(Token offendingToken,
                     int line, int charPositionInLine,
                     String msg,
                     String errorLine) {
  super(msg);
  this.offendingToken = offendingToken;
  this.charPositionInLine = charPositionInLine;
  this.line = line;
  this.errorLine = errorLine;
  this.header = msg;
}
 

public void elaborateFromTemplate(final TemplateDeclaration templateDeclaration,
		final TemplateInstantiationInfo newTypes,
		Token token) {
	templateDeclaration_ = templateDeclaration;
	
	// This turns a TemplateDeclaration into a class
	assert null == baseClass_ || baseClass_.type() instanceof ClassType;
	final ClassType baseClassType = null == baseClass_? null: (ClassType)baseClass_.type();
	
	myEnclosedScope_ = new StaticScope(enclosedScope(), "copy", templateDeclaration.enclosingScope(),
			this, newTypes);
	final ClassType newClassType = new ClassType(name(), myEnclosedScope_, baseClassType);
	newClassType.elaborateFromTemplate(templateDeclaration, baseClassType, myEnclosedScope_, this, token);
	this.setType(newClassType);
}
 

@Override
/**
 * media_expression : LPAREN S* IDENT S* (COLON S* terms)? RPAREN
 */
public MediaExpression visitMedia_expression(CSSParser.Media_expressionContext ctx) {
    logEnter("mediaexpression: ", ctx);
    if (ctxHasErrorNode(ctx)) {
        mq.invalid = true;
        return null;
    }
    MediaExpression expr = rf.createMediaExpression();
    Declaration decl;
    declaration_stack.push(new declaration_scope());
    declaration_stack.peek().d = decl = rf.createDeclaration();
    declaration_stack.peek().invalid = false;

    String property = extractTextUnescaped(ctx.IDENT().getText());
    decl.setProperty(property);
    Token token = ctx.IDENT().getSymbol();
    decl.setSource(extractSource((CSSToken) token));
    if (ctx.terms() != null) {
        List<Term<?>> t = visitTerms(ctx.terms());
        decl.replaceAll(t);
    }

    if (declaration_stack.peek().d != null && !declaration_stack.peek().invalid) { //if the declaration is valid
        expr.setFeature(decl.getProperty());
        expr.replaceAll(decl);
    }
    declaration_stack.pop();

    logLeave("mediaexpression");
    return expr;
}
 

public int[] getFeatures(int i)	{
	CodeBuffTokenStream tokens = doc.tokens;
	TerminalNode node = tokenToNodeMap.get(tokens.get(i));
	if ( node==null ) {
		System.err.println("### No node associated with token "+tokens.get(i));
		return null;
	}

	Token curToken = node.getSymbol();
	Token prevToken = tokens.getPreviousRealToken(i);
	Token prevPrevToken = prevToken!=null ? doc.tokens.getPreviousRealToken(prevToken.getTokenIndex()) : null;

	boolean prevTokenStartsLine = false;
	if ( prevToken!=null && prevPrevToken!=null ) {
		prevTokenStartsLine = prevToken.getLine()>prevPrevToken.getLine();
	}

	boolean curTokenStartsNewLine = false;
	if ( prevToken==null ) curTokenStartsNewLine = true; // we must be at start of file
	else if ( curToken.getLine() > prevToken.getLine() ) curTokenStartsNewLine = true;

	int[] features = getContextFeatures(corpus, tokenToNodeMap, doc, i);

	setListInfoFeatures(corpus.tokenToListInfo, features, curToken);

	features[INDEX_PREV_FIRST_ON_LINE]         = prevTokenStartsLine ? 1 : 0;
	features[INDEX_FIRST_ON_LINE]              = curTokenStartsNewLine ? 1 : 0;

	return features;
}