Java Code Examples for org.antlr.v4.runtime.Token

/**
 * Consume all tokens a whole line at a time until the next token is one expected by the current
 * rule. Each line (as delimited by supplied separator token) starting from the current line up to
 * the last line consumed is placed in an {@link ErrorNode} and inserted as a child of the current
 * rule.
 *
 * @param recognizer The {@link Parser} to whom to delegate creation of each {@link ErrorNode}
 */
private void consumeBlocksUntilWanted(Parser recognizer) {
  IntervalSet expecting = recognizer.getExpectedTokens();
  IntervalSet whatFollowsLoopIterationOrRule = expecting.or(getErrorRecoverySet(recognizer));

  int nextToken;
  do {
    // Eat tokens until we are at the end of the line
    consumeUntilEndOfLine(recognizer);

    // Get the line number and separator text from the separator token
    Token separatorToken = recognizer.getCurrentToken();

    // Insert the current line as an {@link ErrorNode} as a child of the current rule
    createErrorNode(recognizer, recognizer.getContext(), separatorToken);

    // Eat the separator token
    recognizer.consume();

    nextToken = recognizer.getInputStream().LA(1);
  } while (!whatFollowsLoopIterationOrRule.contains(nextToken) && nextToken != Lexer.EOF);
}
 

private Expr<?> createAdditiveExpr(final Expr<?> opsHead, final List<? extends Expr<?>> opsTail,
								   final List<? extends Token> opers) {
	checkArgument(opsTail.size() == opers.size());

	if (opsTail.isEmpty()) {
		return opsHead;
	} else {
		final Expr<?> newOpsHead = opsTail.get(0);
		final List<? extends Expr<?>> newOpsTail = opsTail.subList(1, opsTail.size());

		final Token operHead = opers.get(0);
		final List<? extends Token> opersTail = opers.subList(1, opers.size());

		final Expr<?> subExpr = createAdditiveSubExpr(opsHead, newOpsHead, operHead);

		return createAdditiveExpr(subExpr, newOpsTail, opersTail);
	}
}
 

public static boolean isClassName(TokenStream _input) {
  try {
    int i=1;
    Token token = _input.LT(i);
    while (token!=null && i < _input.size() && _input.LT(i+1).getType() == GroovyParser.DOT) {
      i = i + 2;
      token = _input.LT(i);
    }
    if(token==null)
      return false;
    // TODO here
    return Character.isUpperCase(Character.codePointAt(token.getText(), 0));
  } catch(Exception e) {
    e.printStackTrace();
  }
  
  return false;
}
 

/**
 * Finds appropriate documentation comment which belongs to the given context.
 * Overloaded version for structure field.
 *
 * @param ctx Parser context.
 *
 * @return Parsed documentation comment or null.
 */
public DocComment findDocComment(ZserioParser.StructureFieldDefinitionContext ctx)
{
    // before field alignment
    Token docToken = findDocTokenBefore(ctx.fieldAlignment());
    if (docToken != null)
        return parseDocComment(docToken);

    // before field offset
    docToken = findDocTokenBefore(ctx.fieldOffset());
    if (docToken != null)
        return parseDocComment(docToken);

    // before optional keyword
    docToken = findDocTokenBefore(ctx.OPTIONAL());
    if (docToken != null)
        return parseDocComment(docToken);

    // before field type
    docToken = findDocTokenBefore(ctx.fieldTypeId());
    if (docToken != null)
        return parseDocComment(docToken);

    return null;
}
 

private static IntervalLiteral.IntervalField getIntervalFieldType(Token token)
{
    switch (token.getType()) {
        case SqlBaseLexer.YEAR:
            return IntervalLiteral.IntervalField.YEAR;
        case SqlBaseLexer.MONTH:
            return IntervalLiteral.IntervalField.MONTH;
        case SqlBaseLexer.DAY:
            return IntervalLiteral.IntervalField.DAY;
        case SqlBaseLexer.HOUR:
            return IntervalLiteral.IntervalField.HOUR;
        case SqlBaseLexer.MINUTE:
            return IntervalLiteral.IntervalField.MINUTE;
        case SqlBaseLexer.SECOND:
            return IntervalLiteral.IntervalField.SECOND;
    }

    throw new IllegalArgumentException("Unsupported interval field: " + token.getText());
}
 

protected void reportUnwantedToken(@NotNull Parser recognizer)
{
	if (inErrorRecoveryMode(recognizer))
	{
		return;
	}

	beginErrorCondition(recognizer);

	Token t = recognizer.getCurrentToken();
	String tokenName = getTokenErrorDisplay(t);
	IntervalSet expecting = getExpectedTokens(recognizer);
	String msg = "多余输入 " + tokenName + " 期望 " + expecting.toString(recognizer.getTokenNames());
	BeetlException exception = new BeetlParserException(BeetlException.PARSER_MISS_ERROR, msg);
	//		exception.token = this.getGrammarToken(t);
	exception.pushToken(this.getGrammarToken(t));
	throw exception;
}
 

private Optional<Token> findTrailingComment(@Nullable List<Token> tokensAfter) {
    if (tokensAfter == null) {
        return Optional.empty();
    }
    Optional<Token> trailingComment = Optional.empty();
    for (Token token : tokensAfter) {
        if (token.getType() == LINE_COMMENT) {
            trailingComment = Optional.of(token);
        }
        // Try to find single trailing comment (on the same line).
        // If we hit newline, then we can stop right away.
        if (trailingComment.isPresent() || token.getType() == NL) {
            break;
        }
    }
    return trailingComment;
}
 

/** Get start/stop of an entire rule including semi and then clean up
 *  WS at end.
 */
public static String getRuleText(CommonTokenStream tokens, ParserRuleContext ruleDefNode) {
	Token stop = ruleDefNode.getStop();
	Token semi = stop;
	TerminalNode colonNode = ruleDefNode.getToken(ANTLRv4Parser.COLON, 0);
	Token colon = colonNode.getSymbol();
	Token beforeSemi = tokens.get(stop.getTokenIndex()-1);
	Token afterColon = tokens.get(colon.getTokenIndex()+1);

	// trim whitespace/comments before / after rule text
	List<Token> ignoreBefore = tokens.getHiddenTokensToRight(colon.getTokenIndex());
	List<Token> ignoreAfter = tokens.getHiddenTokensToLeft(semi.getTokenIndex());
	Token textStart = afterColon;
	Token textStop = beforeSemi;
	if ( ignoreBefore!=null ) {
		Token lastWSAfterColon = ignoreBefore.get(ignoreBefore.size()-1);
		textStart = tokens.get(lastWSAfterColon.getTokenIndex()+1);
	}
	if ( ignoreAfter!=null ) {
		int firstWSAtEndOfRule = ignoreAfter.get(0).getTokenIndex()-1;
		textStop = tokens.get(firstWSAtEndOfRule); // stop before 1st ignore token at end
	}

	return tokens.getText(textStart, textStop);
}
 

private Expr<?> createMutliplicativeExpr(final Expr<?> opsHead, final List<? extends Expr<?>> opsTail,
										 final List<? extends Token> opers) {
	checkArgument(opsTail.size() == opers.size());

	if (opsTail.isEmpty()) {
		return opsHead;
	} else {
		final Expr<?> newOpsHead = opsTail.get(0);
		final List<? extends Expr<?>> newOpsTail = opsTail.subList(1, opsTail.size());

		final Token operHead = opers.get(0);
		final List<? extends Token> opersTail = opers.subList(1, opers.size());

		final Expr<?> subExpr = createMultiplicativeSubExpr(opsHead, newOpsHead, operHead);

		return createMutliplicativeExpr(subExpr, newOpsTail, opersTail);
	}
}
 

public void error(final String kind, final TokenSource tokens, final Object offendingSymbol, final int line, final int charPositionInLine, final int length, final String msg, final Exception e) {
	hasError = true;

	final String filename = tokens.getSourceName();

	System.err.print(filename.substring(filename.lastIndexOf(File.separator) + 1) + ": compilation failed: ");
	System.err.print("Encountered " + kind + " error ");
	if (offendingSymbol != null)
		System.err.print("\"" + offendingSymbol + "\" ");
	System.err.print("at line " + line + ", ");
	if (length > 0)
		System.err.print("columns " + charPositionInLine + "-" + (charPositionInLine + length - 1));
	else
		System.err.print("column " + charPositionInLine);
	System.err.println(". " + msg);

	underlineError(tokens, (Token)offendingSymbol, line, charPositionInLine, length);

	if (e != null)
		for (final StackTraceElement st : e.getStackTrace())
			System.err.println("\tat " + st);
	else
		System.err.println("\tat unknown stack");
}
 

public ArithmeticNode createBinary(Token opToken, ExpressionNode leftNode, ExpressionNode rightNode) {
    final ArithmeticNode result;
    switch (opToken.getText()) {
        case "+":
            result = new ArithmeticNode(ArithmeticNode.Operation.ADD, leftNode, rightNode);
            break;
        case "-":
            result = new ArithmeticNode(ArithmeticNode.Operation.SUBTRACT, leftNode, rightNode);
            break;
        case "*":
            result = new ArithmeticNode(ArithmeticNode.Operation.MULTIPLY, leftNode, rightNode);
            break;
        case "/":
            result = new ArithmeticNode(ArithmeticNode.Operation.DIVIDE, leftNode, rightNode);
            break;
        case "%":
            result = new ArithmeticNode(ArithmeticNode.Operation.MODULO, leftNode, rightNode);
            break;
        default:
            // should not happen due to lexer
            throw new GrCUDAInternalException("unexpected operation: " + opToken.getText());
    }
    return result;
}
 

public ExecutableCodeDef(ParserRuleContext tree,
                         String inputFilename,
                         int index,
                         Token startOrRefToken,
                         Map<String,String> attributes,
                         String code)
{
	super(index, startOrRefToken);
	this.tree = tree;
	this.inputFilename = inputFilename;
	this.code = expandTabs(code, 4);
	this.attributes = attributes;
	if ( attributes!=null ) {
		this.label = attributes.get("label");
		if ( attributes.containsKey("hide") ) {
			isCodeVisible = !attributes.get("hide").equals("true");
		}
		if ( attributes.containsKey("output") ) {
			displayExpr = attributes.get("output");
		}
	}
}
 

/**
 * Create an error node with the text of the current line and insert it into parse tree
 *
 * @param recognizer The recognizer with which to create the error node
 * @param separator The token that ends the unrecognized link. This is also used to determine the
 *     index of the line to return in error messages.
 * @return The token contained in the error node
 */
private Token createErrorNode(Parser recognizer, ParserRuleContext ctx, Token separator) {
  if (_recoveredAtEof) {
    _recoveredAtEof = false;
    throw new BatfishRecognitionException(recognizer, recognizer.getInputStream(), ctx);
  }
  if (separator.getType() == Lexer.EOF) {
    _recoveredAtEof = true;
  }
  String lineText = _lines[separator.getLine() - 1] + separator.getText();
  Token lineToken =
      new UnrecognizedLineToken(lineText, separator.getLine(), _parserStateAtRecovery);
  ErrorNode errorNode = recognizer.createErrorNode(ctx, lineToken);
  ctx.addErrorNode(errorNode);
  return lineToken;
}
 

public static List<TerminalNode> getAllRuleRefNodes(Parser parser, ParseTree tree, String ruleName) {
	List<TerminalNode> nodes = new ArrayList<TerminalNode>();
	Collection<ParseTree> ruleRefs;
	if ( Grammar.isTokenName(ruleName) ) {
		ruleRefs = XPath.findAll(tree, "//lexerRuleBlock//TOKEN_REF", parser);
	}
	else {
		ruleRefs = XPath.findAll(tree, "//ruleBlock//RULE_REF", parser);
	}
	for (ParseTree node : ruleRefs) {
		TerminalNode terminal = (TerminalNode)node;
		Token rrefToken = terminal.getSymbol();
		String r = rrefToken.getText();
		if ( r.equals(ruleName) ) {
			nodes.add(terminal);
		}
	}
	if ( nodes.size()==0 ) return null;
	return nodes;
}
 

public int[] getFeatures(int i)	{
	CodeBuffTokenStream tokens = doc.tokens;
	TerminalNode node = tokenToNodeMap.get(tokens.get(i));
	if ( node==null ) {
		System.err.println("### No node associated with token "+tokens.get(i));
		return null;
	}

	Token curToken = node.getSymbol();
	Token prevToken = tokens.getPreviousRealToken(i);
	Token prevPrevToken = prevToken!=null ? doc.tokens.getPreviousRealToken(prevToken.getTokenIndex()) : null;

	boolean prevTokenStartsLine = false;
	if ( prevToken!=null && prevPrevToken!=null ) {
		prevTokenStartsLine = prevToken.getLine()>prevPrevToken.getLine();
	}

	boolean curTokenStartsNewLine = false;
	if ( prevToken==null ) curTokenStartsNewLine = true; // we must be at start of file
	else if ( curToken.getLine() > prevToken.getLine() ) curTokenStartsNewLine = true;

	int[] features = getContextFeatures(corpus, tokenToNodeMap, doc, i);

	setListInfoFeatures(corpus.tokenToListInfo, features, curToken);

	features[INDEX_PREV_FIRST_ON_LINE]         = prevTokenStartsLine ? 1 : 0;
	features[INDEX_FIRST_ON_LINE]              = curTokenStartsNewLine ? 1 : 0;

	return features;
}
 

@Override
public Code visitConstant(ConstantContext ctx) {
    Token token = ((TerminalNode) ctx.getChild(0)).getSymbol();
    String text = token.getText();
    switch (token.getType()) {
    case JetTemplateParser.STRING_DOUBLE:
        return new SegmentCode(String.class, text, ctx);
    case JetTemplateParser.STRING_SINGLE:
        text = StringEscapeUtils.asCanonicalJavaString(text);
        return new SegmentCode(String.class, text, ctx);
    case JetTemplateParser.INTEGER:
    case JetTemplateParser.INTEGER_HEX:
    case JetTemplateParser.FLOATING_POINT:
        Class<?> klass;
        if (text.endsWith("l") || text.endsWith("L")) {
            klass = Long.TYPE;
        } else if (text.endsWith("f") || text.endsWith("F")) {
            klass = Float.TYPE;
        } else if (text.endsWith("d") || text.endsWith("D")) {
            klass = Double.TYPE;
        } else if (token.getType() == JetTemplateParser.FLOATING_POINT) {
            klass = Double.TYPE; // 浮点数默认是double
        } else {
            klass = Integer.TYPE;
        }
        return new SegmentCode(klass, text, ctx);
    case JetTemplateParser.KEYWORD_TRUE:
        return new SegmentCode(Boolean.TYPE, text, ctx);
    case JetTemplateParser.KEYWORD_FALSE:
        return new SegmentCode(Boolean.TYPE, text, ctx);
    case JetTemplateParser.KEYWORD_NULL:
        return new SegmentCode(TypedKlass.NULL, text, ctx);
    default:
        throw reportError("Unexpected token type :" + token.getType(), ctx);
    }
}
 

private static SortItem.NullOrdering getNullOrderingType(Token token)
{
    switch (token.getType()) {
        case SqlBaseLexer.FIRST:
            return SortItem.NullOrdering.FIRST;
        case SqlBaseLexer.LAST:
            return SortItem.NullOrdering.LAST;
    }

    throw new IllegalArgumentException("Unsupported ordering: " + token.getText());
}
 

public ProductExpression(final Expression lhs, final String operator, final Expression rhs, final Token ctxGetStart, final Pass1Listener pass1Listener) {
	super("[" + lhs.getText() + " " + operator + " " + rhs.getText() + "]",
			lhs.type(), lhs.enclosingMegaType());
	lhs_ = lhs;
	rhs_ = rhs;
	lhs_.setResultIsConsumed(true);
	rhs_.setResultIsConsumed(true);
	promoteTypesAsNecessary();
	pass1Listener.binopTypeCheck(lhs_, operator, rhs_, ctxGetStart);
	operator_ = operator;
}
 

@Override protected void checkForAssignmentViolatingConstness(final AssignmentExpression assignment, final Token ctxGetStart) {
	final MethodDeclaration enclosingMethod = super.methodWithinWhichIAmDeclared(currentScope_);
	if (null != enclosingMethod && enclosingMethod.isConst()) {
		final Expression assignee = assignment.lhs();
		checkLhsForAssignmentViolatingConstness(assignee, enclosingMethod, ctxGetStart);
	}
}
 

@Override
public void syntaxError(
    Recognizer<?, ?> recognizer,
    Object symbol,
    int line,
    int column,
    String message,
    RecognitionException error) {

    syntaxErrors.add(symbol instanceof Token
        ? new SyntaxError(stream, message, (Token) symbol)
        : new SyntaxError(stream, message, line - 1, column));
}
 

public static <T extends ASTNode> T configureAST(T astNode, GroovyParser.GroovyParserRuleContext ctx, ASTNode stop) {
    Token start = ctx.getStart();

    astNode.setLineNumber(start.getLine());
    astNode.setColumnNumber(start.getCharPositionInLine() + 1);

    if (asBoolean(stop)) {
        astNode.setLastLineNumber(stop.getLastLineNumber());
        astNode.setLastColumnNumber(stop.getLastColumnNumber());
    } else {
        configureEndPosition(astNode, start);
    }

    return astNode;
}
 

/**
 * Convert word like {@code *} or {@code ID} or {@code expr} to a path
 * element. {@code anywhere} is {@code true} if {@code //} precedes the
 * word.
 */
protected XPathElement getXPathElement(Token wordToken, boolean anywhere) {
	if ( wordToken.getType()==Token.EOF ) {
		throw new IllegalArgumentException("Missing path element at end of path");
	}
	String word = wordToken.getText();
	Integer ttype = tokenTypes.get(word);
	Integer ruleIndex = ruleIndexes.get(word);
	switch ( wordToken.getType() ) {
		case XPathLexer.WILDCARD :
			return anywhere ?
				new XPathWildcardAnywhereElement() :
				new XPathWildcardElement();
		case XPathLexer.TOKEN_REF :
		case XPathLexer.STRING :
			if ( ttype==null || ttype==Token.INVALID_TYPE ) {
				throw new IllegalArgumentException(word+
												   " at index "+
												   wordToken.getStartIndex()+
												   " isn't a valid token name");
			}
			return anywhere ?
				new XPathTokenAnywhereElement(word, ttype) :
				new XPathTokenElement(word, ttype);
		default :
			if ( ruleIndex==null || ruleIndex==-1 ) {
				throw new IllegalArgumentException(word+
												   " at index "+
												   wordToken.getStartIndex()+
												   " isn't a valid rule name");
			}
			return anywhere ?
				new XPathRuleAnywhereElement(word, ruleIndex) :
				new XPathRuleElement(word, ruleIndex);
	}
}
 

@Nonnull
protected ObjectType parseClassType(KalangParser.ClassTypeContext ctx) {
    NullableKind nullable = ctx.nullable == null ? NullableKind.NONNULL : NullableKind.NULLABLE;
    KalangParser.LambdaTypeContext lambdaType = ctx.lambdaType();
    if (lambdaType != null) {
        return this.parseLambdaType(lambdaType);
    }
    Token rawTypeToken = ctx.rawClass;
    List<String> classNameParts = new LinkedList<>();
    for (Token p : ctx.paths) {
        classNameParts.add(p.getText());
    }
    classNameParts.add(rawTypeToken.getText());
    String rawType = String.join(".", classNameParts);
    for (GenericType gt : getCurrentClass().getGenericTypes()) {
        if (rawType.equals(gt.getName())) {
            return gt;
        }
    }
    ObjectType clazzType = requireClassType(rawType, rawTypeToken);
    ClassNode clazzNode = clazzType.getClassNode();
    GenericType[] clzDeclaredGenericTypes = clazzNode.getGenericTypes();
    List<KalangParser.ParameterizedElementTypeContext> parameterTypes = ctx.parameterTypes;
    if (parameterTypes != null && !parameterTypes.isEmpty()) {
        Type[] typeArguments = new Type[parameterTypes.size()];
        if (clzDeclaredGenericTypes.length != parameterTypes.size()) {
            throw new NodeException("wrong number of type arguments", offset(ctx));
        }
        for (int i = 0; i < typeArguments.length; i++) {
            typeArguments[i] = parseParameterizedElementType(parameterTypes.get(i));
            Objects.requireNonNull(typeArguments[i]);
        }
        return Types.getClassType(clazzType.getClassNode(), typeArguments, nullable);
    } else {
        return Types.getClassType(clazzType.getClassNode(), nullable);
    }
}
 

private static IntervalLiteral.Sign getIntervalSign(Token token)
{
    switch (token.getType()) {
        case SqlBaseLexer.MINUS:
            return IntervalLiteral.Sign.NEGATIVE;
        case SqlBaseLexer.PLUS:
            return IntervalLiteral.Sign.POSITIVE;
    }

    throw new IllegalArgumentException("Unsupported sign: " + token.getText());
}
 

@Override protected void lookupOrCreateStagePropDeclaration(final String stagePropName, final Token token, final boolean isStagePropArray) {
	// Return value is through currentRole_
	currentRoleOrStageProp_ = currentScope_.lookupRoleOrStagePropDeclarationRecursive(stagePropName);
	if (null == currentRoleOrStageProp_) {
		assert null != currentRoleOrStageProp_;
	}
}
 

private boolean containsDateTimeLiterals(List<Token> tokens) {
    for (int i=0; i<tokens.size()-1; i++) {
        if (isDateTimeLiteralStart(i, tokens)) {
            return true;
        }
    }
    return false;
}
 

public <T extends CodeLocation> T code(T location, ParserRuleContext context) {
    Token startToken = context.start;
    Token endToken = context.stop;
    int endTokenLength = endToken.getText().length();
    location.setStartPosition(new CodePosition(startToken.getLine(), startToken.getCharPositionInLine()));
    location.setEndPosition(new CodePosition(endToken.getLine(), endToken.getCharPositionInLine() + endTokenLength));
    return location;
}
 

private static FrameBound.Type getBoundedFrameBoundType(Token token)
{
    switch (token.getType()) {
        case SqlBaseLexer.PRECEDING:
            return FrameBound.Type.PRECEDING;
        case SqlBaseLexer.FOLLOWING:
            return FrameBound.Type.FOLLOWING;
    }

    throw new IllegalArgumentException("Unsupported bound type: " + token.getText());
}
 

public TokenNavigator(@Nonnull Token[] tokens) {
    this.tokens = tokens;
    startOffset = new int[tokens.length];
    if(tokens.length>0){
        for (int i=0;i<tokens.length;i++) {
            startOffset[i] = tokens[i].getStartIndex();
        }
        maxCaretOffset = tokens[tokens.length-1].getStopIndex();
    }
}
 

@Override protected <ExprType> Expression expressionFromReturnStatement(final ExprType ctxExpr, final RuleContext ctxParent, final Token ctxGetStart)
{
	Expression expressionReturned = null;
	if (null != ctxExpr) {
		expressionReturned = parsingData_.popExpression();
	}
	final MethodDeclaration methodDecl = (MethodDeclaration)findProperMethodScopeAround(ctxExpr, ctxParent, ctxGetStart);
	assert null == methodDecl || methodDecl instanceof MethodDeclaration;
	
	if (null == methodDecl) {
		expressionReturned = new ErrorExpression(null);
		ErrorLogger.error(ErrorIncidenceType.Fatal, ctxGetStart, "Return statement must be within a method scope ", "", "", "");
	} else {
		if (methodDecl.returnType() == null || methodDecl.returnType().name().equals("void")) {
			if (null == expressionReturned || expressionReturned.type().name().equals("void")) {
				;
			} else {
				ErrorLogger.error(ErrorIncidenceType.Fatal, ctxGetStart, "Return expression `", expressionReturned.getText(), "' of type ",
						expressionReturned.type().getText(), " is incompatible with method that returns no value.", "");
				expressionReturned = new ErrorExpression(null);
			}
		} else if (methodDecl.returnType().canBeConvertedFrom(expressionReturned.type())) {
			;
		} else {
			if (expressionReturned.isntError()) {
				ErrorLogger.error(ErrorIncidenceType.Fatal, ctxGetStart,
						"Return expression `" + expressionReturned.getText(),
						"`' of type `", expressionReturned.type().getText(),
						"' is not compatible with declared return type `",
						methodDecl.returnType().getText(), "'.");
				expressionReturned = new ErrorExpression(expressionReturned);
			}
		}
	}
	return expressionReturned;
}