Java Code Examples for org.antlr.v4.runtime.Token#getCharPositionInLine()

/** Walk upwards from node until we find a child of p at t's char position.
 *  Don't see alignment with self, t, or element *after* us.
 *  return null if there is no such ancestor p.
 */
public static Pair<ParserRuleContext,Integer> earliestAncestorWithChildStartingAtCharPos(ParserRuleContext node, Token t, int charpos) {
	ParserRuleContext p = node;
	while ( p!=null ) {
		// check all children of p to see if one of them starts at charpos
		for (int i = 0; i<p.getChildCount(); i++) {
			ParseTree child = p.getChild(i);
			Token start;
			if ( child instanceof ParserRuleContext ) {
				start = ((ParserRuleContext) child).getStart();
			}
			else { // must be token
				start = ((TerminalNode)child).getSymbol();
			}
			// check that we don't see alignment with self or element *after* us
			if ( start.getTokenIndex()<t.getTokenIndex() && start.getCharPositionInLine()==charpos ) {
				return new Pair<>(p,i);
			}
		}
		p = p.getParent();
	}
	return null;
}
 

public SyntaxError(GyroCharStream stream, String message, Token token) {
    this.stream = stream;
    this.message = message;
    this.line = token.getLine() - 1;
    this.startColumn = token.getCharPositionInLine();

    int start = token.getStartIndex();
    int stop = token.getStopIndex();

    if (start >= 0 && stop >= 0 && stop > start) {
        this.stopColumn = this.startColumn + stop - start;

    } else {
        this.stopColumn = this.startColumn;
    }
}
 

public static int getInjectWSCategory(CodeBuffTokenStream tokens, int i) {
	int precedingNL = getPrecedingNL(tokens, i); // how many lines to inject

	Token curToken = tokens.get(i);
	Token prevToken = tokens.getPreviousRealToken(i);

	int ws = 0;
	if ( precedingNL==0 ) {
		ws = curToken.getCharPositionInLine() -
			(prevToken.getCharPositionInLine()+prevToken.getText().length());
	}

	int injectNL_WS = CAT_NO_WS;
	if ( precedingNL>0 ) {
		injectNL_WS = nlcat(precedingNL);
	}
	else if ( ws>0 ) {
		injectNL_WS = wscat(ws);
	}

	return injectNL_WS;
}
 

/**
 * Constructor from grammar token.
 *
 * @param token Grammar token to localize AST node in the sources.
 */
public AstLocation(Token token)
{
    if (token == null)
    {
        fileName = null;
        line = 0;
        column = 0;
    }
    else
    {
        fileName = token.getInputStream().getSourceName();
        line = token.getLine();
        column = token.getCharPositionInLine() + 1;
    }
}
 

@Override
public void exitBackQuotedIdentifier(SqlBaseParser.BackQuotedIdentifierContext context) {
    Token token = context.BACKQUOTED_IDENTIFIER().getSymbol();
    throw new ParsingException(
        "backquoted identifiers are not supported; use double quotes to quote identifiers",
        null,
        token.getLine(),
        token.getCharPositionInLine());
}
 

private Location toLocation(Scope scope, ParseTree node) {
    Token start;
    if (node instanceof ParserRuleContext) {
        start = ((ParserRuleContext) node).start;
    } else if (node instanceof TerminalNode) {
        start = ((TerminalNode) node).getSymbol();
    } else {
        throw new ProgramCompileException("Location is not available for type " + node.getClass());
    }
    Location location = new Location(scope != null ? scope.programName : "<string>", start.getLine(), start.getCharPositionInLine());
    return location;
}
 

@Override
public void exitDigitIdentifier(SqlBaseParser.DigitIdentifierContext context)
{
    Token token = context.DIGIT_IDENTIFIER().getSymbol();
    throw new ParsingException(
            "identifiers must not start with a digit; surround the identifier with double quotes",
            null,
            token.getLine(),
            token.getCharPositionInLine());
}
 

@Override
public void exitBackQuotedIdentifier(SqlBaseParser.BackQuotedIdentifierContext context)
{
    Token token = context.BACKQUOTED_IDENTIFIER().getSymbol();
    throw new ParsingException(
            "backquoted identifiers are not supported; use double quotes to quote identifiers",
            null,
            token.getLine(),
            token.getCharPositionInLine());
}
 

private AstLocation getLocation(Token token)
{
    final String fileName = docCommentToken.getInputStream().getSourceName();
    final int line = docCommentToken.getLine() + token.getLine() - 1; // lines are numbered from 1!
    final int charPositionInLine = token.getLine() == 1
            ? docCommentToken.getCharPositionInLine() + token.getCharPositionInLine()
            : token.getCharPositionInLine();

    return new AstLocation(fileName, line, charPositionInLine);
}
 

private ParseException toParseException(final RecognitionException e) {
    final Token token = e.getOffendingToken();

    if (token.getType() == HeroicQueryLexer.UnterminatedQutoedString) {
        return new ParseException(String.format("unterminated string: %s", token.getText()),
            null, token.getLine(), token.getCharPositionInLine());
    }

    return new ParseException("unexpected token: " + token.getText(), null, token.getLine(),
        token.getCharPositionInLine());
}
 

/**
 * Ensures the ANTLR parser will throw an exception after the first error
 *
 * @param recognizer the parser being used
 * @return no actual return value
 * @throws RecognitionException not used as a ParseException wrapped in a RuntimeException is thrown instead
 */
@Override
public Token recoverInline(Parser recognizer) throws RecognitionException {
  Token token = recognizer.getCurrentToken();
  String message = "unexpected token " + getTokenErrorDisplay(token) +
      " on line (" + token.getLine() + ") position (" + token.getCharPositionInLine() + ")" +
      " was expecting one of " + recognizer.getExpectedTokens().toString(recognizer.getVocabulary());
  ParseException parseException = new ParseException(message, token.getStartIndex());
  throw new RuntimeException(parseException);
}
 

private void assertIn(Token to, Term<?> t1, Term<?> t2) {
    Type t = t2.type();
    if (!(t instanceof SetType)) {
        throw new SpecException(filename, to.getCharPositionInLine(), "The right-hand side must be a collection. Got '" + t2.type() + "'");
    }
    SetType st = (SetType) t;
    if (!st.enclosingType().equals(t1.type())) {
        throw new SpecException(filename, to.getCharPositionInLine(),
                "Type mismatch. Expected '" + st.enclosingType() + "' for left-hand side. Got '" + t1.type() + "'");

    }
}
 

@Override
public void exitQuotedIdentifier(SqlBaseParser.QuotedIdentifierContext context)
{
    Token token = context.QUOTED_IDENTIFIER().getSymbol();
    if (token.getText().length() == 2) { // empty identifier
        throw new ParsingException("Zero-length delimited identifier not allowed", null, token.getLine(), token.getCharPositionInLine());
    }
}
 

TomlPosition(ParserRuleContext ctx, int offset) {
  Token token = ctx.getStart();
  this.line = token.getLine();
  this.column = token.getCharPositionInLine() + 1 + offset;
}
 

private static NodeLocation getLocation(Token token) {
    requireNonNull(token, "token is null");
    return new NodeLocation(token.getLine(), token.getCharPositionInLine());
}
 

private static NodeLocation getLocation(Token token) {
  requireNonNull(token, "token is null");
  return new NodeLocation(token.getLine(), token.getCharPositionInLine());
}
 

TomlPosition(ParserRuleContext ctx, int offset) {
  Token token = ctx.getStart();
  this.line = token.getLine();
  this.column = token.getCharPositionInLine() + 1 + offset;
}
 

public void processToken(int indexIntoRealTokens, int tokenIndexInStream, boolean collectAnalysis) {
	CommonToken curToken = (CommonToken)testDoc.tokens.get(tokenIndexInStream);
	String tokText = curToken.getText();
	TerminalNode node = tokenToNodeMap.get(curToken);

	int[] features = getFeatures(testDoc, tokenIndexInStream);
	int[] featuresForAlign = new int[features.length];
	System.arraycopy(features, 0, featuresForAlign, 0, features.length);

	int injectNL_WS = wsClassifier.classify(k, features, Trainer.MAX_WS_CONTEXT_DIFF_THRESHOLD);

	injectNL_WS = emitCommentsToTheLeft(tokenIndexInStream, injectNL_WS);

	int newlines = 0;
	int ws = 0;
	if ( (injectNL_WS&0xFF)==CAT_INJECT_NL ) {
		newlines = Trainer.unnlcat(injectNL_WS);
	}
	else if ( (injectNL_WS&0xFF)==CAT_INJECT_WS ) {
		ws = Trainer.unwscat(injectNL_WS);
	}

	if ( newlines==0 && ws==0 && cannotJoin(realTokens.get(indexIntoRealTokens-1), curToken) ) { // failsafe!
		ws = 1;
	}

	int alignOrIndent = CAT_ALIGN;

	if ( newlines>0 ) {
		output.append(Tool.newlines(newlines));
		line+=newlines;
		charPosInLine = 0;

		// getFeatures() doesn't know what line curToken is on. If \n, we need to find exemplars that start a line
		featuresForAlign[INDEX_FIRST_ON_LINE] = 1; // use \n prediction to match exemplars for alignment

		alignOrIndent = hposClassifier.classify(k, featuresForAlign, MAX_ALIGN_CONTEXT_DIFF_THRESHOLD);

		if ( (alignOrIndent&0xFF)==CAT_ALIGN_WITH_ANCESTOR_CHILD ) {
			align(alignOrIndent, node);
		}
		else if ( (alignOrIndent&0xFF)==CAT_INDENT_FROM_ANCESTOR_CHILD ) {
			indent(alignOrIndent, node);
		}
		else if ( (alignOrIndent&0xFF)==CAT_ALIGN ) {
			List<Token> tokensOnPreviousLine = getTokensOnPreviousLine(testDoc.tokens, tokenIndexInStream, line);
			if ( tokensOnPreviousLine.size()>0 ) {
				Token firstTokenOnPrevLine = tokensOnPreviousLine.get(0);
				int indentCol = firstTokenOnPrevLine.getCharPositionInLine();
				charPosInLine = indentCol;
				output.append(Tool.spaces(indentCol));
			}
		}
		else if ( (alignOrIndent&0xFF)==CAT_INDENT ) {
			indent(alignOrIndent, node);
		}
	}
	else {
		// inject whitespace instead of \n?
		output.append(Tool.spaces(ws));
		charPosInLine += ws;
	}

	// update Token object with position information now that we are about
	// to emit it.
	curToken.setLine(line);
	curToken.setCharPositionInLine(charPosInLine);

	TokenPositionAnalysis tokenPositionAnalysis =
		getTokenAnalysis(features, featuresForAlign, tokenIndexInStream, injectNL_WS, alignOrIndent, collectAnalysis);

	analysis.set(tokenIndexInStream, tokenPositionAnalysis);

	int n = tokText.length();
	tokenPositionAnalysis.charIndexStart = output.length();
	tokenPositionAnalysis.charIndexStop = tokenPositionAnalysis.charIndexStart + n - 1;

	// emit
	output.append(tokText);
	charPosInLine += n;
}
 

public static NodeLocation getLocation(Token token)
{
    requireNonNull(token, "token is null");
    return new NodeLocation(token.getLine(), token.getCharPositionInLine());
}
 

/** Get the token type and tree ancestor features. These are computed
 *  the same for both training and formatting.
 */
public static int[] getContextFeatures(Corpus corpus,
                                       Map<Token, TerminalNode> tokenToNodeMap,
                                       InputDocument doc,
                                       int i)
{
	int[] features = new int[NUM_FEATURES];
	CodeBuffTokenStream tokens = doc.tokens;
	TerminalNode node = tokenToNodeMap.get(tokens.get(i));
	if ( node==null ) {
		System.err.println("### No node associated with token "+tokens.get(i));
		return features;
	}
	Token curToken = node.getSymbol();

	// Get context information for previous token
	Token prevToken = tokens.getPreviousRealToken(i);
	TerminalNode prevNode = tokenToNodeMap.get(prevToken);

	ParserRuleContext prevEarliestRightAncestor = earliestAncestorEndingWithToken(prevNode);
	int prevEarliestAncestorRuleIndex = prevEarliestRightAncestor.getRuleIndex();
	int prevEarliestAncestorRuleAltNum = prevEarliestRightAncestor.getAltNumber();

	// Get context information for current token
	ParserRuleContext earliestLeftAncestor = earliestAncestorStartingWithToken(node);

	ParserRuleContext earliestLeftAncestorParent  =
		earliestLeftAncestor!=null ? earliestLeftAncestor.getParent() : null;

	ParserRuleContext earliestLeftAncestorParent2 =
		earliestLeftAncestorParent!=null ? earliestLeftAncestorParent.getParent() : null;

	ParserRuleContext earliestLeftAncestorParent3 =
		earliestLeftAncestorParent2!=null ? earliestLeftAncestorParent2.getParent() : null;

	ParserRuleContext earliestLeftAncestorParent4 =
		earliestLeftAncestorParent3!=null ? earliestLeftAncestorParent3.getParent() : null;

	ParserRuleContext earliestLeftAncestorParent5 =
		earliestLeftAncestorParent4!=null ? earliestLeftAncestorParent4.getParent() : null;

	features[INDEX_PREV_TYPE]                     = prevToken.getType();
	features[INDEX_PREV_EARLIEST_RIGHT_ANCESTOR]  = rulealt(prevEarliestAncestorRuleIndex,prevEarliestAncestorRuleAltNum);
	features[INDEX_CUR_TOKEN_TYPE]                = curToken.getType();
	features[INDEX_CUR_TOKEN_CHILD_INDEX]         = getChildIndexOrListMembership(node);
	features[INDEX_EARLIEST_LEFT_ANCESTOR]        = rulealt(earliestLeftAncestor);
	features[INDEX_ANCESTORS_CHILD_INDEX]         = getChildIndexOrListMembership(earliestLeftAncestor);
	features[INDEX_ANCESTORS_PARENT_RULE]         = earliestLeftAncestorParent!=null ? rulealt(earliestLeftAncestorParent) : -1;
	features[INDEX_ANCESTORS_PARENT_CHILD_INDEX]  = getChildIndexOrListMembership(earliestLeftAncestorParent);
	features[INDEX_ANCESTORS_PARENT2_RULE]        = earliestLeftAncestorParent2!=null ? rulealt(earliestLeftAncestorParent2) : -1;
	features[INDEX_ANCESTORS_PARENT2_CHILD_INDEX] = getChildIndexOrListMembership(earliestLeftAncestorParent2);
	features[INDEX_ANCESTORS_PARENT3_RULE]        = earliestLeftAncestorParent3!=null ? rulealt(earliestLeftAncestorParent3) : -1;
	features[INDEX_ANCESTORS_PARENT3_CHILD_INDEX] = getChildIndexOrListMembership(earliestLeftAncestorParent3);
	features[INDEX_ANCESTORS_PARENT4_RULE]        = earliestLeftAncestorParent4!=null ? rulealt(earliestLeftAncestorParent4) : -1;
	features[INDEX_ANCESTORS_PARENT4_CHILD_INDEX] = getChildIndexOrListMembership(earliestLeftAncestorParent4);
	features[INDEX_ANCESTORS_PARENT5_RULE]        = earliestLeftAncestorParent5!=null ? rulealt(earliestLeftAncestorParent5) : -1;
	features[INDEX_ANCESTORS_PARENT5_CHILD_INDEX] = getChildIndexOrListMembership(earliestLeftAncestorParent5);

	features[INDEX_MATCHING_TOKEN_STARTS_LINE] = getMatchingSymbolStartsLine(corpus, doc, node);
	features[INDEX_MATCHING_TOKEN_ENDS_LINE]   = getMatchingSymbolEndsLine(corpus, doc, node);

	features[INDEX_INFO_FILE]    = 0; // dummy; _toString() dumps filename w/o this value; placeholder for col in printout
	features[INDEX_INFO_LINE]    = curToken.getLine();
	features[INDEX_INFO_CHARPOS] = curToken.getCharPositionInLine();

	return features;
}