org.antlr.runtime.TokenStream Java Examples

@Test
public void testAddErrorListener() throws Exception
{
    SyntaxErrorCounter firstCounter = new SyntaxErrorCounter();
    SyntaxErrorCounter secondCounter = new SyntaxErrorCounter();

    CharStream stream = new ANTLRStringStream("SELECT * FORM test;");
    CqlLexer lexer = new CqlLexer(stream);

    TokenStream tokenStream = new CommonTokenStream(lexer);
    CqlParser parser = new CqlParser(tokenStream);
    parser.addErrorListener(firstCounter);
    parser.addErrorListener(secondCounter);

    parser.query();

    // ANTLR 3.5 reports 2 errors in the sentence above (missing FROM and missing EOF).
    assertTrue(firstCounter.count > 0);
    assertTrue(secondCounter.count > 0);
}
 

/**
 * Returns true if there was an unexpected EOL.
 */
public static boolean hasDisallowedEOL(Callback callback) {
	TokenStream input = callback.getInput();
	Token lt = input.LT(1);

	// Start on the position before the current token and scan backwards off channel tokens until the previous on
	// channel token.
	for (int ix = lt.getTokenIndex() - 1; ix > 0; ix--) {
		lt = input.get(ix);
		if (lt.getChannel() == Token.DEFAULT_CHANNEL) {
			// On channel token found: stop scanning.
			break;
		} else if (isSemicolonEquivalent(lt)) {
			return true;
		}
	}
	return false;
}
 

@Test
public void testRemoveErrorListener() throws Exception
{
    SyntaxErrorCounter firstCounter = new SyntaxErrorCounter();
    SyntaxErrorCounter secondCounter = new SyntaxErrorCounter();

    CharStream stream = new ANTLRStringStream("SELECT * FORM test;");
    CqlLexer lexer = new CqlLexer(stream);

    TokenStream tokenStream = new CommonTokenStream(lexer);
    CqlParser parser = new CqlParser(tokenStream);
    parser.addErrorListener(firstCounter);
    parser.addErrorListener(secondCounter);
    parser.removeErrorListener(secondCounter);

    parser.query();

    assertTrue(firstCounter.count > 0);
    assertEquals(0, secondCounter.count);
}
 

@Override
public CommonTree process(TokenStream tokens, CommonTree tree) {
	Tree querySpec = firstChildOfType( tree, HQLParser.QUERY_SPEC );
	StringBuilder from = new StringBuilder();
	Tree node = firstChildOfType( querySpec, HQLParser.SELECT_FROM );
	if ( node != null && ( node = firstChildOfType( node, HQLParser.FROM ) ) != null ) {
		processSubtree( from, node );
	}
	StringBuilder where = new StringBuilder();
	node = firstChildOfType( querySpec, HQLParser.WHERE );
	if ( node != null && node.getChildCount() > 0 ) {
		processSubtree( where, node.getChild( 0 ) );
	}
	StringBuilder orderBy = new StringBuilder();
	node = firstChildOfType( tree, HQLParser.ORDER_BY );
	if ( node != null ) {
		processSubtree( orderBy, node );
	}
	queryRenderer.from = from.toString();
	queryRenderer.where = where.toString();
	queryRenderer.orderBy = orderBy.toString();
	return tree;
}
 

public static CommonTree compileWhereClause(String expression) throws RecognitionException {
	//lexer splits input into tokens
	ANTLRStringStream input = new ANTLRStringStream(expression);
	TokenStream tokens = new CommonTokenStream(new WhereClauseLexer(input));
	
	//parser generates abstract syntax tree
	WhereClauseParser parser = new WhereClauseParser(tokens);
    WhereClauseParser.whereclause_return ret = parser.whereclause();
	
	//acquire parse result
	CommonTree ast = (CommonTree) ret.getTree();
	if (logger.isDebugEnabled()) print(ast, 0);
	return ast;
}
 

public boolean mismatchIsMissingToken(TokenStream input,
        int[] follow) {
    if (follow == null) {
        // we have no information about the follow; we can only consume
        // a single token and hope for the best
        return false;
    }
    // TODO: implement this error recovery strategy
    return false;
}
 

public static DRLExpressions getDRLExpressions(TokenStream input, RecognizerSharedState state, ParserHelper helper, LanguageLevelOption languageLevel ) {
    switch (languageLevel) {
        case DRL5:
            return new DRL5Expressions(input, state, helper);
        case DRL6:
        case DRL6_STRICT:
            return new DRL6Expressions(input, state, helper);
    }
    throw new RuntimeException("Unknown language level");
}
 

public ParserHelper(TokenStream input,
                    RecognizerSharedState state,
                    LanguageLevelOption languageLevel) {
    this.errorMessageFactory = new DroolsParserExceptionFactory( paraphrases, languageLevel );
    this.input = input;
    this.state = state;
    this.languageLevel = languageLevel;
}
 

/** 
 *  Match current input symbol against ttype and optionally
 *  check the text of the token against text.  Attempt
 *  single token insertion or deletion error recovery.  If
 *  that fails, throw MismatchedTokenException.
 */
Token match( TokenStream input,
             int ttype,
             String text,
             int[] follow,
             DroolsEditorType etype ) throws RecognitionException {
    Token matchedSymbol = null;
    matchedSymbol = input.LT(1);
    if (input.LA(1) == ttype && (text == null || text.equals(matchedSymbol.getText()))) {
        input.consume();
        state.errorRecovery = false;
        state.failed = false;
        helper.emit(matchedSymbol,
                etype);
        return matchedSymbol;
    }
    if (state.backtracking > 0) {
        state.failed = true;
        return matchedSymbol;
    }
    matchedSymbol = recoverFromMismatchedToken(input,
            ttype,
            text,
            follow);
    helper.emit(matchedSymbol,
            etype);
    return matchedSymbol;
}
 

public boolean mismatchIsMissingToken( TokenStream input,
                                       int[] follow ) {
    if ( follow == null ) {
        // we have no information about the follow; we can only consume
        // a single token and hope for the best
        return false;
    }
    // TODO: implement this error recovery strategy
    return false;
}
 

/** 
 *  Match current input symbol against ttype and optionally
 *  check the text of the token against text.  Attempt
 *  single token insertion or deletion error recovery.  If
 *  that fails, throw MismatchedTokenException.
 */
Token match( TokenStream input,
             int ttype,
             String text,
             int[] follow,
             DroolsEditorType etype ) throws RecognitionException {
    Token matchedSymbol = null;
    matchedSymbol = input.LT(1);
    if (input.LA(1) == ttype && (text == null || text.equals(matchedSymbol.getText()))) {
        input.consume();
        state.errorRecovery = false;
        state.failed = false;
        helper.emit(matchedSymbol,
                etype);
        return matchedSymbol;
    }
    if (state.backtracking > 0) {
        state.failed = true;
        return matchedSymbol;
    }
    matchedSymbol = recoverFromMismatchedToken(input,
            ttype,
            text,
            follow);
    helper.emit(matchedSymbol,
            etype);
    return matchedSymbol;
}
 

/**
 * Protected to allow access from subtypes and same package.
 */
protected InternalHighlightingParser(TokenStream input, N4JSGrammarAccess grammarAccess,
		TokenTypeRewriter rewriter) {
	super(input, grammarAccess);
	this.rewriter = rewriter;
	this.recoverySets = computeRecoverySets();
}
 

/**
 * <p>
 * Promotes EOL which may lead to an automatically inserted semicolon. This is probably the most important method
 * for automatic semicolon insertion, as it is only possible to insert a semicolon in case of line breaks (even if
 * they are hidden in a multi-line comment!).
 * </p>
 */
public static void promoteEOL(Callback callback) {
	RecognizerSharedState state = callback.getState();
	TokenStream input = callback.getInput();
	// Don't promote EOL if there was a syntax error at EOF
	if (state.lastErrorIndex == input.size()) {
		return;
	}
	// Get current token and its type (the possibly offending token).
	Token prev = input.LT(-1);
	Token next = input.LT(1);
	int la = next.getType();
	if (la == InternalN4JSParser.Semicolon) {
		return;
	}

	// Promoting an EOL means switching it from off channel to on channel.
	// A ML_COMMENT gets promoted when it contains an EOL.
	for (int idx = prev == null ? 0 : prev.getTokenIndex() + 1, max = la == Token.EOF ? input.size()
			: next.getTokenIndex(); idx < max; idx++) {
		Token lt = input.get(idx);
		if (lt.getChannel() == Token.DEFAULT_CHANNEL) {
			// On channel token found: stop scanning (previously promoted)
			break;
		} else if (isSemicolonEquivalent(lt)) {
			// We found our EOL: promote the token to on channel, position the input on it and reset the rule
			// start.
			lt.setChannel(Token.DEFAULT_CHANNEL);
			input.seek(idx);
			break;
		}
	}
}
 

public static CommonTree compileSelectClause(String selectParam)  throws RecognitionException {
	//lexer splits input into tokens
	ANTLRStringStream input = new ANTLRStringStream(selectParam);
	TokenStream tokens = new CommonTokenStream(new WhereClauseLexer(input));
	
	//parser generates abstract syntax tree
	WhereClauseParser parser = new WhereClauseParser(tokens);
	WhereClauseParser.selectClause_return ret = parser.selectClause();
	
	//acquire parse result
	CommonTree ast = (CommonTree) ret.getTree();
	if (logger.isDebugEnabled()) print(ast, 0);
	return ast;
}
 

/**
 * A "," cannot be followed by an automatically inserted semicolon. This is in particular true in case of variable
 * statements, in which the last declaration is ended with a comma (which might easily happen in case of copying the
 * initializer from a list or object literal (cf. IDEBUG-214).
 */
private static boolean findCommaBeforeEOL(TokenStream casted, int startIndex) {
	for (int ix = startIndex - 1; ix > 0; ix--) {
		Token lt = casted.get(ix);
		if (lt.getType() == InternalN4JSParser.Comma) {
			// System.out.println("Found Comma, EOL is not valid");
			return true;
		}
		if (lt.getChannel() == Token.DEFAULT_CHANNEL) { // any other real char ends this search
			break;
		}
	}
	return false;
}
 

public UnrecognizedOption(TokenStream input, Token start, Token stop, RecognitionException e) {
    super(start);        
    this.input = input;
    this.start = start;
    this.stop = stop;
    this.e = e;
    if (start != null) {
        setName(start.getText());
    }
    setValue(new OptionValue.SwitchOnly(true));
}
 

public ExtCss3Parser(TokenStream input, NbParseTreeBuilder dbg, String mimeType) {
    super(input, dbg);        
    if(mimeType != null) {
        this.isLessSource = mimeType.equals("text/less");
        this.isScssSource = mimeType.equals("text/scss");
    }
}
 

/**
 * Construct from a token stream.
 * @param input the token stream
 * @param errorHandler handles error messages
 */
public CobolStructureParserImpl(
        final TokenStream input,
        final RecognizerErrorHandler errorHandler) {
    super(input);
    _errorHandler = errorHandler;
}
 

/**
 * Appends a query snippet to the message to help the user to understand the problem.
 *
 * @param parser the parser used to parse the query
 * @param builder the <code>StringBuilder</code> used to build the error message
 */
private void appendQuerySnippet(Parser parser, StringBuilder builder)
{
    TokenStream tokenStream = parser.getTokenStream();
    int index = tokenStream.index();
    int size = tokenStream.size();

    Token from = tokenStream.get(getSnippetFirstTokenIndex(index));
    Token to = tokenStream.get(getSnippetLastTokenIndex(index, size));
    Token offending = tokenStream.get(getOffendingTokenIndex(index, size));

    appendSnippet(builder, from, to, offending);
}
 

/**
 * Construct from a token stream and a shared state.
 * @param input the token stream
 * @param state the shared state
 * @param errorHandler handles error messages
 */
public CobolStructureParserImpl(
        final TokenStream input,
        final RecognizerSharedState state,
        final RecognizerErrorHandler errorHandler) {
    super(input, state);
    _errorHandler = errorHandler;
}
 

public GrammarRootAST(Token t, TokenStream tokenStream) {
	super(t);
	if (tokenStream == null) {
		throw new NullPointerException("tokenStream");
	}

	this.tokenStream = tokenStream;
}
 

public GrammarRootAST(int type, Token t, TokenStream tokenStream) {
	super(type, t);
	if (tokenStream == null) {
		throw new NullPointerException("tokenStream");
	}

	this.tokenStream = tokenStream;
}
 

public GrammarRootAST(int type, Token t, String text, TokenStream tokenStream) {
super(type,t,text);
if (tokenStream == null) {
	throw new NullPointerException("tokenStream");
}

this.tokenStream = tokenStream;
  }
 

private static JPA2Parser createParser(String input) {
    if (input.contains("~"))
        throw new IllegalArgumentException("Input string cannot contain \"~\"");

    CharStream cs = new AntlrNoCaseStringStream(input);
    JPA2Lexer lexer = new JPA2Lexer(cs);
    TokenStream tstream = new CommonTokenStream(lexer);
    return new JPA2Parser(tstream);
}
 

@Test
public void testTypeField() throws Exception {
    String query = "where e.model.type = :component$filter.model_type89015";
    CharStream cs = new AntlrNoCaseStringStream(query);
    JPA2Lexer lexer = new JPA2Lexer(cs);
    TokenStream tstream = new CommonTokenStream(lexer);
    JPA2Parser jpa2Parser = new JPA2Parser(tstream);
    JPA2Parser.where_clause_return aReturn = jpa2Parser.where_clause();
    Assertions.assertTrue(isValid((CommonTree) aReturn.getTree()));
}
 

private void testQuery(String query) throws RecognitionException {
    CharStream cs = new AntlrNoCaseStringStream(query);
    JPA2Lexer lexer = new JPA2Lexer(cs);
    TokenStream tstream = new CommonTokenStream(lexer);
    JPA2Parser jpa2Parser = new JPA2Parser(tstream);
    JPA2Parser.ql_statement_return aReturn = jpa2Parser.ql_statement();
    Assertions.assertTrue(isValid((CommonTree) aReturn.getTree()));
}
 

@Test
public void testUpdate() throws Exception {
    String query = "update sec$User u set u.group = :group where u.id = :userId";
    CharStream cs = new AntlrNoCaseStringStream(query);
    JPA2Lexer lexer = new JPA2Lexer(cs);
    TokenStream tstream = new CommonTokenStream(lexer);
    JPA2Parser jpa2Parser = new JPA2Parser(tstream);
    JPA2Parser.update_statement_return aReturn = jpa2Parser.update_statement();
    Assertions.assertTrue(isValid((CommonTree) aReturn.getTree()));
}
 

private static ParsePosition get_token_start_position(
		TokenStream tokenStream, int tokenIndex) {
	CommonToken token = (CommonToken) tokenStream.get(tokenIndex);
	int line = token.getLine();
	int column = token.getCharPositionInLine();
	return new ParsePosition(line, column);
}
 

public BaseInternalContentAssistParser(TokenStream input) {
	super(input);
	this.grammarElements = new ArrayList<EObject>();
	this.localTrace = new ArrayList<EObject>();
	this.followElements = new LinkedHashSetWithoutNull<FollowElement>();
	this.paramStack = new ArrayList<Integer>();
	this.grammarElementsWithParams = new ArrayList<Integer>();
}
 

/**
 * Set token stream. Called by the parser when parser is configured.
 *
 * @param input
 *          Token stream
 */
public void setTokenStream(final TokenStream input) {
  if (input != null) {
    beforeParse(input);
  }
  this.tokenStream = input;
}