Java Code Examples for org.antlr.v4.runtime.CommonTokenStream#fill()

The following examples show how to use org.antlr.v4.runtime.CommonTokenStream#fill() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: PluginHelper.java    From sonar-tsql-plugin with GNU General Public License v3.0 6 votes vote down vote up 
public static AntlrContext createRequestFromStream(final InputFile file, final Charset encoding,
		final CharStream mainStream, InputStream fileInputStream) {
	final SourceLinesProvider linesProvider = new SourceLinesProvider();
	final CharStream charStream = new CaseChangingCharStream(mainStream, true);
	final TSqlLexer lexer = new TSqlLexer(charStream);

	lexer.removeErrorListeners();

	final CommonTokenStream stream = new CommonTokenStream(lexer);

	stream.fill();
	final TSqlParser parser = new TSqlParser(stream);
	parser.removeErrorListeners();
	final ParseTree root = parser.tsql_file();
	final AntlrContext antrlFile = new AntlrContext(file, stream, root,
			linesProvider.getLines(fileInputStream, encoding));
	return antrlFile;
}
Example 2
Source File: DefaultLinesProviderTest.java    From sonar-tsql-plugin with GNU General Public License v3.0 6 votes vote down vote up 
@Test
public void test() {

	final CharStream charStream = CharStreams.fromString("\r\nSELECT\r\n 1");

	final TSqlLexer lexer = new TSqlLexer(charStream);

	final CommonTokenStream stream = new CommonTokenStream(lexer);

	stream.fill();
	TSqlParser parser = new TSqlParser(stream);
	ParseTree child = parser.tsql_file().getChild(0);
	DefaultLinesProvider lines = new DefaultLinesProvider(stream);
	int line = lines.getLine(new ParsedNode(child));
	Assert.assertEquals(2, line);

}
Example 3
Source File: EagleQueryParser.java    From Eagle with Apache License 2.0 6 votes vote down vote up 
public ORExpression parse() throws EagleQueryParseException{
	try{
		EagleFilterLexer lexer = new EagleFilterLexer(new ANTLRInputStream(_query));
		CommonTokenStream tokens = new CommonTokenStream(lexer);
		tokens.fill();
		EagleFilterParser p = new EagleFilterParser(tokens);
		p.setErrorHandler(new EagleANTLRErrorStrategy());
		p.setBuildParseTree(true);
		EagleQueryFilterListenerImpl listener = new EagleQueryFilterListenerImpl(); 
		p.addParseListener(listener);
		EagleFilterParser.FilterContext fc = p.filter();
		if(fc.exception != null){
			LOG.error("Can not successfully parse the query:" + _query, fc.exception);
			throw fc.exception;
		}
		return listener.result();
	}catch(Exception ex){
		LOG.error("Can not successfully parse the query:", ex);
		throw new EagleQueryParseException("can not successfully parse the query:" + _query);
	}
}
Example 4
Source File: EagleQueryParser.java    From eagle with Apache License 2.0 6 votes vote down vote up 
public ORExpression parse() throws EagleQueryParseException {
    try {
        EagleFilterLexer lexer = new EagleFilterLexer(new ANTLRInputStream(query));
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        tokens.fill();
        EagleFilterParser p = new EagleFilterParser(tokens);
        p.setErrorHandler(new EagleANTLRErrorStrategy());
        p.setBuildParseTree(true);
        EagleQueryFilterListenerImpl listener = new EagleQueryFilterListenerImpl();
        p.addParseListener(listener);
        EagleFilterParser.FilterContext fc = p.filter();
        if (fc.exception != null) {
            LOG.error("Can not successfully parse the query:" + query, fc.exception);
            throw fc.exception;
        }
        return listener.result();
    } catch (Exception ex) {
        LOG.error("Can not successfully parse the query:", ex);
        throw new EagleQueryParseException("can not successfully parse the query:" + query);
    }
}
Example 5
Source File: AntlrContext.java    From sonar-tsql-plugin with GNU General Public License v3.0 5 votes vote down vote up 
public static AntlrContext fromStreams(InputFile inputFile, InputStream file, InputStream linesStream,
		Charset charset) throws IOException {
	final SourceLinesProvider linesProvider = new SourceLinesProvider();
	final CharStream charStream = new CaseChangingCharStream(CharStreams.fromStream(file, charset), true);
	final TSqlLexer lexer = new TSqlLexer(charStream);
	lexer.removeErrorListeners();
	final CommonTokenStream stream = new CommonTokenStream(lexer);
	stream.fill();
	final TSqlParser parser = new TSqlParser(stream);
	parser.removeErrorListeners();
	final ParseTree root = parser.tsql_file();
	final SourceLine[] lines = linesProvider.getLines(linesStream, charset);
	return new AntlrContext(inputFile, stream, root, lines);
}
Example 6
Source File: CodeTokenizer.java    From scava with Eclipse Public License 2.0 4 votes vote down vote up 
private static List<Token> tokenize(Lexer lexer)
{
	CommonTokenStream tokens = new CommonTokenStream(lexer);
	tokens.fill();
	return tokens.getTokens();
}
Example 7
Source File: CodeTokenizer.java    From scava with Eclipse Public License 2.0 4 votes vote down vote up 
private static List<Token> tokenize(Lexer lexer)
{
	CommonTokenStream tokens = new CommonTokenStream(lexer);
	tokens.fill();
	return tokens.getTokens();
}
Example 8
Source File: TestRig.java    From codebuff with BSD 2-Clause "Simplified" License 4 votes vote down vote up 
protected void process(Lexer lexer, Class<? extends Parser> parserClass, Parser parser, InputStream is, Reader r) throws IOException, IllegalAccessException, InvocationTargetException, PrintException {
	try {
		ANTLRInputStream input = new ANTLRInputStream(r);
		lexer.setInputStream(input);
		CommonTokenStream tokens = new CommonTokenStream(lexer);

		tokens.fill();

		if ( showTokens ) {
			for (Object tok : tokens.getTokens()) {
				System.out.println(tok);
			}
		}

		if ( startRuleName.equals(LEXER_START_RULE_NAME) ) return;

		if ( diagnostics ) {
			parser.addErrorListener(new DiagnosticErrorListener());
			parser.getInterpreter().setPredictionMode(PredictionMode.LL_EXACT_AMBIG_DETECTION);
		}

		if ( printTree || gui || psFile!=null ) {
			parser.setBuildParseTree(true);
		}

		if ( SLL ) { // overrides diagnostics
			parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
		}

		parser.setTokenStream(tokens);
		parser.setTrace(trace);

		try {
			Method startRule = parserClass.getMethod(startRuleName);
			ParserRuleContext tree = (ParserRuleContext)startRule.invoke(parser, (Object[])null);

			if ( printTree ) {
				System.out.println(tree.toStringTree(parser));
			}
			if ( gui ) {
				Trees.inspect(tree, parser);
			}
			if ( psFile!=null ) {
				Trees.save(tree, parser, psFile); // Generate postscript
			}
		}
		catch (NoSuchMethodException nsme) {
			System.err.println("No method for rule "+startRuleName+" or it has arguments");
		}
	}
	finally {
		if ( r!=null ) r.close();
		if ( is!=null ) is.close();
	}
}