Java Code Examples for org.antlr.v4.runtime.CharStream

The following examples show how to use org.antlr.v4.runtime.CharStream. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: openCypher   Source File: BNFProcessor.java    License: Apache License 2.0 6 votes vote down vote up
private Grammar processAntrlStream(CharStream inStream)
    {
        BNFLexer lexer = new BNFLexer(inStream);
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        BNFParser parser = new BNFParser(tokens);
        // leaving the old listeners in gives a nice error messsage
        // parser.removeErrorListeners();
        // lexer.removeErrorListeners();
        lexer.addErrorListener(new FailingErrorListener());
        parser.addErrorListener(new FailingErrorListener());

        ParseTree tree = parser.rulelist();

        ParseTreeWalker walker = new ParseTreeWalker();
        BNFListener listener = new BNFListener(tokens);
        walker.walk(listener, tree);

        GrammarTop itemTree = listener.getTreeTop();
//        LOGGER.warn("bnf gave {}", itemTree.getStructure(""));
        // convert to openCypher grammar
        GrammarConverter converter = new GrammarConverter(itemTree);
        return converter.convert();
    }
 
Example 2
Source Project: protostuff-compiler   Source File: EnumParseListenerTest.java    License: Apache License 2.0 6 votes vote down vote up
private Enum parseEnumBlock(String input) {
    CharStream stream = CharStreams.fromString(input);
    ProtoLexer lexer = new ProtoLexer(stream);
    lexer.removeErrorListeners();
    lexer.addErrorListener(TestUtils.ERROR_LISTENER);
    CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    ProtoParser parser = new ProtoParser(tokenStream);
    parser.setErrorHandler(new BailErrorStrategy());
    parser.removeErrorListeners();
    parser.addErrorListener(TestUtils.ERROR_LISTENER);
    ProtoContext context = new ProtoContext("test.proto");
    Proto proto = new Proto();
    context.push(proto);
    EnumParseListener enumParseListener = new EnumParseListener(tokenStream, context);
    OptionParseListener optionParseListener = new OptionParseListener(tokenStream, context);
    parser.addParseListener(enumParseListener);
    parser.addParseListener(optionParseListener);
    parser.enumBlock();
    return proto.getEnums().get(0);
}
 
Example 3
@Test
public void invalidStringLiteral()
{
    final CharStream input = CharStreams.fromString(
            " \"invalid unicode escape character \\uBAD\" " +
            " \"invalid octal escape character \\09\" " +
            " \"invalid hexadecimal escape character \\xA\" "
    );

    ZserioLexer lexer = new ZserioLexer(input);

    checkToken(lexer, ZserioLexer.INVALID_STRING_LITERAL, "\"invalid unicode escape character \\uBAD\"");
    checkToken(lexer, ZserioLexer.INVALID_STRING_LITERAL, "\"invalid octal escape character \\09\"");
    checkToken(lexer, ZserioLexer.INVALID_STRING_LITERAL, "\"invalid hexadecimal escape character \\xA\"");

    checkEOF(lexer);
}
 
Example 4
Source Project: depends   Source File: JavaFileParser.java    License: MIT License 6 votes vote down vote up
@Override
public void parse() throws IOException {
       CharStream input = CharStreams.fromFileName(fileFullPath);
       Lexer lexer = new JavaLexer(input);
       lexer.setInterpreter(new LexerATNSimulator(lexer, lexer.getATN(), lexer.getInterpreter().decisionToDFA, new PredictionContextCache()));
       CommonTokenStream tokens = new CommonTokenStream(lexer);
       JavaParser parser = new JavaParser(tokens);
       ParserATNSimulator interpreter = new ParserATNSimulator(parser, parser.getATN(), parser.getInterpreter().decisionToDFA, new PredictionContextCache());
       parser.setInterpreter(interpreter);
       JavaListener bridge = new JavaListener(fileFullPath, entityRepo,inferer);
    ParseTreeWalker walker = new ParseTreeWalker();
    try {
    	walker.walk(bridge, parser.compilationUnit());
		Entity fileEntity = entityRepo.getEntity(fileFullPath);
		((FileEntity)fileEntity).cacheAllExpressions();
		interpreter.clearDFA();
		bridge.done();
    	
    }catch (Exception e) {
    	System.err.println("error encountered during parse..." );
    	e.printStackTrace();
    }
    
   }
 
Example 5
Source Project: elepy   Source File: Queries.java    License: Apache License 2.0 6 votes vote down vote up
public static Query parse(String input) {

        if (StringUtils.isEmpty(input)) {
            return Queries.empty();
        }
        //Lex (with Antlr's generated lexer)
        CharStream inputStream = CharStreams.fromString(input);
        EleQueryLexer lexer = new EleQueryLexer(inputStream);
        CommonTokenStream tokens = new CommonTokenStream(lexer);

        //Parse (with Antlr's generated parser)
        EleQueryParser parser = new EleQueryParser(tokens);
        ParseTree parseTree = parser.query();

        //Extract AST from the Antlr parse tree
        QueryListener listener = new QueryListener();
        ParseTreeWalker walker = new ParseTreeWalker();
        walker.walk(listener, parseTree);

        return listener.getQuery();
    }
 
Example 6
Source Project: Concurnas   Source File: Utils.java    License: MIT License 6 votes vote down vote up
public static ConcurnasParser miniParse(String text, String name, int startingLine, int startingColumn, LexParseErrorCapturer errors) {
	CharStream input = CharStreams.fromString(text, name);
	
	ConcurnasLexer lexer = new ConcurnasLexer(input);
	if(startingLine > -1) {
		lexer.setTokenFactory(new LocationRepointCommonTokenFactory(startingLine,startingColumn));
	}
	
	lexer.permitDollarPrefixRefName = true;
	CommonTokenStream tokens = new CommonTokenStream(lexer);
	ConcurnasParser parser = new ConcurnasParser(tokens);
	
	parser.getInterpreter().setPredictionMode(PredictionMode.SLL); 
	
	parser.removeErrorListeners(); // remove ConsoleErrorListener
	parser.addErrorListener(errors); // add ours
	
	lexer.removeErrorListeners(); // remove ConsoleErrorListener
	lexer.addErrorListener(errors); // add ours
	
	return parser;
}
 
Example 7
@Test
public void octalLiteral()
{
    final CharStream input = CharStreams.fromString(
            "0101 0770 012 00 01234567"
    );

    ZserioLexer lexer = new ZserioLexer(input);

    checkToken(lexer, ZserioLexer.OCTAL_LITERAL, "0101");
    checkToken(lexer, ZserioLexer.OCTAL_LITERAL, "0770");
    checkToken(lexer, ZserioLexer.OCTAL_LITERAL, "012");
    checkToken(lexer, ZserioLexer.OCTAL_LITERAL, "00");
    checkToken(lexer, ZserioLexer.OCTAL_LITERAL, "01234567");

    checkEOF(lexer);
}
 
Example 8
Source Project: bookish   Source File: Tool.java    License: MIT License 6 votes vote down vote up
/** Parse a file and also traverse parse tree to set XML attributes Map
	 *  for any attrs rule refs.
	 */
	public Object parse(Artifact artifact, String fileName, boolean isRoot) throws IOException {
		CharStream input = CharStreams.fromFileName(fileName);
		BookishLexer lexer = new BookishLexer(input);
		CommonTokenStream tokens = new CommonTokenStream(lexer);
		BookishParser parser = new BookishParser(tokens);

		lexer.removeErrorListeners();
		lexer.addErrorListener(getErrorListener());
		parser.removeErrorListeners();
		parser.addErrorListener(getErrorListener());

		ParserRuleContext doctree;
		if ( isRoot ) doctree = parser.rootdocument();
		else doctree = parser.chapter();

		verifyAndGatherXMLAttributes(parser, doctree);

//		System.out.println();
//		System.out.println(doctree.toStringTree(Arrays.asList(BookishParser.ruleNames)));

		if ( isRoot ) {
			return new RootDocInfo(artifact, parser, doctree);
		}
		return new ChapDocInfo(artifact, parser, (BookishParser.ChapterContext)doctree);
	}
 
Example 9
@Test
public void floatOrDoubleLiteral()
{
    final CharStream input = CharStreams.fromString(
            "0e1 0e-1 1e2f 9. 9.12 9.f 9.1f 9.e2 9.e2f 9.e-2f .0 .0e-1 .0f .1e3f"
    );

    ZserioLexer lexer = new ZserioLexer(input);

    checkToken(lexer, ZserioLexer.DOUBLE_LITERAL, "0e1");
    checkToken(lexer, ZserioLexer.DOUBLE_LITERAL, "0e-1");
    checkToken(lexer, ZserioLexer.FLOAT_LITERAL, "1e2f");
    checkToken(lexer, ZserioLexer.DOUBLE_LITERAL, "9.");
    checkToken(lexer, ZserioLexer.DOUBLE_LITERAL, "9.12");
    checkToken(lexer, ZserioLexer.FLOAT_LITERAL, "9.f");
    checkToken(lexer, ZserioLexer.FLOAT_LITERAL, "9.1f");
    checkToken(lexer, ZserioLexer.DOUBLE_LITERAL, "9.e2");
    checkToken(lexer, ZserioLexer.FLOAT_LITERAL, "9.e2f");
    checkToken(lexer, ZserioLexer.FLOAT_LITERAL, "9.e-2f");
    checkToken(lexer, ZserioLexer.DOUBLE_LITERAL, ".0");
    checkToken(lexer, ZserioLexer.DOUBLE_LITERAL, ".0e-1");
    checkToken(lexer, ZserioLexer.FLOAT_LITERAL, ".0f");
    checkToken(lexer, ZserioLexer.FLOAT_LITERAL, ".1e3f");

    checkEOF(lexer);
}
 
Example 10
private Message parseMessage(String input) {
    CharStream stream = CharStreams.fromString(input);
    ProtoLexer lexer = new ProtoLexer(stream);
    lexer.removeErrorListeners();
    lexer.addErrorListener(TestUtils.ERROR_LISTENER);
    CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    ProtoParser parser = new ProtoParser(tokenStream);
    parser.removeErrorListeners();
    parser.addErrorListener(TestUtils.ERROR_LISTENER);
    ProtoContext context = new ProtoContext("test.proto");
    MessageParseListener messageParseListener = new MessageParseListener(tokenStream, context);
    OptionParseListener optionParseListener = new OptionParseListener(tokenStream, context);
    Proto proto = new Proto();
    context.push(proto);
    parser.addParseListener(messageParseListener);
    parser.addParseListener(optionParseListener);
    parser.messageBlock();
    return proto.getMessages().get(0);
}
 
Example 11
Source Project: openCypher   Source File: G4Processor.java    License: Apache License 2.0 6 votes vote down vote up
private Grammar processAntrlStream(CharStream inStream)
    {
        Gee4Lexer lexer = new Gee4Lexer(inStream);
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        Gee4Parser parser = new Gee4Parser(tokens);
        // leaving the old listeners in gives a nice error messsage
        // parser.removeErrorListeners();
        // lexer.removeErrorListeners();
        lexer.addErrorListener(new FailingErrorListener());
        parser.addErrorListener(new FailingErrorListener());

        ParseTree tree = parser.wholegrammar();

        ParseTreeWalker walker = new ParseTreeWalker();
        G4Listener listener = new G4Listener(tokens);
        walker.walk(listener, tree);
        
        GrammarTop itemTree = listener.getTreeTop();
//        LOGGER.warn("bnf gave {}", itemTree.getStructure(""));
        // convert to openCypher grammar
        GrammarConverter converter = new GrammarConverter(itemTree);
        return converter.convert();
    }
 
Example 12
Source Project: citeproc-java   Source File: PageParser.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Parses a given page or range of pages. If the given string cannot
 * be parsed, the method will return a page range with a literal string.
 * @param pages the page or range of pages
 * @return the parsed page or page range (never {@code null})
 */
public static PageRange parse(String pages) {
    CharStream cs = CharStreams.fromString(pages);
    InternalPageLexer lexer = new InternalPageLexer(cs);
    lexer.removeErrorListeners(); // do not output errors to console
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    InternalPageParser parser = new InternalPageParser(tokens);
    parser.removeErrorListeners(); // do not output errors to console
    PagesContext ctx = parser.pages();
    if (ctx.literal == null || ctx.literal.isEmpty() ||
            ctx.exception != null || parser.getNumberOfSyntaxErrors() > 0) {
        // unparsable fall back to literal string
        return new PageRange(pages, null, null, false);
    }
    return new PageRange(ctx.literal, ctx.pageFrom, ctx.numberOfPages,
            ctx.multiplePages);
}
 
Example 13
Source Project: ignite   Source File: XGModelParser.java    License: Apache License 2.0 6 votes vote down vote up
/** {@inheritDoc} */
@Override public XGModelComposition parse(byte[] mdl) {
    try (ByteArrayInputStream bais = new ByteArrayInputStream(mdl)) {
        CharStream cStream = CharStreams.fromStream(bais);
        XGBoostModelLexer lexer = new XGBoostModelLexer(cStream);
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        XGBoostModelParser parser = new XGBoostModelParser(tokens);

        XGModelVisitor visitor = new XGModelVisitor();

        return visitor.visit(parser.xgModel());
    }
    catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 
Example 14
@Test
public void docComment()
{
    final CharStream input = CharStreams.fromString(
            "/** doc comment */\n" +
            "/**\n" +
            " * doc comment\n" +
            " * multiline\n" +
            " */"
    );

    ZserioLexer lexer = new ZserioLexer(input);

    checkToken(lexer, ZserioLexer.DOC_COMMENT, ZserioLexer.DOC, "/** doc comment */");
    checkToken(lexer, ZserioLexer.DOC_COMMENT, ZserioLexer.DOC, "/**\n * doc comment\n * multiline\n */");

    checkEOF(lexer);
}
 
Example 15
Source Project: RankPL   Source File: RPLBaseTest.java    License: MIT License 5 votes vote down vote up
protected static Object evalProgram(String program) {
       CharStream charStream = new ANTLRInputStream(program);
       RankPLLexer lexer = new RankPLLexer(charStream);
       TokenStream tokens = new CommonTokenStream(lexer);
       RankPLParser parser = new RankPLParser(tokens);

       ConcreteParser classVisitor = new ConcreteParser();
       ExpContext ctx = parser.exp();
       AbstractExpression res = (AbstractExpression)classVisitor.visit(ctx);
       
       
       return res;
}
 
Example 16
private static String getOriginalCodeFragment(ParserRuleContext ctx) {

    int startIdx = ctx.start.getStartIndex();
    int stopIdx = ctx.stop != null ? ctx.stop.getStopIndex() : -1;

    CharStream cs = ctx.getStart().getInputStream();

    if (cs.size() > 0) {
      return startIdx <= stopIdx ? cs.getText(Interval.of(startIdx, stopIdx)) : "";
    } else {
      return "";
    }
  }
 
Example 17
Source Project: antsdb   Source File: ExprGenerator.java    License: GNU Lesser General Public License v3.0 5 votes vote down vote up
static String getString(TerminalNode rule, Decoder decoder) {
    Token token = rule.getSymbol();
    CharStream cs = token.getInputStream();
    int pos = cs.index();
    cs.seek(token.getStartIndex() + 1);
    int len = token.getStopIndex() - token.getStartIndex() - 1;
    IntSupplier supplier = new IntSupplier() {
        int i = 0;
        
        @Override
        public int getAsInt() {
            if (i >= len) {
                return -1;
            }
            int ch = cs.LA(i + 1);
            if (ch == '\\') {
                i++;
                ch = cs.LA(i + 1);
                if (ch == '0') {
                    ch = 0;
                }
                else if (ch == 'n') {
                    ch = '\n';
                }
                else if (ch == 'r') {
                    ch = '\r';
                }
                else if (ch == 'Z') {
                    ch = '\032';
                }
            }
            i++;
            return ch;
        }
    };
    String result = decoder.toString(supplier);
    cs.seek(pos);
    return result;
}
 
Example 18
Source Project: RankPL   Source File: ProgramBuilder.java    License: MIT License 5 votes vote down vote up
public ProgramBuilder add(String statement) {
       CharStream charStream = new ANTLRInputStream(statement);
       RankPLLexer lexer = new RankPLLexer(charStream);
       TokenStream tokens = new CommonTokenStream(lexer);
       RankPLParser parser = new RankPLParser(tokens);
       ConcreteParser classVisitor = new ConcreteParser();
       return add((AbstractStatement)classVisitor.visit(parser.program()));
}
 
Example 19
Source Project: Concurnas   Source File: Conc.java    License: MIT License 5 votes vote down vote up
public ConcBuilder createBuilder() {
	ErrorCap errors = new ErrorCap();
	ConcBuilder builder;
	try {
		CharStream input = CharStreams.fromString(inputString, "cmd");
		
		ConcLexer lexer = new ConcLexer(input);
		CommonTokenStream tokens = new CommonTokenStream(lexer);
		ConcParser parser = new ConcParser(tokens);
		
		parser.getInterpreter().setPredictionMode(PredictionMode.SLL); 
		
		parser.removeErrorListeners(); // remove ConsoleErrorListener
		parser.addErrorListener(errors); // add ours
		
		lexer.removeErrorListeners(); // remove ConsoleErrorListener
		lexer.addErrorListener(errors); // add ours
		
		builder = new ConcBuilder();
		parser.conc().accept(builder);
		
	}catch(Throwable e) {
		return null;
	}
	
	if(errors.hasErrors) {
		return null;
	}
	
	return builder;
}
 
Example 20
@Nullable
@Override
public CharStream read(String contentOrName) {
  if (!contentReaded) {
    contentReaded = true;
    return CharStreams.fromString(contentOrName);
  }

  return importReader.read(contentOrName);
}
 
Example 21
Source Project: jaybird   Source File: TestGrammar.java    License: GNU Lesser General Public License v2.1 5 votes vote down vote up
private JaybirdSqlParser createParser(String testString) {
    CharStream stream = CharStreams.fromString(testString);
    
    JaybirdSqlLexer lexer = new JaybirdSqlLexer(stream);
    CommonTokenStream tokenStream = new CommonTokenStream(lexer);

    return new JaybirdSqlParser(tokenStream);
}
 
Example 22
Source Project: Concurnas   Source File: LocationRepointCommonTokenFactory.java    License: MIT License 5 votes vote down vote up
@Override
public CommonToken create(Pair<TokenSource, CharStream> source, int type, String text,
						  int channel, int start, int stop,
						  int line, int charPositionInLine)
{
	return super.create( source,  type,  text, channel,  start,  stop, /*line +*/ lineoffset,  charPositionInLine + coloffset);
}
 
Example 23
static FishParser.ScriptContext parse(CharStream cs) {
    FishLexer lexer = new FishLexer(cs);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    tokens.setTokenSource(lexer);
    FishParser parser = new FishParser(tokens);
    parser.setErrorHandler(new BailErrorStrategy());
    FishParser.ScriptContext script = parser.script();
    return script;
}
 
Example 24
Source Project: protostuff-compiler   Source File: CompositeFileReader.java    License: Apache License 2.0 5 votes vote down vote up
@Nullable
@Override
public CharStream read(String name) {
    for (FileReader delegate : delegateList) {
        CharStream result = delegate.read(name);
        if (result != null) {
            return result;
        }
    }
    return null;
}
 
Example 25
Source Project: incubator-iotdb   Source File: ParseDriver.java    License: Apache License 2.0 5 votes vote down vote up
public Operator parse(String sql, ZoneId zoneId) throws ParseCancellationException {
  LogicalGenerator logicalGenerator = new LogicalGenerator(zoneId);
  CharStream charStream1 = CharStreams.fromString(sql);
  SqlBaseLexer lexer1 = new SqlBaseLexer(charStream1);
  CommonTokenStream tokens1 = new CommonTokenStream(lexer1);
  SqlBaseParser parser1 = new SqlBaseParser(tokens1);
  parser1.getInterpreter().setPredictionMode(PredictionMode.SLL);
  parser1.removeErrorListeners();
  parser1.addErrorListener(LogicalGeneratorError.INSTANCE);
  ParseTree tree;
  try {
    tree = parser1.singleStatement();  // STAGE 1
  }
  catch (Exception ex) {
    CharStream charStream2 = CharStreams.fromString(sql);
    SqlBaseLexer lexer2 = new SqlBaseLexer(charStream2);
    CommonTokenStream tokens2 = new CommonTokenStream(lexer2);
    SqlBaseParser parser2 = new SqlBaseParser(tokens2);
    parser2.getInterpreter().setPredictionMode(PredictionMode.LL);
    parser2.removeErrorListeners();
    parser2.addErrorListener(LogicalGeneratorError.INSTANCE);
    tree = parser2.singleStatement();  // STAGE 2
    // if we parse ok, it's LL not SLL
  }
  walker.walk(logicalGenerator, tree);
  return logicalGenerator.getLogicalPlan();
}
 
Example 26
Source Project: protostuff-compiler   Source File: LocalFileReaderTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testRead() throws Exception {
    LocalFileReader reader = new LocalFileReader(tempDirectory1, tempDirectory2);
    CharStream a = reader.read("1.proto");
    CharStream b = reader.read("2.proto");
    CharStream c = reader.read("3.proto");
    assertNotNull(a);
    assertNotNull(b);
    assertNull(c);
    assertEquals("1", a.getText(Interval.of(0, 1)));
    assertEquals("2", b.getText(Interval.of(0, 1)));
}
 
Example 27
@Test
public void testRead() throws Exception {
    ClasspathFileReader reader = new ClasspathFileReader();
    CharStream a = reader.read("protostuff_unittest/messages_sample.proto");
    CharStream b = reader.read("this_file_does_not_exist.proto");
    assertNotNull(a);
    assertNull(b);
}
 
Example 28
Source Project: siddhi   Source File: SiddhiCompiler.java    License: Apache License 2.0 5 votes vote down vote up
public static Expression parseExpression(String expression) {
    CharStream input = CharStreams.fromString(expression);
    SiddhiQLLexer lexer = new SiddhiQLLexer(input);
    lexer.removeErrorListeners();
    lexer.addErrorListener(SiddhiErrorListener.INSTANCE);

    CommonTokenStream tokens = new CommonTokenStream(lexer);
    SiddhiQLParser parser = new SiddhiQLParser(tokens);
    parser.removeErrorListeners();
    parser.addErrorListener(SiddhiErrorListener.INSTANCE);
    ParseTree tree = parser.expression();

    SiddhiQLVisitor eval = new SiddhiQLBaseVisitorImpl();
    return (Expression) eval.visit(tree);
}
 
Example 29
Source Project: protostuff-compiler   Source File: ClasspathFileReader.java    License: Apache License 2.0 5 votes vote down vote up
@Nullable
@Override
public CharStream read(String name) {
    try {
        InputStream resource = readResource(name);
        if (resource != null) {
            return CharStreams.fromStream(resource);
        }
    } catch (Exception e) {
        LOGGER.error("Could not read {}", name, e);
    }
    return null;
}
 
Example 30
Source Project: beam   Source File: FieldAccessDescriptorParser.java    License: Apache License 2.0 5 votes vote down vote up
public static FieldAccessDescriptor parse(String expr) {
  CharStream charStream = CharStreams.fromString(expr);
  FieldSpecifierNotationLexer lexer = new FieldSpecifierNotationLexer(charStream);
  TokenStream tokens = new CommonTokenStream(lexer);
  FieldSpecifierNotationParser parser = new FieldSpecifierNotationParser(tokens);
  return new BuildFieldAccessDescriptor().visit(parser.dotExpression());
}