org.antlr.v4.runtime.CharStreams Java Examples

The following examples show how to use org.antlr.v4.runtime.CharStreams. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: MigrationSQLChecker.java    From kripton with Apache License 2.0 6 votes vote down vote up
/**
 * Prepare parser.
 *
 * @param jql the jql
 * @return the pair
 */
protected Pair<ParserRuleContext, CommonTokenStream> prepareParser(final String jql) {
	JqlLexer lexer = new JqlLexer(CharStreams.fromString(jql));
	CommonTokenStream tokens = new CommonTokenStream(lexer);
	JqlParser parser = new JqlParser(tokens);

	parser.removeErrorListeners();
	parser.addErrorListener(new JQLBaseErrorListener() {
		@Override
		public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line,
				int charPositionInLine, String msg, RecognitionException e) {
			KriptonAssert.assertTrue(false, "unespected char at pos %s of SQL '%s'", charPositionInLine, jql);
		}
	});

	ParserRuleContext context = parser.parse();
	return new Pair<>(context, tokens);
}
 
Example #2
Source File: ZserioLexerTest.java    From zserio with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
@Test
public void octalLiteral()
{
    final CharStream input = CharStreams.fromString(
            "0101 0770 012 00 01234567"
    );

    ZserioLexer lexer = new ZserioLexer(input);

    checkToken(lexer, ZserioLexer.OCTAL_LITERAL, "0101");
    checkToken(lexer, ZserioLexer.OCTAL_LITERAL, "0770");
    checkToken(lexer, ZserioLexer.OCTAL_LITERAL, "012");
    checkToken(lexer, ZserioLexer.OCTAL_LITERAL, "00");
    checkToken(lexer, ZserioLexer.OCTAL_LITERAL, "01234567");

    checkEOF(lexer);
}
 
Example #3
Source File: ZserioLexerTest.java    From zserio with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
@Test
public void invalidStringLiteral()
{
    final CharStream input = CharStreams.fromString(
            " \"invalid unicode escape character \\uBAD\" " +
            " \"invalid octal escape character \\09\" " +
            " \"invalid hexadecimal escape character \\xA\" "
    );

    ZserioLexer lexer = new ZserioLexer(input);

    checkToken(lexer, ZserioLexer.INVALID_STRING_LITERAL, "\"invalid unicode escape character \\uBAD\"");
    checkToken(lexer, ZserioLexer.INVALID_STRING_LITERAL, "\"invalid octal escape character \\09\"");
    checkToken(lexer, ZserioLexer.INVALID_STRING_LITERAL, "\"invalid hexadecimal escape character \\xA\"");

    checkEOF(lexer);
}
 
Example #4
Source File: ZserioLexerTest.java    From zserio with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
@Test
public void floatOrDoubleLiteral()
{
    final CharStream input = CharStreams.fromString(
            "0e1 0e-1 1e2f 9. 9.12 9.f 9.1f 9.e2 9.e2f 9.e-2f .0 .0e-1 .0f .1e3f"
    );

    ZserioLexer lexer = new ZserioLexer(input);

    checkToken(lexer, ZserioLexer.DOUBLE_LITERAL, "0e1");
    checkToken(lexer, ZserioLexer.DOUBLE_LITERAL, "0e-1");
    checkToken(lexer, ZserioLexer.FLOAT_LITERAL, "1e2f");
    checkToken(lexer, ZserioLexer.DOUBLE_LITERAL, "9.");
    checkToken(lexer, ZserioLexer.DOUBLE_LITERAL, "9.12");
    checkToken(lexer, ZserioLexer.FLOAT_LITERAL, "9.f");
    checkToken(lexer, ZserioLexer.FLOAT_LITERAL, "9.1f");
    checkToken(lexer, ZserioLexer.DOUBLE_LITERAL, "9.e2");
    checkToken(lexer, ZserioLexer.FLOAT_LITERAL, "9.e2f");
    checkToken(lexer, ZserioLexer.FLOAT_LITERAL, "9.e-2f");
    checkToken(lexer, ZserioLexer.DOUBLE_LITERAL, ".0");
    checkToken(lexer, ZserioLexer.DOUBLE_LITERAL, ".0e-1");
    checkToken(lexer, ZserioLexer.FLOAT_LITERAL, ".0f");
    checkToken(lexer, ZserioLexer.FLOAT_LITERAL, ".1e3f");

    checkEOF(lexer);
}
 
Example #5
Source File: DistkvParser.java    From distkv with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
public DistkvParsedResult parse(String command) {
  DistkvNewSqlListener dstNewSqlHandler = new DistkvNewSqlListener();
  DistkvNewSQLLexer lexer = new DistkvNewSQLLexer(CharStreams.fromString(command));

  //Add  DstErrorListener
  lexer.removeErrorListeners();
  lexer.addErrorListener(DistkvSqlErrorListener.INSTANCE);
  CommonTokenStream tokens = new CommonTokenStream(lexer);

  DistkvNewSQLParser parser = new DistkvNewSQLParser(tokens);
  parser.removeErrorListeners();
  parser.addErrorListener(DistkvSqlErrorListener.INSTANCE);

  DistkvNewSQLParser.StatementContext statement = parser.statement();
  ParseTreeWalker parseTreeWalker = new ParseTreeWalker();
  parseTreeWalker.walk(dstNewSqlHandler, statement);

  return dstNewSqlHandler.getParsedResult();
}
 
Example #6
Source File: ContentUriChecker.java    From kripton with Apache License 2.0 6 votes vote down vote up
/**
 * Prepare path.
 *
 * @param input the input
 * @return the pair
 */
private Pair<ParserRuleContext, CommonTokenStream> preparePath(final String input) {
	UriLexer lexer = new UriLexer(CharStreams.fromString(input));
	CommonTokenStream tokens = new CommonTokenStream(lexer);
	UriParser parser = new UriParser(tokens);

	parser.removeErrorListeners();
	parser.addErrorListener(new ContentUriBaseErrorListener() {
		@Override
		public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) {
			AssertKripton.assertTrue(false, "unespected char at pos %s of URI '%s'", charPositionInLine, input);
		}
	});

	ParserRuleContext context = parser.path();
	return new Pair<>(context, tokens);
}
 
Example #7
Source File: ZserioLexerTest.java    From zserio with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
@Test
public void binaryLiteral()
{
    final CharStream input = CharStreams.fromString(
            "1001b 0101b 00b 0B 1B 101B"
    );

    ZserioLexer lexer = new ZserioLexer(input);

    checkToken(lexer, ZserioLexer.BINARY_LITERAL, "1001b");
    checkToken(lexer, ZserioLexer.BINARY_LITERAL, "0101b");
    checkToken(lexer, ZserioLexer.BINARY_LITERAL, "00b");
    checkToken(lexer, ZserioLexer.BINARY_LITERAL, "0B");
    checkToken(lexer, ZserioLexer.BINARY_LITERAL, "1B");
    checkToken(lexer, ZserioLexer.BINARY_LITERAL, "101B");

    checkEOF(lexer);
}
 
Example #8
Source File: DefaultLinesProviderTest.java    From sonar-tsql-plugin with GNU General Public License v3.0 6 votes vote down vote up
@Test
public void test() {

	final CharStream charStream = CharStreams.fromString("\r\nSELECT\r\n 1");

	final TSqlLexer lexer = new TSqlLexer(charStream);

	final CommonTokenStream stream = new CommonTokenStream(lexer);

	stream.fill();
	TSqlParser parser = new TSqlParser(stream);
	ParseTree child = parser.tsql_file().getChild(0);
	DefaultLinesProvider lines = new DefaultLinesProvider(stream);
	int line = lines.getLine(new ParsedNode(child));
	Assert.assertEquals(2, line);

}
 
Example #9
Source File: Queries.java    From elepy with Apache License 2.0 6 votes vote down vote up
public static Query parse(String input) {

        if (StringUtils.isEmpty(input)) {
            return Queries.empty();
        }
        //Lex (with Antlr's generated lexer)
        CharStream inputStream = CharStreams.fromString(input);
        EleQueryLexer lexer = new EleQueryLexer(inputStream);
        CommonTokenStream tokens = new CommonTokenStream(lexer);

        //Parse (with Antlr's generated parser)
        EleQueryParser parser = new EleQueryParser(tokens);
        ParseTree parseTree = parser.query();

        //Extract AST from the Antlr parse tree
        QueryListener listener = new QueryListener();
        ParseTreeWalker walker = new ParseTreeWalker();
        walker.walk(listener, parseTree);

        return listener.getQuery();
    }
 
Example #10
Source File: InsertPSTCode.java    From latexdraw with GNU General Public License v3.0 6 votes vote down vote up
@Override
protected void doCmdBody() {
	final PSTLatexdrawListener listener = new PSTLatexdrawListener();
	final PSTLexer lexer = new PSTLexer(CharStreams.fromString(code));
	final PSTParser parser = new PSTParser(new CommonTokenStream(lexer));
	parser.addParseListener(listener);
	parser.pstCode(new PSTContext());

	final Group group = ShapeFactory.INST.createGroup();
	group.getShapes().addAll(listener.flatShapes());

	if(!group.isEmpty()) {
		final Shape sh = group.size() > 1 ? group : group.getShapeAt(0).orElseThrow();
		final Point tl = sh.getTopLeftPoint();
		final double tx = tl.getX() < 0d ? -tl.getX() + 50d : 0d;
		final double ty = tl.getY() < 0d ? -tl.getY() + 50d : 0d;

		shapes = Optional.of(sh);
		sh.translate(tx, ty);
		redo();

		statusBar.setText(lang.getString("codeConverted"));
	}
	parser.getInterpreter().clearDFA();
	lexer.getInterpreter().clearDFA();
}
 
Example #11
Source File: ProtoParseListenerTest.java    From protostuff-compiler with Apache License 2.0 6 votes vote down vote up
private Proto parseProto(String input) {
    CharStream stream = CharStreams.fromString(input);
    ProtoLexer lexer = new ProtoLexer(stream);
    lexer.removeErrorListeners();
    lexer.addErrorListener(TestUtils.ERROR_LISTENER);
    CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    ProtoParser parser = new ProtoParser(tokenStream);
    parser.removeErrorListeners();
    parser.addErrorListener(TestUtils.ERROR_LISTENER);
    ProtoContext context = new ProtoContext("test.proto");
    ProtoParseListener protoParseListener = new ProtoParseListener(tokenStream, context);
    OptionParseListener optionParseListener = new OptionParseListener(tokenStream, context);
    parser.addParseListener(protoParseListener);
    parser.addParseListener(optionParseListener);
    parser.proto();
    return context.getProto();
}
 
Example #12
Source File: PomFileParser.java    From depends with MIT License 6 votes vote down vote up
@Override
public void parse() throws IOException {
	/* If file already exist, skip it */
	Entity fileEntity = entityRepo.getEntity(fileFullPath);
	if (fileEntity!=null && fileEntity instanceof FileEntity) {
		return;
	}
	/*parse file*/
       CharStream input = CharStreams.fromFileName(fileFullPath);
       Lexer lexer = new XMLLexer(input);
       CommonTokenStream tokens = new CommonTokenStream(lexer);
       XMLParser parser = new XMLParser(tokens);
       PomListener bridge = new PomListener(fileFullPath, entityRepo, includePaths,parseCreator,inferer);
    ParseTreeWalker walker = new ParseTreeWalker();
    walker.walk(bridge, parser.document());
	fileEntity = entityRepo.getEntity(fileFullPath);
	bridge.done();
	((FileEntity)fileEntity).cacheAllExpressions();
}
 
Example #13
Source File: PageParser.java    From citeproc-java with Apache License 2.0 6 votes vote down vote up
/**
 * Parses a given page or range of pages. If the given string cannot
 * be parsed, the method will return a page range with a literal string.
 * @param pages the page or range of pages
 * @return the parsed page or page range (never {@code null})
 */
public static PageRange parse(String pages) {
    CharStream cs = CharStreams.fromString(pages);
    InternalPageLexer lexer = new InternalPageLexer(cs);
    lexer.removeErrorListeners(); // do not output errors to console
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    InternalPageParser parser = new InternalPageParser(tokens);
    parser.removeErrorListeners(); // do not output errors to console
    PagesContext ctx = parser.pages();
    if (ctx.literal == null || ctx.literal.isEmpty() ||
            ctx.exception != null || parser.getNumberOfSyntaxErrors() > 0) {
        // unparsable fall back to literal string
        return new PageRange(pages, null, null, false);
    }
    return new PageRange(ctx.literal, ctx.pageFrom, ctx.numberOfPages,
            ctx.multiplePages);
}
 
Example #14
Source File: XGModelParser.java    From ignite with Apache License 2.0 6 votes vote down vote up
/** {@inheritDoc} */
@Override public XGModelComposition parse(byte[] mdl) {
    try (ByteArrayInputStream bais = new ByteArrayInputStream(mdl)) {
        CharStream cStream = CharStreams.fromStream(bais);
        XGBoostModelLexer lexer = new XGBoostModelLexer(cStream);
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        XGBoostModelParser parser = new XGBoostModelParser(tokens);

        XGModelVisitor visitor = new XGModelVisitor();

        return visitor.visit(parser.xgModel());
    }
    catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 
Example #15
Source File: AbstractDataSetResource.java    From Knowage-Server with GNU Affero General Public License v3.0 6 votes vote down vote up
public String validateFormula(String formula, List<SimpleSelectionField> columns) throws ValidationException, JSONException {

		validateBrackets(formula);
		validateFields(formula, columns);

		formula = "select ".concat(formula);
		CharStream inputStream = CharStreams.fromString(formula);
		SQLiteLexer tokenSource = new SQLiteLexer(new CaseChangingCharStream(inputStream, true));
		TokenStream tokenStream = new CommonTokenStream(tokenSource);
		SQLiteParser sQLiteParser = new SQLiteParser(tokenStream);

		sQLiteParser.addErrorListener(ThrowingErrorListener.INSTANCE);
		try {
			ParseTree root = sQLiteParser.select_stmt();
			root.toStringTree();
		} catch (Exception e) {
			throw new ValidationException(e);
		}
		if (sQLiteParser.getNumberOfSyntaxErrors() > 0) {
			throw new ValidationException();

		}

		return VALIDATION_OK;
	}
 
Example #16
Source File: ZserioLexerTest.java    From zserio with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
@Test
public void id()
{
    final CharStream input = CharStreams.fromString(
            "id _id i1 _1 _ __ I _I s0mE_UgLy_id"
    );

    ZserioLexer lexer = new ZserioLexer(input);

    checkToken(lexer, ZserioLexer.ID, "id");
    checkToken(lexer, ZserioLexer.ID, "_id");
    checkToken(lexer, ZserioLexer.ID, "i1");
    checkToken(lexer, ZserioLexer.ID, "_1");
    checkToken(lexer, ZserioLexer.ID, "_");
    checkToken(lexer, ZserioLexer.ID, "__");
    checkToken(lexer, ZserioLexer.ID, "I");
    checkToken(lexer, ZserioLexer.ID, "_I");
    checkToken(lexer, ZserioLexer.ID, "s0mE_UgLy_id");

    checkEOF(lexer);
}
 
Example #17
Source File: StatementParserImpl.java    From jaybird with GNU Lesser General Public License v2.1 6 votes vote down vote up
@Override
public JaybirdStatementModel parseStatement(String sql) throws ParseException {
    try {
        CharStream stream = CharStreams.fromString(sql);
        JaybirdSqlLexer lexer = new JaybirdSqlLexer(stream);
        CommonTokenStream tokenStream = new CommonTokenStream(lexer);
        
        JaybirdSqlParser parser = new JaybirdSqlParser(tokenStream);
        parser.removeErrorListeners();
        parser.statement();

        JaybirdStatementModel statementModel = parser.getStatementModel();
        if (statementModel.getStatementType() == JaybirdStatementModel.UNDETECTED_TYPE) {
            throw new ParseException("Unable to detect statement type or unsupported statement type");
        }
        if (statementModel.getTableName() == null) {
            throw new ParseException("Unable to parse query: no table name found");
        }
        return statementModel;
    } catch (RecognitionException e) {
        throw new ParseException("Unable to parse query", e);
    }
}
 
Example #18
Source File: AtlasDSL.java    From atlas with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
static AtlasDSLParser.QueryContext parse(String queryStr) throws AtlasBaseException {
    AtlasDSLParser.QueryContext ret;
    try {
        InputStream    stream           = new ByteArrayInputStream(queryStr.getBytes());
        AtlasDSLLexer  lexer            = new AtlasDSLLexer(CharStreams.fromStream(stream));
        Validator      validator        = new Validator();
        TokenStream    inputTokenStream = new CommonTokenStream(lexer);
        AtlasDSLParser parser           = new AtlasDSLParser(inputTokenStream);

        parser.removeErrorListeners();
        parser.addErrorListener(validator);

        // Validate the syntax of the query here
        ret = parser.query();
        if (!validator.isValid()) {
            LOG.error("Invalid DSL: {} Reason: {}", queryStr, validator.getErrorMsg());
            throw new AtlasBaseException(AtlasErrorCode.INVALID_DSL_QUERY, queryStr, validator.getErrorMsg());
        }

    } catch (IOException e) {
        throw new AtlasBaseException(e);
    }

    return ret;
}
 
Example #19
Source File: InstanceIdentifierParser.java    From yangtools with Eclipse Public License 1.0 6 votes vote down vote up
final Absolute interpretAsInstanceIdentifier(final YangLiteralExpr expr) throws XPathExpressionException {
    final xpathLexer lexer = new xpathLexer(CharStreams.fromString(expr.getLiteral()));
    final instanceIdentifierParser parser = new instanceIdentifierParser(new CommonTokenStream(lexer));
    final CapturingErrorListener listener = new CapturingErrorListener();
    lexer.removeErrorListeners();
    lexer.addErrorListener(listener);
    parser.removeErrorListeners();
    parser.addErrorListener(listener);

    final InstanceIdentifierContext id = parser.instanceIdentifier();
    listener.reportError();

    final int length = id.getChildCount();
    final List<Step> steps = new ArrayList<>(length / 2);
    for (int i = 1; i < length; i += 2) {
        steps.add(parsePathArgument(getChild(id, PathArgumentContext.class, i)));
    }

    return YangLocationPath.absolute(steps);
}
 
Example #20
Source File: ZserioTool.java    From zserio with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
private Package parsePackage(ZserioAstBuilder astBuilder,
        String inputFileFullName) throws Exception
{
    ZserioToolPrinter.printMessage("Parsing " + inputFileFullName);

    inputFileManager.registerFile(inputFileFullName);

    final CharStream inputStream = CharStreams.fromFileName(inputFileFullName, Charset.forName("UTF-8"));
    final ParseErrorListener parseErrorListener = new ParseErrorListener();
    final ZserioLexer lexer = new ZserioLexer(inputStream);
    lexer.removeErrorListeners();
    lexer.addErrorListener(parseErrorListener);
    final CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    final ZserioParser parser = new ZserioParser(tokenStream);
    parser.removeErrorListeners();
    parser.addErrorListener(parseErrorListener);

    final ParseTree tree = parser.packageDeclaration();

    final ZserioParseTreeChecker parseTreeChecker = new ZserioParseTreeChecker(inputFileManager);
    parseTreeChecker.visit(tree);

    final Package parsedPackage = (Package)astBuilder.visit(tree, tokenStream);

    return parsedPackage;
}
 
Example #21
Source File: Tool.java    From bookish with MIT License 6 votes vote down vote up
public String translateString(Translator trans, String markdown, String startRule) throws Exception {
	CharStream input = CharStreams.fromString(markdown);
	BookishLexer lexer = new BookishLexer(input);
	CommonTokenStream tokens = new CommonTokenStream(lexer);
	BookishParser parser = new BookishParser(tokens,null, 0);
	parser.removeErrorListeners();
	parser.addErrorListener(new ConsoleErrorListener() {
		@Override
		public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) {
			msg = "Parsing author string: "+msg;
			super.syntaxError(recognizer, offendingSymbol, line, charPositionInLine, msg, e);
		}
	});
	Method startMethod = BookishParser.class.getMethod(startRule, (Class[])null);
	ParseTree doctree = (ParseTree)startMethod.invoke(parser, (Object[])null);

	OutputModelObject omo = trans.visit(doctree); // get single chapter

	ModelConverter converter = new ModelConverter(trans.templates);
	ST outputST = converter.walk(omo);
	return outputST.render();
}
 
Example #22
Source File: QueryHelper.java    From hasor with Apache License 2.0 6 votes vote down vote up
/**
 * 编译已经解析好的 DataQL
 * @param queryModel 解析之后的 DataQL 查询模型。
 * @param compilerArguments 编译参数,编译参数中包含了编译器需要的一些列变量。例如:编译变量,相当于在脚本中预先 执行 var xxx = null;
 *                    其意义在于在编译期就把脚本中尚未定义过的变量预先进行声明从而免去通过 ${...} 或类似方式查找变量。
 * @param importFinder import 导入用到的资源加载器。
 */
public static QIL queryCompiler(QueryModel queryModel, CompilerArguments compilerArguments, Finder importFinder) throws IOException {
    RootBlockSet rootBlockSet = null;
    if (queryModel instanceof RootBlockSet) {
        rootBlockSet = (RootBlockSet) queryModel;
    } else {
        rootBlockSet = (RootBlockSet) queryParser(CharStreams.fromString(queryModel.toQueryString()));
    }
    //
    compilerArguments = (compilerArguments == null) ? CompilerArguments.DEFAULT : compilerArguments;
    Set<String> compilerVar = compilerArguments.getCompilerVar();
    if (compilerVar == null) {
        compilerVar = Collections.emptySet();
    }
    //
    InstQueue queue = new InstQueue(compilerArguments);
    CompilerContext compilerContext = new CompilerContext(new CompilerEnvironment(importFinder));
    Map<String, Integer> compilerVarMap = new HashMap<>();
    compilerVar.forEach(var -> {
        int localIdx = compilerContext.push(var);
        compilerVarMap.put(var, localIdx);
    });
    compilerContext.findInstCompilerByInst(rootBlockSet).doCompiler(queue);
    Instruction[][] queueSet = queue.buildArrays();
    return new QIL(queueSet, compilerVarMap);
}
 
Example #23
Source File: SiddhiCompiler.java    From siddhi with Apache License 2.0 6 votes vote down vote up
public static OnDemandQuery parseOnDemandQuery(String onDemandQueryString) throws SiddhiParserException {
    CharStream input = CharStreams.fromString(onDemandQueryString);
    SiddhiQLLexer lexer = new SiddhiQLLexer(input);
    lexer.removeErrorListeners();
    lexer.addErrorListener(SiddhiErrorListener.INSTANCE);

    CommonTokenStream tokens = new CommonTokenStream(lexer);
    SiddhiQLParser parser = new SiddhiQLParser(tokens);
    parser.removeErrorListeners();
    parser.addErrorListener(SiddhiErrorListener.INSTANCE);
    ParseTree tree = parser.store_query_final();

    SiddhiQLVisitor eval = new SiddhiQLBaseVisitorImpl();
    OnDemandQuery onDemandQuery = (OnDemandQuery) eval.visit(tree);
    return onDemandQuery;
}
 
Example #24
Source File: CodeTokenizer.java    From scava with Eclipse Public License 2.0 6 votes vote down vote up
/**
 * This method will tokenize the program and get all the elements that are
 * variable names, variables types, methods names. Depending on the library
 * it will imported libraries names too.
 * @param program
 * @param language
 * @param keepOrder If false, then the program's tokens will be filtered using multiple threads
 * @return 
 */
public static Set<String> tokenize(String program, String language, boolean keepOrder)
{
	CodePointCharStream code = CharStreams.fromString(program);
	Set<String> filteresTokens;
	switch(language)
	{
		case "c": 
			filteresTokens=CTokens(code, keepOrder);
			break;
		case "java":
			filteresTokens=JavaTokens(code, keepOrder);
			break;
		case "javascript":
			filteresTokens=JavaScriptTokens(code, keepOrder);
			break;
		case "php":
			filteresTokens=PhpTokens(code, keepOrder);
			break;
		default:
			System.err.println("Not supported language");
			filteresTokens=new HashSet<String>(0);
			break;
	}
	return filteresTokens;
}
 
Example #25
Source File: ZserioLexerTest.java    From zserio with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
@Test
public void blockComment()
{
    final CharStream input = CharStreams.fromString(
            "/* block comment */\n" +
            "/* block comment\n" +
            "   multiline */"
    );

    ZserioLexer lexer = new ZserioLexer(input);

    checkToken(lexer, ZserioLexer.BLOCK_COMMENT, Lexer.HIDDEN, "/* block comment */");
    checkToken(lexer, ZserioLexer.BLOCK_COMMENT, Lexer.HIDDEN, "/* block comment\n   multiline */");

    checkEOF(lexer);
}
 
Example #26
Source File: ZserioLexerTest.java    From zserio with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
@Test
public void docComment()
{
    final CharStream input = CharStreams.fromString(
            "/** doc comment */\n" +
            "/**\n" +
            " * doc comment\n" +
            " * multiline\n" +
            " */"
    );

    ZserioLexer lexer = new ZserioLexer(input);

    checkToken(lexer, ZserioLexer.DOC_COMMENT, ZserioLexer.DOC, "/** doc comment */");
    checkToken(lexer, ZserioLexer.DOC_COMMENT, ZserioLexer.DOC, "/**\n * doc comment\n * multiline\n */");

    checkEOF(lexer);
}
 
Example #27
Source File: JRubyFileParser.java    From depends with MIT License 6 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public void parse() throws IOException {
	/** If file already exist, skip it */
	Entity fileEntity = entityRepo.getEntity(fileFullPath);
	if (fileEntity!=null && fileEntity instanceof FileEntity) {
		return;
	}
	
       CharStream input = CharStreams.fromFileName(fileFullPath);
	Parser rubyParser = new Parser();
	StringReader in = new StringReader(input.toString());
	CompatVersion version = CompatVersion.RUBY2_3;
	ParserConfiguration config = new ParserConfiguration(0, version);
	try {
		Node node = rubyParser.parse("<code>", in, config);
		JRubyVisitor parser = new JRubyVisitor(fileFullPath, entityRepo, includesFileLocator,executor,inferer,parserCreator);
		node.accept(parser);
		fileEntity = entityRepo.getEntity(fileFullPath);
		((FileEntity)fileEntity).cacheAllExpressions();
		parser.done();
	}catch(Exception e) {
		System.err.println("parsing error in "+fileFullPath);
	}
}
 
Example #28
Source File: RegexLiterals.java    From onedev with MIT License 5 votes vote down vote up
public RegexLiterals(String regex) {
	CharStream stream = CharStreams.fromString(regex);
	PCRELexer lexer = new PCRELexer(stream);
	lexer.removeErrorListeners();
	lexer.addErrorListener(ErrorListener.INSTANCE);
	CommonTokenStream tokens = new CommonTokenStream(lexer);
	
	PCREParser parser = new PCREParser(tokens);
	parser.removeErrorListeners();
	parser.addErrorListener(ErrorListener.INSTANCE);
	
	OrLiterals orLiterals = (OrLiterals) new LiteralVisitor().visit(parser.parse());
	rows = orLiterals.flattern(true);
}
 
Example #29
Source File: Tool.java    From bookish with MIT License 5 votes vote down vote up
public static BookishParser.ContentContext parseString(String inputStr, String location) {
	CharStream input = CharStreams.fromString(inputStr);
	BookishLexer lexer = new BookishLexer(input);
	CommonTokenStream tokens = new CommonTokenStream(lexer);
	BookishParser parser = new BookishParser(tokens);

	lexer.removeErrorListeners();
	lexer.addErrorListener(getErrorListener(location));
	parser.removeErrorListeners();
	parser.addErrorListener(getErrorListener(location));

	BookishParser.ContentContext tree = parser.content();
	return tree;
}
 
Example #30
Source File: LexerWrapper.java    From antlr4-autosuggest with Apache License 2.0 5 votes vote down vote up
private static CharStream toCharStream(String text) {
    CharStream inputStream;
    try {
        inputStream = CharStreams.fromReader(new StringReader(text));
    } catch (IOException e) {
        throw new IllegalStateException("Unexpected while reading input string", e);
    }
    return inputStream;
}