org.antlr.v4.runtime.CharStreams Java Examples

/**
 * Prepare parser.
 *
 * @param jql the jql
 * @return the pair
 */
protected Pair<ParserRuleContext, CommonTokenStream> prepareParser(final String jql) {
	JqlLexer lexer = new JqlLexer(CharStreams.fromString(jql));
	CommonTokenStream tokens = new CommonTokenStream(lexer);
	JqlParser parser = new JqlParser(tokens);

	parser.removeErrorListeners();
	parser.addErrorListener(new JQLBaseErrorListener() {
		@Override
		public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line,
				int charPositionInLine, String msg, RecognitionException e) {
			KriptonAssert.assertTrue(false, "unespected char at pos %s of SQL '%s'", charPositionInLine, jql);
		}
	});

	ParserRuleContext context = parser.parse();
	return new Pair<>(context, tokens);
}
 

@Test
public void octalLiteral()
{
    final CharStream input = CharStreams.fromString(
            "0101 0770 012 00 01234567"
    );

    ZserioLexer lexer = new ZserioLexer(input);

    checkToken(lexer, ZserioLexer.OCTAL_LITERAL, "0101");
    checkToken(lexer, ZserioLexer.OCTAL_LITERAL, "0770");
    checkToken(lexer, ZserioLexer.OCTAL_LITERAL, "012");
    checkToken(lexer, ZserioLexer.OCTAL_LITERAL, "00");
    checkToken(lexer, ZserioLexer.OCTAL_LITERAL, "01234567");

    checkEOF(lexer);
}
 

@Test
public void invalidStringLiteral()
{
    final CharStream input = CharStreams.fromString(
            " \"invalid unicode escape character \\uBAD\" " +
            " \"invalid octal escape character \\09\" " +
            " \"invalid hexadecimal escape character \\xA\" "
    );

    ZserioLexer lexer = new ZserioLexer(input);

    checkToken(lexer, ZserioLexer.INVALID_STRING_LITERAL, "\"invalid unicode escape character \\uBAD\"");
    checkToken(lexer, ZserioLexer.INVALID_STRING_LITERAL, "\"invalid octal escape character \\09\"");
    checkToken(lexer, ZserioLexer.INVALID_STRING_LITERAL, "\"invalid hexadecimal escape character \\xA\"");

    checkEOF(lexer);
}
 

@Test
public void floatOrDoubleLiteral()
{
    final CharStream input = CharStreams.fromString(
            "0e1 0e-1 1e2f 9. 9.12 9.f 9.1f 9.e2 9.e2f 9.e-2f .0 .0e-1 .0f .1e3f"
    );

    ZserioLexer lexer = new ZserioLexer(input);

    checkToken(lexer, ZserioLexer.DOUBLE_LITERAL, "0e1");
    checkToken(lexer, ZserioLexer.DOUBLE_LITERAL, "0e-1");
    checkToken(lexer, ZserioLexer.FLOAT_LITERAL, "1e2f");
    checkToken(lexer, ZserioLexer.DOUBLE_LITERAL, "9.");
    checkToken(lexer, ZserioLexer.DOUBLE_LITERAL, "9.12");
    checkToken(lexer, ZserioLexer.FLOAT_LITERAL, "9.f");
    checkToken(lexer, ZserioLexer.FLOAT_LITERAL, "9.1f");
    checkToken(lexer, ZserioLexer.DOUBLE_LITERAL, "9.e2");
    checkToken(lexer, ZserioLexer.FLOAT_LITERAL, "9.e2f");
    checkToken(lexer, ZserioLexer.FLOAT_LITERAL, "9.e-2f");
    checkToken(lexer, ZserioLexer.DOUBLE_LITERAL, ".0");
    checkToken(lexer, ZserioLexer.DOUBLE_LITERAL, ".0e-1");
    checkToken(lexer, ZserioLexer.FLOAT_LITERAL, ".0f");
    checkToken(lexer, ZserioLexer.FLOAT_LITERAL, ".1e3f");

    checkEOF(lexer);
}
 

public DistkvParsedResult parse(String command) {
  DistkvNewSqlListener dstNewSqlHandler = new DistkvNewSqlListener();
  DistkvNewSQLLexer lexer = new DistkvNewSQLLexer(CharStreams.fromString(command));

  //Add  DstErrorListener
  lexer.removeErrorListeners();
  lexer.addErrorListener(DistkvSqlErrorListener.INSTANCE);
  CommonTokenStream tokens = new CommonTokenStream(lexer);

  DistkvNewSQLParser parser = new DistkvNewSQLParser(tokens);
  parser.removeErrorListeners();
  parser.addErrorListener(DistkvSqlErrorListener.INSTANCE);

  DistkvNewSQLParser.StatementContext statement = parser.statement();
  ParseTreeWalker parseTreeWalker = new ParseTreeWalker();
  parseTreeWalker.walk(dstNewSqlHandler, statement);

  return dstNewSqlHandler.getParsedResult();
}
 

/**
 * Prepare path.
 *
 * @param input the input
 * @return the pair
 */
private Pair<ParserRuleContext, CommonTokenStream> preparePath(final String input) {
	UriLexer lexer = new UriLexer(CharStreams.fromString(input));
	CommonTokenStream tokens = new CommonTokenStream(lexer);
	UriParser parser = new UriParser(tokens);

	parser.removeErrorListeners();
	parser.addErrorListener(new ContentUriBaseErrorListener() {
		@Override
		public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) {
			AssertKripton.assertTrue(false, "unespected char at pos %s of URI '%s'", charPositionInLine, input);
		}
	});

	ParserRuleContext context = parser.path();
	return new Pair<>(context, tokens);
}
 

@Test
public void binaryLiteral()
{
    final CharStream input = CharStreams.fromString(
            "1001b 0101b 00b 0B 1B 101B"
    );

    ZserioLexer lexer = new ZserioLexer(input);

    checkToken(lexer, ZserioLexer.BINARY_LITERAL, "1001b");
    checkToken(lexer, ZserioLexer.BINARY_LITERAL, "0101b");
    checkToken(lexer, ZserioLexer.BINARY_LITERAL, "00b");
    checkToken(lexer, ZserioLexer.BINARY_LITERAL, "0B");
    checkToken(lexer, ZserioLexer.BINARY_LITERAL, "1B");
    checkToken(lexer, ZserioLexer.BINARY_LITERAL, "101B");

    checkEOF(lexer);
}
 

@Test
public void test() {

	final CharStream charStream = CharStreams.fromString("\r\nSELECT\r\n 1");

	final TSqlLexer lexer = new TSqlLexer(charStream);

	final CommonTokenStream stream = new CommonTokenStream(lexer);

	stream.fill();
	TSqlParser parser = new TSqlParser(stream);
	ParseTree child = parser.tsql_file().getChild(0);
	DefaultLinesProvider lines = new DefaultLinesProvider(stream);
	int line = lines.getLine(new ParsedNode(child));
	Assert.assertEquals(2, line);

}
 

public static Query parse(String input) {

        if (StringUtils.isEmpty(input)) {
            return Queries.empty();
        }
        //Lex (with Antlr's generated lexer)
        CharStream inputStream = CharStreams.fromString(input);
        EleQueryLexer lexer = new EleQueryLexer(inputStream);
        CommonTokenStream tokens = new CommonTokenStream(lexer);

        //Parse (with Antlr's generated parser)
        EleQueryParser parser = new EleQueryParser(tokens);
        ParseTree parseTree = parser.query();

        //Extract AST from the Antlr parse tree
        QueryListener listener = new QueryListener();
        ParseTreeWalker walker = new ParseTreeWalker();
        walker.walk(listener, parseTree);

        return listener.getQuery();
    }
 

@Override
protected void doCmdBody() {
	final PSTLatexdrawListener listener = new PSTLatexdrawListener();
	final PSTLexer lexer = new PSTLexer(CharStreams.fromString(code));
	final PSTParser parser = new PSTParser(new CommonTokenStream(lexer));
	parser.addParseListener(listener);
	parser.pstCode(new PSTContext());

	final Group group = ShapeFactory.INST.createGroup();
	group.getShapes().addAll(listener.flatShapes());

	if(!group.isEmpty()) {
		final Shape sh = group.size() > 1 ? group : group.getShapeAt(0).orElseThrow();
		final Point tl = sh.getTopLeftPoint();
		final double tx = tl.getX() < 0d ? -tl.getX() + 50d : 0d;
		final double ty = tl.getY() < 0d ? -tl.getY() + 50d : 0d;

		shapes = Optional.of(sh);
		sh.translate(tx, ty);
		redo();

		statusBar.setText(lang.getString("codeConverted"));
	}
	parser.getInterpreter().clearDFA();
	lexer.getInterpreter().clearDFA();
}
 

private Proto parseProto(String input) {
    CharStream stream = CharStreams.fromString(input);
    ProtoLexer lexer = new ProtoLexer(stream);
    lexer.removeErrorListeners();
    lexer.addErrorListener(TestUtils.ERROR_LISTENER);
    CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    ProtoParser parser = new ProtoParser(tokenStream);
    parser.removeErrorListeners();
    parser.addErrorListener(TestUtils.ERROR_LISTENER);
    ProtoContext context = new ProtoContext("test.proto");
    ProtoParseListener protoParseListener = new ProtoParseListener(tokenStream, context);
    OptionParseListener optionParseListener = new OptionParseListener(tokenStream, context);
    parser.addParseListener(protoParseListener);
    parser.addParseListener(optionParseListener);
    parser.proto();
    return context.getProto();
}
 

@Override
public void parse() throws IOException {
	/* If file already exist, skip it */
	Entity fileEntity = entityRepo.getEntity(fileFullPath);
	if (fileEntity!=null && fileEntity instanceof FileEntity) {
		return;
	}
	/*parse file*/
       CharStream input = CharStreams.fromFileName(fileFullPath);
       Lexer lexer = new XMLLexer(input);
       CommonTokenStream tokens = new CommonTokenStream(lexer);
       XMLParser parser = new XMLParser(tokens);
       PomListener bridge = new PomListener(fileFullPath, entityRepo, includePaths,parseCreator,inferer);
    ParseTreeWalker walker = new ParseTreeWalker();
    walker.walk(bridge, parser.document());
	fileEntity = entityRepo.getEntity(fileFullPath);
	bridge.done();
	((FileEntity)fileEntity).cacheAllExpressions();
}
 

/**
 * Parses a given page or range of pages. If the given string cannot
 * be parsed, the method will return a page range with a literal string.
 * @param pages the page or range of pages
 * @return the parsed page or page range (never {@code null})
 */
public static PageRange parse(String pages) {
    CharStream cs = CharStreams.fromString(pages);
    InternalPageLexer lexer = new InternalPageLexer(cs);
    lexer.removeErrorListeners(); // do not output errors to console
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    InternalPageParser parser = new InternalPageParser(tokens);
    parser.removeErrorListeners(); // do not output errors to console
    PagesContext ctx = parser.pages();
    if (ctx.literal == null || ctx.literal.isEmpty() ||
            ctx.exception != null || parser.getNumberOfSyntaxErrors() > 0) {
        // unparsable fall back to literal string
        return new PageRange(pages, null, null, false);
    }
    return new PageRange(ctx.literal, ctx.pageFrom, ctx.numberOfPages,
            ctx.multiplePages);
}
 

/** {@inheritDoc} */
@Override public XGModelComposition parse(byte[] mdl) {
    try (ByteArrayInputStream bais = new ByteArrayInputStream(mdl)) {
        CharStream cStream = CharStreams.fromStream(bais);
        XGBoostModelLexer lexer = new XGBoostModelLexer(cStream);
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        XGBoostModelParser parser = new XGBoostModelParser(tokens);

        XGModelVisitor visitor = new XGModelVisitor();

        return visitor.visit(parser.xgModel());
    }
    catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 

public String validateFormula(String formula, List<SimpleSelectionField> columns) throws ValidationException, JSONException {

		validateBrackets(formula);
		validateFields(formula, columns);

		formula = "select ".concat(formula);
		CharStream inputStream = CharStreams.fromString(formula);
		SQLiteLexer tokenSource = new SQLiteLexer(new CaseChangingCharStream(inputStream, true));
		TokenStream tokenStream = new CommonTokenStream(tokenSource);
		SQLiteParser sQLiteParser = new SQLiteParser(tokenStream);

		sQLiteParser.addErrorListener(ThrowingErrorListener.INSTANCE);
		try {
			ParseTree root = sQLiteParser.select_stmt();
			root.toStringTree();
		} catch (Exception e) {
			throw new ValidationException(e);
		}
		if (sQLiteParser.getNumberOfSyntaxErrors() > 0) {
			throw new ValidationException();

		}

		return VALIDATION_OK;
	}
 

@Test
public void id()
{
    final CharStream input = CharStreams.fromString(
            "id _id i1 _1 _ __ I _I s0mE_UgLy_id"
    );

    ZserioLexer lexer = new ZserioLexer(input);

    checkToken(lexer, ZserioLexer.ID, "id");
    checkToken(lexer, ZserioLexer.ID, "_id");
    checkToken(lexer, ZserioLexer.ID, "i1");
    checkToken(lexer, ZserioLexer.ID, "_1");
    checkToken(lexer, ZserioLexer.ID, "_");
    checkToken(lexer, ZserioLexer.ID, "__");
    checkToken(lexer, ZserioLexer.ID, "I");
    checkToken(lexer, ZserioLexer.ID, "_I");
    checkToken(lexer, ZserioLexer.ID, "s0mE_UgLy_id");

    checkEOF(lexer);
}
 

@Override
public JaybirdStatementModel parseStatement(String sql) throws ParseException {
    try {
        CharStream stream = CharStreams.fromString(sql);
        JaybirdSqlLexer lexer = new JaybirdSqlLexer(stream);
        CommonTokenStream tokenStream = new CommonTokenStream(lexer);
        
        JaybirdSqlParser parser = new JaybirdSqlParser(tokenStream);
        parser.removeErrorListeners();
        parser.statement();

        JaybirdStatementModel statementModel = parser.getStatementModel();
        if (statementModel.getStatementType() == JaybirdStatementModel.UNDETECTED_TYPE) {
            throw new ParseException("Unable to detect statement type or unsupported statement type");
        }
        if (statementModel.getTableName() == null) {
            throw new ParseException("Unable to parse query: no table name found");
        }
        return statementModel;
    } catch (RecognitionException e) {
        throw new ParseException("Unable to parse query", e);
    }
}
 

@VisibleForTesting
static AtlasDSLParser.QueryContext parse(String queryStr) throws AtlasBaseException {
    AtlasDSLParser.QueryContext ret;
    try {
        InputStream    stream           = new ByteArrayInputStream(queryStr.getBytes());
        AtlasDSLLexer  lexer            = new AtlasDSLLexer(CharStreams.fromStream(stream));
        Validator      validator        = new Validator();
        TokenStream    inputTokenStream = new CommonTokenStream(lexer);
        AtlasDSLParser parser           = new AtlasDSLParser(inputTokenStream);

        parser.removeErrorListeners();
        parser.addErrorListener(validator);

        // Validate the syntax of the query here
        ret = parser.query();
        if (!validator.isValid()) {
            LOG.error("Invalid DSL: {} Reason: {}", queryStr, validator.getErrorMsg());
            throw new AtlasBaseException(AtlasErrorCode.INVALID_DSL_QUERY, queryStr, validator.getErrorMsg());
        }

    } catch (IOException e) {
        throw new AtlasBaseException(e);
    }

    return ret;
}
 

final Absolute interpretAsInstanceIdentifier(final YangLiteralExpr expr) throws XPathExpressionException {
    final xpathLexer lexer = new xpathLexer(CharStreams.fromString(expr.getLiteral()));
    final instanceIdentifierParser parser = new instanceIdentifierParser(new CommonTokenStream(lexer));
    final CapturingErrorListener listener = new CapturingErrorListener();
    lexer.removeErrorListeners();
    lexer.addErrorListener(listener);
    parser.removeErrorListeners();
    parser.addErrorListener(listener);

    final InstanceIdentifierContext id = parser.instanceIdentifier();
    listener.reportError();

    final int length = id.getChildCount();
    final List<Step> steps = new ArrayList<>(length / 2);
    for (int i = 1; i < length; i += 2) {
        steps.add(parsePathArgument(getChild(id, PathArgumentContext.class, i)));
    }

    return YangLocationPath.absolute(steps);
}
 

private Package parsePackage(ZserioAstBuilder astBuilder,
        String inputFileFullName) throws Exception
{
    ZserioToolPrinter.printMessage("Parsing " + inputFileFullName);

    inputFileManager.registerFile(inputFileFullName);

    final CharStream inputStream = CharStreams.fromFileName(inputFileFullName, Charset.forName("UTF-8"));
    final ParseErrorListener parseErrorListener = new ParseErrorListener();
    final ZserioLexer lexer = new ZserioLexer(inputStream);
    lexer.removeErrorListeners();
    lexer.addErrorListener(parseErrorListener);
    final CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    final ZserioParser parser = new ZserioParser(tokenStream);
    parser.removeErrorListeners();
    parser.addErrorListener(parseErrorListener);

    final ParseTree tree = parser.packageDeclaration();

    final ZserioParseTreeChecker parseTreeChecker = new ZserioParseTreeChecker(inputFileManager);
    parseTreeChecker.visit(tree);

    final Package parsedPackage = (Package)astBuilder.visit(tree, tokenStream);

    return parsedPackage;
}
 

public String translateString(Translator trans, String markdown, String startRule) throws Exception {
	CharStream input = CharStreams.fromString(markdown);
	BookishLexer lexer = new BookishLexer(input);
	CommonTokenStream tokens = new CommonTokenStream(lexer);
	BookishParser parser = new BookishParser(tokens,null, 0);
	parser.removeErrorListeners();
	parser.addErrorListener(new ConsoleErrorListener() {
		@Override
		public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) {
			msg = "Parsing author string: "+msg;
			super.syntaxError(recognizer, offendingSymbol, line, charPositionInLine, msg, e);
		}
	});
	Method startMethod = BookishParser.class.getMethod(startRule, (Class[])null);
	ParseTree doctree = (ParseTree)startMethod.invoke(parser, (Object[])null);

	OutputModelObject omo = trans.visit(doctree); // get single chapter

	ModelConverter converter = new ModelConverter(trans.templates);
	ST outputST = converter.walk(omo);
	return outputST.render();
}
 

/**
 * 编译已经解析好的 DataQL
 * @param queryModel 解析之后的 DataQL 查询模型。
 * @param compilerArguments 编译参数，编译参数中包含了编译器需要的一些列变量。例如：编译变量，相当于在脚本中预先 执行 var xxx = null;
 *                    其意义在于在编译期就把脚本中尚未定义过的变量预先进行声明从而免去通过 ${...} 或类似方式查找变量。
 * @param importFinder import 导入用到的资源加载器。
 */
public static QIL queryCompiler(QueryModel queryModel, CompilerArguments compilerArguments, Finder importFinder) throws IOException {
    RootBlockSet rootBlockSet = null;
    if (queryModel instanceof RootBlockSet) {
        rootBlockSet = (RootBlockSet) queryModel;
    } else {
        rootBlockSet = (RootBlockSet) queryParser(CharStreams.fromString(queryModel.toQueryString()));
    }
    //
    compilerArguments = (compilerArguments == null) ? CompilerArguments.DEFAULT : compilerArguments;
    Set<String> compilerVar = compilerArguments.getCompilerVar();
    if (compilerVar == null) {
        compilerVar = Collections.emptySet();
    }
    //
    InstQueue queue = new InstQueue(compilerArguments);
    CompilerContext compilerContext = new CompilerContext(new CompilerEnvironment(importFinder));
    Map<String, Integer> compilerVarMap = new HashMap<>();
    compilerVar.forEach(var -> {
        int localIdx = compilerContext.push(var);
        compilerVarMap.put(var, localIdx);
    });
    compilerContext.findInstCompilerByInst(rootBlockSet).doCompiler(queue);
    Instruction[][] queueSet = queue.buildArrays();
    return new QIL(queueSet, compilerVarMap);
}
 

public static OnDemandQuery parseOnDemandQuery(String onDemandQueryString) throws SiddhiParserException {
    CharStream input = CharStreams.fromString(onDemandQueryString);
    SiddhiQLLexer lexer = new SiddhiQLLexer(input);
    lexer.removeErrorListeners();
    lexer.addErrorListener(SiddhiErrorListener.INSTANCE);

    CommonTokenStream tokens = new CommonTokenStream(lexer);
    SiddhiQLParser parser = new SiddhiQLParser(tokens);
    parser.removeErrorListeners();
    parser.addErrorListener(SiddhiErrorListener.INSTANCE);
    ParseTree tree = parser.store_query_final();

    SiddhiQLVisitor eval = new SiddhiQLBaseVisitorImpl();
    OnDemandQuery onDemandQuery = (OnDemandQuery) eval.visit(tree);
    return onDemandQuery;
}
 

/**
 * This method will tokenize the program and get all the elements that are
 * variable names, variables types, methods names. Depending on the library
 * it will imported libraries names too.
 * @param program
 * @param language
 * @param keepOrder If false, then the program's tokens will be filtered using multiple threads
 * @return 
 */
public static Set<String> tokenize(String program, String language, boolean keepOrder)
{
	CodePointCharStream code = CharStreams.fromString(program);
	Set<String> filteresTokens;
	switch(language)
	{
		case "c": 
			filteresTokens=CTokens(code, keepOrder);
			break;
		case "java":
			filteresTokens=JavaTokens(code, keepOrder);
			break;
		case "javascript":
			filteresTokens=JavaScriptTokens(code, keepOrder);
			break;
		case "php":
			filteresTokens=PhpTokens(code, keepOrder);
			break;
		default:
			System.err.println("Not supported language");
			filteresTokens=new HashSet<String>(0);
			break;
	}
	return filteresTokens;
}
 

@Test
public void blockComment()
{
    final CharStream input = CharStreams.fromString(
            "/* block comment */\n" +
            "/* block comment\n" +
            "   multiline */"
    );

    ZserioLexer lexer = new ZserioLexer(input);

    checkToken(lexer, ZserioLexer.BLOCK_COMMENT, Lexer.HIDDEN, "/* block comment */");
    checkToken(lexer, ZserioLexer.BLOCK_COMMENT, Lexer.HIDDEN, "/* block comment\n   multiline */");

    checkEOF(lexer);
}
 

@Test
public void docComment()
{
    final CharStream input = CharStreams.fromString(
            "/** doc comment */\n" +
            "/**\n" +
            " * doc comment\n" +
            " * multiline\n" +
            " */"
    );

    ZserioLexer lexer = new ZserioLexer(input);

    checkToken(lexer, ZserioLexer.DOC_COMMENT, ZserioLexer.DOC, "/** doc comment */");
    checkToken(lexer, ZserioLexer.DOC_COMMENT, ZserioLexer.DOC, "/**\n * doc comment\n * multiline\n */");

    checkEOF(lexer);
}
 

@SuppressWarnings("unchecked")
@Override
public void parse() throws IOException {
	/** If file already exist, skip it */
	Entity fileEntity = entityRepo.getEntity(fileFullPath);
	if (fileEntity!=null && fileEntity instanceof FileEntity) {
		return;
	}
	
       CharStream input = CharStreams.fromFileName(fileFullPath);
	Parser rubyParser = new Parser();
	StringReader in = new StringReader(input.toString());
	CompatVersion version = CompatVersion.RUBY2_3;
	ParserConfiguration config = new ParserConfiguration(0, version);
	try {
		Node node = rubyParser.parse("<code>", in, config);
		JRubyVisitor parser = new JRubyVisitor(fileFullPath, entityRepo, includesFileLocator,executor,inferer,parserCreator);
		node.accept(parser);
		fileEntity = entityRepo.getEntity(fileFullPath);
		((FileEntity)fileEntity).cacheAllExpressions();
		parser.done();
	}catch(Exception e) {
		System.err.println("parsing error in "+fileFullPath);
	}
}
 

public RegexLiterals(String regex) {
	CharStream stream = CharStreams.fromString(regex);
	PCRELexer lexer = new PCRELexer(stream);
	lexer.removeErrorListeners();
	lexer.addErrorListener(ErrorListener.INSTANCE);
	CommonTokenStream tokens = new CommonTokenStream(lexer);
	
	PCREParser parser = new PCREParser(tokens);
	parser.removeErrorListeners();
	parser.addErrorListener(ErrorListener.INSTANCE);
	
	OrLiterals orLiterals = (OrLiterals) new LiteralVisitor().visit(parser.parse());
	rows = orLiterals.flattern(true);
}
 

public static BookishParser.ContentContext parseString(String inputStr, String location) {
	CharStream input = CharStreams.fromString(inputStr);
	BookishLexer lexer = new BookishLexer(input);
	CommonTokenStream tokens = new CommonTokenStream(lexer);
	BookishParser parser = new BookishParser(tokens);

	lexer.removeErrorListeners();
	lexer.addErrorListener(getErrorListener(location));
	parser.removeErrorListeners();
	parser.addErrorListener(getErrorListener(location));

	BookishParser.ContentContext tree = parser.content();
	return tree;
}
 

private static CharStream toCharStream(String text) {
    CharStream inputStream;
    try {
        inputStream = CharStreams.fromReader(new StringReader(text));
    } catch (IOException e) {
        throw new IllegalStateException("Unexpected while reading input string", e);
    }
    return inputStream;
}