Java Code Examples for org.antlr.v4.runtime.CharStreams#fromStream()

@VisibleForTesting
static AtlasDSLParser.QueryContext parse(String queryStr) throws AtlasBaseException {
    AtlasDSLParser.QueryContext ret;
    try {
        InputStream    stream           = new ByteArrayInputStream(queryStr.getBytes());
        AtlasDSLLexer  lexer            = new AtlasDSLLexer(CharStreams.fromStream(stream));
        Validator      validator        = new Validator();
        TokenStream    inputTokenStream = new CommonTokenStream(lexer);
        AtlasDSLParser parser           = new AtlasDSLParser(inputTokenStream);

        parser.removeErrorListeners();
        parser.addErrorListener(validator);

        // Validate the syntax of the query here
        ret = parser.query();
        if (!validator.isValid()) {
            LOG.error("Invalid DSL: {} Reason: {}", queryStr, validator.getErrorMsg());
            throw new AtlasBaseException(AtlasErrorCode.INVALID_DSL_QUERY, queryStr, validator.getErrorMsg());
        }

    } catch (IOException e) {
        throw new AtlasBaseException(e);
    }

    return ret;
}
 

private FeatureParser(Feature feature, InputStream is) {
    this.feature = feature;
    CharStream stream;
    try {
        stream = CharStreams.fromStream(is, StandardCharsets.UTF_8);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    KarateLexer lexer = new KarateLexer(stream);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    KarateParser parser = new KarateParser(tokens);
    parser.addErrorListener(errorListener);
    RuleContext tree = parser.feature();
    if (logger.isTraceEnabled()) {
        logger.debug(tree.toStringTree(parser));
    }
    ParseTreeWalker walker = new ParseTreeWalker();
    walker.walk(this, tree);
    if (errorListener.isFail()) {
        String errorMessage = errorListener.getMessage();
        logger.error("not a valid feature file: {} - {}", feature.getResource().getRelativePath(), errorMessage);
        throw new RuntimeException(errorMessage);
    }
}
 

@Override
public ArrayList<String> tokenize(String s) throws Exception {
    tokens = new ArrayList<>();
    InputStream stream = new ByteArrayInputStream(s.getBytes(StandardCharsets.UTF_8));
    JavaLexer lexer = new JavaLexer(CharStreams.fromStream(stream, StandardCharsets.UTF_8));
    List<? extends Token> tokenList = lexer.getAllTokens();
    for (Token token : tokenList) {
        String symbolicName = JavaLexer.VOCABULARY.getSymbolicName(token.getType());
        // normalize the token except white space (skip)
        if (!symbolicName.equals("WS")
                && !symbolicName.equals("COMMENT")
                && !symbolicName.equals("LINE_COMMENT")) {
            tokens.add(normalizer.normalizeAToken(token.getText().trim(), symbolicName));
        }
    }

    return tokens;
}
 

/** {@inheritDoc} */
@Override public XGModelComposition parse(byte[] mdl) {
    try (ByteArrayInputStream bais = new ByteArrayInputStream(mdl)) {
        CharStream cStream = CharStreams.fromStream(bais);
        XGBoostModelLexer lexer = new XGBoostModelLexer(cStream);
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        XGBoostModelParser parser = new XGBoostModelParser(tokens);

        XGModelVisitor visitor = new XGModelVisitor();

        return visitor.visit(parser.xgModel());
    }
    catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 

private static StatementContext parseYangSource(final SourceIdentifier source, final InputStream stream)
        throws IOException, YangSyntaxErrorException {
    final YangStatementLexer lexer = new YangStatementLexer(CharStreams.fromStream(stream));
    final YangStatementParser parser = new YangStatementParser(new CommonTokenStream(lexer));
    // disconnect from console error output
    lexer.removeErrorListeners();
    parser.removeErrorListeners();

    final YangErrorListener errorListener = new YangErrorListener(source);
    lexer.addErrorListener(errorListener);
    parser.addErrorListener(errorListener);

    final StatementContext result = parser.statement();
    errorListener.validate();

    // Walk the resulting tree and replace each children with an immutable list, lowering memory requirements
    // and making sure the resulting tree will not get accidentally modified. An alternative would be to use
    // org.antlr.v4.runtime.Parser.TrimToSizeListener, but that does not make the tree immutable.
    ParseTreeWalker.DEFAULT.walk(MAKE_IMMUTABLE_LISTENER, result);

    return result;
}
 

public static void generateTreeFile(final File vb6InputFile, final File outputDirectory) throws IOException {
	final File outputFile = new File(outputDirectory + "/" + vb6InputFile.getName() + TREE_EXTENSION);

	final boolean createdNewFile = outputFile.createNewFile();

	if (createdNewFile || RENEW_TREE_FILE) {
		LOG.info("Creating tree file {}.", outputFile);

		final InputStream inputStream = new FileInputStream(vb6InputFile);
		final VisualBasic6Lexer lexer = new VisualBasic6Lexer(CharStreams.fromStream(inputStream));
		final CommonTokenStream tokens = new CommonTokenStream(lexer);
		final VisualBasic6Parser parser = new VisualBasic6Parser(tokens);
		final StartRuleContext startRule = parser.startRule();
		final String inputFileTree = TreeUtils.toStringTree(startRule, parser);

		final PrintWriter pWriter = new PrintWriter(new FileWriter(outputFile));

		pWriter.write(inputFileTree);
		pWriter.flush();
		pWriter.close();
	}
}
 

@SuppressWarnings("WeakerAccess")
public static XMLParser.DocumentContext parseRaw( InputStream inputStream )
{
  try
  {
    XMLLexer lexer = new XMLLexer( CharStreams.fromStream( inputStream ) );
    CommonTokenStream tokens = new CommonTokenStream( lexer );
    XMLParser parser = new XMLParser( tokens );
    return parser.document();
  }
  catch( Exception e )
  {
    throw new RuntimeException( e );
  }
}
 

@Test
public void checkLexer() throws IOException {
    File f = new File("resources/lexer_test/233.java");
    String code = FileUtils.readFileToString(f);
    InputStream stream = new ByteArrayInputStream(code.getBytes(StandardCharsets.UTF_8));
    JavaLexer lexer = new JavaLexer(CharStreams.fromStream(stream, StandardCharsets.UTF_8));
    List<? extends Token> tokenList;
    tokenList = lexer.getAllTokens();
    for (Token token: tokenList) {
        String symbolicName = JavaLexer.VOCABULARY.getSymbolicName(token.getType());
        System.out.println(token.getText().trim() + "," + symbolicName);
    }
}
 

private Source_textContext parseTextContext(File file) throws IOException {
	InputStream inputStream = new FileInputStream(file);
	Lexer lexer = new Verilog2001Lexer(CharStreams.fromStream(inputStream));
	TokenStream tokenStream = new CommonTokenStream(lexer);
	Verilog2001Parser parser = new Verilog2001Parser(tokenStream);
	Source_textContext parsedVerilog = parser.source_text();
	inputStream.close();
	return parsedVerilog;
}
 

public static AntlrContext fromStreams(InputFile inputFile, InputStream file, InputStream linesStream,
		Charset charset) throws IOException {
	final SourceLinesProvider linesProvider = new SourceLinesProvider();
	final CharStream charStream = new CaseChangingCharStream(CharStreams.fromStream(file, charset), true);
	final TSqlLexer lexer = new TSqlLexer(charStream);
	lexer.removeErrorListeners();
	final CommonTokenStream stream = new CommonTokenStream(lexer);
	stream.fill();
	final TSqlParser parser = new TSqlParser(stream);
	parser.removeErrorListeners();
	final ParseTree root = parser.tsql_file();
	final SourceLine[] lines = linesProvider.getLines(linesStream, charset);
	return new AntlrContext(inputFile, stream, root, lines);
}
 

@Nullable
@Override
public CharStream read(String name) {
    try {
        InputStream resource = readResource(name);
        if (resource != null) {
            return CharStreams.fromStream(resource);
        }
    } catch (Exception e) {
        LOGGER.error("Could not read {}", name, e);
    }
    return null;
}
 

protected void doParse(final File inputFile, final VbParserParams params) throws IOException {
	final Charset charset = params.getCharset();

	LOG.info("Parsing file {} with charset {}.", inputFile.getName(), charset);

	final InputStream inputStream = new FileInputStream(inputFile);
	final VisualBasic6Lexer lexer = new VisualBasic6Lexer(CharStreams.fromStream(inputStream, charset));

	if (!params.getIgnoreSyntaxErrors()) {
		lexer.removeErrorListeners();
		lexer.addErrorListener(new ThrowingErrorListener());
	}

	final CommonTokenStream tokens = new CommonTokenStream(lexer);
	final VisualBasic6Parser parser = new VisualBasic6Parser(tokens);

	if (!params.getIgnoreSyntaxErrors()) {
		parser.removeErrorListeners();
		parser.addErrorListener(new ThrowingErrorListener());
	}

	final StartRuleContext startRule = parser.startRule();
	final File treeFile = new File(inputFile.getAbsolutePath() + TREE_SUFFIX);

	if (treeFile.exists()) {
		doCompareParseTree(treeFile, startRule, parser);
	}
}
 

/**
 * Compile it to Java and save.
 *
 * @throws IOException If fails
 */
public void compile() throws IOException {
    final String[] lines = new TextOf(this.input).asString().split("\n");
    final ANTLRErrorListener errors = new BaseErrorListener() {
        // @checkstyle ParameterNumberCheck (10 lines)
        @Override
        public void syntaxError(final Recognizer<?, ?> recognizer,
            final Object symbol, final int line,
            final int position, final String msg,
            final RecognitionException error) {
            throw new CompileException(
                String.format(
                    "[%d:%d] %s: \"%s\"",
                    line, position, msg, lines[line - 1]
                ),
                error
            );
        }
    };
    final org.eolang.compiler.ProgramLexer lexer =
        new org.eolang.compiler.ProgramLexer(
            CharStreams.fromStream(this.input.stream())
        );
    lexer.removeErrorListeners();
    lexer.addErrorListener(errors);
    final org.eolang.compiler.ProgramParser parser =
        new org.eolang.compiler.ProgramParser(
            new CommonTokenStream(lexer)
        );
    parser.removeErrorListeners();
    parser.addErrorListener(errors);
    final Tree tree = parser.program().ret;
    new IoCheckedScalar<>(
        new And(
            path -> {
                new LengthOf(
                    new TeeInput(
                        path.getValue(),
                        this.target.apply(path.getKey())
                    )
                ).value();
            },
            tree.java().entrySet()
        )
    ).value();
}
 

public ParseProgram fromStdin() throws IOException {
  CharStream input = CharStreams.fromStream(System.in);
  return fromCharStream(input);
}
 

/**
 * Parse a TOML input stream.
 *
 * @param is The input stream to read the TOML document from.
 * @param version The version level to parse at.
 * @return The parse result.
 * @throws IOException If an IO error occurs.
 */
public static TomlParseResult parse(InputStream is, TomlVersion version) throws IOException {
  CharStream stream = CharStreams.fromStream(is);
  return Parser.parse(stream, version.canonical);
}
 

/**
 * Parse a TOML input stream.
 *
 * @param is The input stream to read the TOML document from.
 * @param version The version level to parse at.
 * @return The parse result.
 * @throws IOException If an IO error occurs.
 */
public static TomlParseResult parse(InputStream is, TomlVersion version) throws IOException {
  CharStream stream = CharStreams.fromStream(is);
  return Parser.parse(stream, version.canonical);
}