org.antlr.runtime.CharStream Java Examples

public void testReplaceRangeThenInsertAtLeftEdge() throws Exception {
	Grammar g = new Grammar(
		"lexer grammar t;\n"+
		"A : 'a';\n" +
		"B : 'b';\n" +
		"C : 'c';\n");
	CharStream input = new ANTLRStringStream("abcccba");
	Interpreter lexEngine = new Interpreter(g, input);
	TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
	tokens.LT(1); // fill buffer
	tokens.replace(2, 4, "x");
	tokens.insertBefore(2, "y");
	String result = tokens.toString();
	String expecting = "abyxba";
	assertEquals(expecting, result);
}
 

public void testDisjointInserts() throws Exception {
	Grammar g = new Grammar(
		"lexer grammar t;\n"+
		"A : 'a';\n" +
		"B : 'b';\n" +
		"C : 'c';\n");
	CharStream input = new ANTLRStringStream("abc");
	Interpreter lexEngine = new Interpreter(g, input);
	TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
	tokens.LT(1); // fill buffer
	tokens.insertBefore(1, "x");
	tokens.insertBefore(2, "y");
	tokens.insertBefore(0, "z");
	String result = tokens.toString();
	String expecting = "zaxbyc";
	assertEquals(expecting, result);
}
 

/** Load template stream into this group. {@code unqualifiedFileName} is
 *  {@code "a.st"}. The {@code prefix} is path from group root to
 *  {@code unqualifiedFileName} like {@code "/subdir"} if file is in
 *  {@code /subdir/a.st}.
 */

public CompiledST loadTemplateFile(String prefix, String unqualifiedFileName, CharStream templateStream) {
    GroupLexer lexer = new GroupLexer(templateStream);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    GroupParser parser = new GroupParser(tokens);
    parser.group = this;
    lexer.group = this;
    try {
        parser.templateDef(prefix);
    }
    catch (RecognitionException re) {
        errMgr.groupSyntaxError(ErrorType.SYNTAX_ERROR, unqualifiedFileName, re, re.getMessage());
    }

    String templateName = Misc.getFileNameNoSuffix(unqualifiedFileName);
    if ( prefix !=null && prefix.length()>0 ) templateName = prefix+templateName;
    CompiledST impl = rawGetTemplate(templateName);
    impl.prefix = prefix;
    return impl;
}
 

@Test
public void test2() throws IOException {
    String query = "A = load 'input' using PigStorage(';');" +
                   "B = foreach ^ A generate string.concatsep( ';', $1, $2 );";
    CharStream input = new QueryParserStringStream( query, null );
    QueryLexer lexer = new QueryLexer( input );
    Token token;
    try {
        while( ( token = lexer.nextToken() ).getType() != Token.EOF ) {
            if( token.getChannel() == Token.HIDDEN_CHANNEL )
                continue;
            if( token.getText().equals( ";" ) ) {
                System.out.println( token.getText() );
            } else {
                System.out.print( token.getText() + "(" + token.getType() + ") " );
            }
        }
    } catch(Exception ex) {
        Assert.assertTrue( ex.getMessage().contains( "Unexpected character" ) );
        return;
    }
    Assert.fail( "Query should fail." );
}
 

/** Load template stream into this group. {@code unqualifiedFileName} is
 *  {@code "a.st"}. The {@code prefix} is path from group root to
 *  {@code unqualifiedFileName} like {@code "/subdir"} if file is in
 *  {@code /subdir/a.st}.
 */
public CompiledST loadTemplateFile(String prefix, String unqualifiedFileName, CharStream templateStream) {
	GroupLexer lexer = new GroupLexer(templateStream);
	CommonTokenStream tokens = new CommonTokenStream(lexer);
	GroupParser parser = new GroupParser(tokens);
	parser.group = this;
	lexer.group = this;
	try {
		parser.templateDef(prefix);
	}
	catch (RecognitionException re) {
		errMgr.groupSyntaxError(ErrorType.SYNTAX_ERROR,
								unqualifiedFileName,
								re, re.getMessage());
	}
	String templateName = Misc.getFileNameNoSuffix(unqualifiedFileName);
	if ( prefix!=null && prefix.length()>0 ) templateName = prefix+templateName;
	CompiledST impl = rawGetTemplate(templateName);
	impl.prefix = prefix;
	return impl;
}
 

public void testLeaveAloneDisjointInsert() throws Exception {
	Grammar g = new Grammar(
		"lexer grammar t;\n"+
		"A : 'a';\n" +
		"B : 'b';\n" +
		"C : 'c';\n");
	CharStream input = new ANTLRStringStream("abcc");
	Interpreter lexEngine = new Interpreter(g, input);
	TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
	tokens.LT(1); // fill buffer
	tokens.insertBefore(1, "x");
	tokens.replace(2, 3, "foo");
	String result = tokens.toString();
	String expecting = "axbfoo";
	assertEquals(expecting, result);
}
 

public void testCombineInsertOnLeftWithReplace() throws Exception {
	Grammar g = new Grammar(
		"lexer grammar t;\n"+
		"A : 'a';\n" +
		"B : 'b';\n" +
		"C : 'c';\n");
	CharStream input = new ANTLRStringStream("abc");
	Interpreter lexEngine = new Interpreter(g, input);
	TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
	tokens.LT(1); // fill buffer
	tokens.replace(0, 2, "foo");
	tokens.insertBefore(0, "z"); // combine with left edge of rewrite
	String result = tokens.toString();
	String expecting = "zfoo";
	assertEquals(expecting, result);
}
 

public void testInsertThenReplaceSameIndex() throws Exception {
	Grammar g = new Grammar(
		"lexer grammar t;\n"+
		"A : 'a';\n" +
		"B : 'b';\n" +
		"C : 'c';\n");
	CharStream input = new ANTLRStringStream("abc");
	Interpreter lexEngine = new Interpreter(g, input);
	TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
	tokens.LT(1); // fill buffer
	tokens.insertBefore(0, "0");
	tokens.replace(0, "x"); // supercedes insert at 0
	String result = tokens.toString();
	String expecting = "xbc";
	assertEquals(expecting, result);
}
 

public void testCombineInserts() throws Exception {
	Grammar g = new Grammar(
		"lexer grammar t;\n"+
		"A : 'a';\n" +
		"B : 'b';\n" +
		"C : 'c';\n");
	CharStream input = new ANTLRStringStream("abc");
	Interpreter lexEngine = new Interpreter(g, input);
	TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
	tokens.LT(1); // fill buffer
	tokens.insertBefore(0, "x");
	tokens.insertBefore(0, "y");
	String result = tokens.toString();
	String expecting = "yxabc";
	assertEquals(expecting, result);
}
 

public void testReplaceThenInsertAfterLastIndex() throws Exception {
	Grammar g = new Grammar(
		"lexer grammar t;\n"+
		"A : 'a';\n" +
		"B : 'b';\n" +
		"C : 'c';\n");
	CharStream input = new ANTLRStringStream("abc");
	Interpreter lexEngine = new Interpreter(g, input);
	TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
	tokens.LT(1); // fill buffer
	tokens.replace(2, "x");
	tokens.insertAfter(2, "y");
	String result = tokens.toString();
	String expecting = "abxy";
	assertEquals(expecting, result);
}
 

public void testOverlappingReplace3() throws Exception {
	Grammar g = new Grammar(
		"lexer grammar t;\n"+
		"A : 'a';\n" +
		"B : 'b';\n" +
		"C : 'c';\n");
	CharStream input = new ANTLRStringStream("abcc");
	Interpreter lexEngine = new Interpreter(g, input);
	TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
	tokens.LT(1); // fill buffer
	tokens.replace(1, 2, "foo");
	tokens.replace(0, 2, "bar"); // wipes prior nested replace
	String result = tokens.toString();
	String expecting = "barc";
	assertEquals(expecting, result);
}
 

public void testInsertThenReplaceLastIndex() throws Exception {
	Grammar g = new Grammar(
		"lexer grammar t;\n"+
		"A : 'a';\n" +
		"B : 'b';\n" +
		"C : 'c';\n");
	CharStream input = new ANTLRStringStream("abc");
	Interpreter lexEngine = new Interpreter(g, input);
	TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
	tokens.LT(1); // fill buffer
	tokens.insertBefore(2, "y");
	tokens.replace(2, "x");
	String result = tokens.toString();
	String expecting = "abx";
	assertEquals(expecting, result);
}
 

/**
 * Parses a breakpoint condition string.
 *
 * @param conditionString The condition string to parse.
 *
 * @return The parsed breakpoint condition tree.
 *
 * @throws RecognitionException Thrown if the condition string could not be parsed.
 * @throws MaybeNullException Thrown if an empty condition string is passed to the function.
 */
public static ConditionNode parse(final String conditionString) throws RecognitionException,
    MaybeNullException {
  if (conditionString.trim().isEmpty()) {
    throw new MaybeNullException();
  }
  final CharStream charStream = new ANTLRStringStream(conditionString);
  final ConditionLexer lexer = new ConditionLexer(charStream);
  final CommonTokenStream tokens = new CommonTokenStream();
  tokens.setTokenSource(lexer);
  final ConditionParser parser = new ConditionParser(tokens);
  parser.setTreeAdaptor(adaptor);
  try {
    final ConditionParser.prog_return parserResult = parser.prog();
    final CommonTree ast = (CommonTree) parserResult.getTree();

    if (parser.input.index() < parser.input.size()) {
      throw new RecognitionException();
    }
    return convert(ast);
  } catch (final IllegalArgumentException e) {
    throw new RecognitionException();
  }
}
 

public void testReplaceSingleMiddleThenOverlappingSuperset() throws Exception {
	Grammar g = new Grammar(
		"lexer grammar t;\n"+
		"A : 'a';\n" +
		"B : 'b';\n" +
		"C : 'c';\n");
	CharStream input = new ANTLRStringStream("abcba");
	Interpreter lexEngine = new Interpreter(g, input);
	TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
	tokens.LT(1); // fill buffer
	tokens.replace(2, 2, "xyz");
	tokens.replace(0, 3, "foo");
	String result = tokens.toString();
	String expecting = "fooa";
	assertEquals(expecting, result);
}
 

public void testLeaveAloneDisjointInsert2() throws Exception {
	Grammar g = new Grammar(
		"lexer grammar t;\n"+
		"A : 'a';\n" +
		"B : 'b';\n" +
		"C : 'c';\n");
	CharStream input = new ANTLRStringStream("abcc");
	Interpreter lexEngine = new Interpreter(g, input);
	TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
	tokens.LT(1); // fill buffer
	tokens.replace(2, 3, "foo");
	tokens.insertBefore(1, "x");
	String result = tokens.toString();
	String expecting = "axbfoo";
	assertEquals(expecting, result);
}
 

@Test
public void testRemoveErrorListener() throws Exception
{
    SyntaxErrorCounter firstCounter = new SyntaxErrorCounter();
    SyntaxErrorCounter secondCounter = new SyntaxErrorCounter();

    CharStream stream = new ANTLRStringStream("SELECT * FORM test;");
    CqlLexer lexer = new CqlLexer(stream);

    TokenStream tokenStream = new CommonTokenStream(lexer);
    CqlParser parser = new CqlParser(tokenStream);
    parser.addErrorListener(firstCounter);
    parser.addErrorListener(secondCounter);
    parser.removeErrorListener(secondCounter);

    parser.query();

    assertTrue(firstCounter.count > 0);
    assertEquals(0, secondCounter.count);
}
 

@Override
public CompiledST loadTemplateFile(String prefix, String unqualifiedFileName, CharStream templateStream) {
    String template = templateStream.substring(0, templateStream.size()- 1);
    String templateName = Misc.getFileNameNoSuffix(unqualifiedFileName);
    String fullyQualifiedTemplateName = prefix+templateName;
    CompiledST impl = new Compiler(this).compile(fullyQualifiedTemplateName, template);
    CommonToken nameT = new CommonToken(STLexer.SEMI); // Seems like a hack, best I could come up with.
    nameT.setInputStream(templateStream);
    rawDefineTemplate(fullyQualifiedTemplateName, impl, nameT);
    impl.defineImplicitlyDefinedTemplates(this);
    return impl;
}
 

public static Query parseSparqlString(String sparql,
		Map<String, String> rdfStorePrefixes) {
	CharStream input = new ANTLRStringStream(sparql);
	Query q = null;
	try {
		q = parseSparql(input, rdfStorePrefixes);
	} catch (SPARQLsyntaxError se) {
		se.setSQL(sparql);
		throw se;
	}

	return q;
}
 

private static Query getQueryExt(CharStream sparqlFile)
		throws RecognitionException {
	// System.out.println("Parsing: "+sparqlFile+"\n");
	IbmSparqlExtLexer lex = new IbmSparqlExtLexer(sparqlFile);
	CommonTokenStream tokens = new CommonTokenStream(lex);
	IbmSparqlExtParser parser = new IbmSparqlExtParser(tokens);

	parser.setTreeAdaptor(new CommonTreeAdaptor() {
		@Override
		public Object create(Token t) {
			return new XTree(t);
		}
	});

	IbmSparqlExtParser.queryUnit_return ret = parser.queryUnit();
	CommonTree ast = (CommonTree) ret.getTree();

	//
	System.out.println(ast.toStringTree());
	// SparqlParserUtilities.dump_tree(ast, tokens, 0);

	BufferedTreeNodeStream nodes = new BufferedTreeNodeStream(ast);
	nodes.setTokenStream(tokens);
	IbmSparqlExtAstWalker walker = new IbmSparqlExtAstWalker(nodes);
	QueryExt query = walker.queryUnit();
	return query;
}
 

public STLexer(ErrorManager errMgr,
			   CharStream input,
			   Token templateToken,
			   char delimiterStartChar,
			   char delimiterStopChar)
{
	this.errMgr = errMgr;
	this.input = input;
	c = (char)input.LA(1); // prime lookahead
	this.templateToken = templateToken;
	this.delimiterStartChar = delimiterStartChar;
	this.delimiterStopChar = delimiterStopChar;
}
 

public void compileTimeError(ErrorType error, Token templateToken, Token t) {
    CharStream input = t.getInputStream();
    String srcName = null;
    if ( input!=null ) {
        srcName = input.getSourceName();
        if ( srcName!=null ) srcName = Misc.getFileName(srcName);
    }
    listener.compileTimeError(new STCompiletimeMessage(error, srcName, templateToken, t, null, t.getText()));
}
 

@Override
public CompiledST loadTemplateFile(String prefix, String unqualifiedFileName, CharStream templateStream) {
    String template = templateStream.substring(0, templateStream.size()- 1);
    String templateName = Misc.getFileNameNoSuffix(unqualifiedFileName);
    String fullyQualifiedTemplateName = prefix+templateName;
    CompiledST impl = new Compiler(this).compile(fullyQualifiedTemplateName, template);
    CommonToken nameT = new CommonToken(STLexer.SEMI); // Seems like a hack, best I could come up with.
    nameT.setInputStream(templateStream);
    rawDefineTemplate(fullyQualifiedTemplateName, impl, nameT);
    impl.defineImplicitlyDefinedTemplates(this);
    return impl;
}
 

public void testLexer() throws IOException, RecognitionException
{
    ClassLoader cl = FTSTest.class.getClassLoader();
    InputStream modelStream = cl.getResourceAsStream("org/alfresco/repo/search/impl/parsers/fts_test.gunit");

    CharStream input = new ANTLRInputStream(modelStream);

    gUnitExecutor executer = new gUnitExecutor(parse(input), "FTS");

    String result = executer.execTest();
    System.out.print(executer.execTest()); // unit test result

    assertEquals("Failures: " + result, 0, executer.failures.size());
    assertEquals("Invalids " + result, 0, executer.invalids.size());
}
 

@Override
public CompiledST loadTemplateFile(String prefix, String unqualifiedFileName, CharStream templateStream) {
    String template = templateStream.substring(0, templateStream.size()- 1);
    String templateName = Misc.getFileNameNoSuffix(unqualifiedFileName);
    String fullyQualifiedTemplateName = prefix+templateName;
    CompiledST impl = new Compiler(this).compile(fullyQualifiedTemplateName, template);
    CommonToken nameT = new CommonToken(STLexer.SEMI); // Seems like a hack, best I could come up with.
    nameT.setInputStream(templateStream);
    rawDefineTemplate(fullyQualifiedTemplateName, impl, nameT);
    impl.defineImplicitlyDefinedTemplates(this);
    return impl;
}
 

public STLexer(ErrorManager errMgr, CharStream input, Token templateToken, char delimiterStartChar, char delimiterStopChar) {
    this.errMgr = errMgr;
    this.input = input;
    c = (char)input.LA(1); // prime lookahead
    this.templateToken = templateToken;
    this.delimiterStartChar = delimiterStartChar;
    this.delimiterStopChar = delimiterStopChar;
}
 

@SuppressWarnings("unused")
static public Constraint buildFTS(String ftsExpression, QueryModelFactory factory, FunctionEvaluationContext functionEvaluationContext, Selector selector,
        Map<String, Column> columnMap, String defaultField)
{
    // TODO: Decode sql escape for '' should do in CMIS layer

    // parse templates to trees ...

    CMIS_FTSParser parser = null;
    try
    {
        CharStream cs = new ANTLRStringStream(ftsExpression);
        CMIS_FTSLexer lexer = new CMIS_FTSLexer(cs);
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        parser = new CMIS_FTSParser(tokens);
        CommonTree ftsNode = (CommonTree) parser.cmisFtsQuery().getTree();
        return buildFTSConnective(ftsNode, factory, functionEvaluationContext, selector, columnMap, defaultField);
    }
    catch (RecognitionException e)
    {
        if (parser != null)
        {
            String[] tokenNames = parser.getTokenNames();
            String hdr = parser.getErrorHeader(e);
            String msg = parser.getErrorMessage(e, tokenNames);
            throw new FTSQueryException(hdr + "\n" + msg, e);
        }
        return null;
    }

}
 

static Query getQuery(CharStream sparqlFile)
		throws RecognitionException {
	CommonTree ast = getParseTree(sparqlFile);
	// System.out.println(ast.toStringTree());
	BufferedTreeNodeStream nodes = new BufferedTreeNodeStream(ast);
	// nodes.setTokenStream(tokens);
	// IbmSparqlAstRewriter astRewriter = new IbmSparqlAstRewriter(nodes);
	// CommonTree ast2 = (CommonTree)astRewriter.downup(ast, false);
	// System.out.println(ast2.toStringTree());
	// BufferedTreeNodeStream nodes2 = new BufferedTreeNodeStream(ast2);
	// nodes2.setTokenStream(tokens);
	IbmSparqlAstWalker walker = new IbmSparqlAstWalker(nodes);
	Query q = walker.queryUnit();
	return q;
}
 

public STLexer(ErrorManager errMgr, CharStream input, Token templateToken, char delimiterStartChar, char delimiterStopChar) {
    this.errMgr = errMgr;
    this.input = input;
    c = (char)input.LA(1); // prime lookahead
    this.templateToken = templateToken;
    this.delimiterStartChar = delimiterStartChar;
    this.delimiterStopChar = delimiterStopChar;
}
 

public void testInsertBeforeIndex0() throws Exception {
	Grammar g = new Grammar(
		"lexer grammar t;\n"+
		"A : 'a';\n" +
		"B : 'b';\n" +
		"C : 'c';\n");
	CharStream input = new ANTLRStringStream("abc");
	Interpreter lexEngine = new Interpreter(g, input);
	TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
	tokens.LT(1); // fill buffer
	tokens.insertBefore(0, "0");
	String result = tokens.toString();
	String expecting = "0abc";
	assertEquals(expecting, result);
}
 

private int parse(String query) throws IOException, RecognitionException  {
    CharStream input = new QueryParserStringStream( query, null );
    QueryLexer lexer = new QueryLexer(input);
    CommonTokenStream tokens = new  CommonTokenStream(lexer);

    QueryParser parser = QueryParserUtils.createParser(tokens);
    QueryParser.query_return result = parser.query();

    Tree ast = (Tree)result.getTree();

    System.out.println( ast.toStringTree() );
    TreePrinter.printTree((CommonTree) ast, 0);
    Assert.assertEquals(0, lexer.getNumberOfSyntaxErrors());
    return parser.getNumberOfSyntaxErrors();
}