Java Code Examples for org.antlr.runtime.CommonTokenStream#getTokens()

/**
 * Apply a lexer to a source and check that the token stream produced
 * is as expected.
 * @param source the source code
 * @param expected the expected token stream
 */
public void lexAndCheck(final String source, final String expected) {
    try {
        CommonTokenStream ts = lex(source);
        ts.fill();
        StringBuilder sb = new StringBuilder();
        for (Object token : ts.getTokens()) {
            sb.append(toString((Token) token));
        }
        CobolStructureLexerImpl lexer = (CobolStructureLexerImpl) ts.getTokenSource();
        if (lexer.getErrorHandler().getErrorMessages().size() > 0) {
            throw new RecognizerException(
                    lexer.getErrorHandler().getErrorMessages().get(0));
        }
        assertEquals(expected, sb.toString());
    } catch (RecognizerException e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}
 

public static void debugTokenStream(CommonTokenStream tokens, PrintStream out) {
	List<? extends Token> list = tokens.getTokens();
	Iterator<? extends Token> iterator = list.iterator();
	int ii = -1;
	while (iterator.hasNext()) {
		++ii;
		Object object = iterator.next();
		out.println(object + "     (" + ii + ")");
	}
}
 

public void testLexerOutput() throws IOException
{
    String str = "~woof^2";
    CharStream input = new ANTLRInputStream(new ByteArrayInputStream(str.getBytes("UTF-8")));
    FTSLexer lexer = new FTSLexer(input);
    CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    for(CommonToken token : (List<CommonToken>)tokenStream.getTokens())
    {
        System.out.println(token.toString());
    }
    
}
 

@Test public void testLexerError_01() throws Exception {
      String model = "a /* incomplete comment *";
      InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
      lexer.setCharStream(new ANTLRStringStream(model));
      CommonTokenStream stream = new CommonTokenStream(lexer);
      @SuppressWarnings("unchecked")
List<CommonToken> tokens = stream.getTokens();
      assertEquals(tokens.toString(), 3, tokens.size());
      assertEquals("a", tokens.get(0).getText());
      assertEquals(" ", tokens.get(1).getText());
      assertEquals("/* incomplete comment *", tokens.get(2).getText());
      assertEquals(0, tokens.get(2).getType());
  }
 

@Test public void testLexerError_02() throws Exception {
	String model = "a 'incomplete string";
	InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
	lexer.setCharStream(new ANTLRStringStream(model));
	CommonTokenStream stream = new CommonTokenStream(lexer);
	@SuppressWarnings("unchecked")
	List<CommonToken> tokens = stream.getTokens();
       assertEquals(tokens.toString(), 3, tokens.size());
       assertEquals("a", tokens.get(0).getText());
       assertEquals(" ", tokens.get(1).getText());
       assertEquals("'incomplete string", tokens.get(2).getText());
       assertEquals(0, tokens.get(2).getType());
}
 

@Test public void testLexerError_03() throws Exception {
	String model = "a '\\ incomplete string with bad escape sequence";
	InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
	lexer.setCharStream(new ANTLRStringStream(model));
	CommonTokenStream stream = new CommonTokenStream(lexer);
	@SuppressWarnings("unchecked")
	List<CommonToken> tokens = stream.getTokens();
       assertEquals(tokens.toString(), 3, tokens.size());
       assertEquals("a", tokens.get(0).getText());
       assertEquals(" ", tokens.get(1).getText());
       assertEquals("'\\ incomplete string with bad escape sequence", tokens.get(2).getText());
       assertEquals(0, tokens.get(2).getType());
}
 

@Test public void testLexerError_04() throws Exception {
	String model = "a 'incomplete string with bad escape sequence \\";
	InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
	lexer.setCharStream(new ANTLRStringStream(model));
	CommonTokenStream stream = new CommonTokenStream(lexer);
	@SuppressWarnings("unchecked")
	List<CommonToken> tokens = stream.getTokens();
	assertEquals(tokens.toString(), 3, tokens.size());
	assertEquals("a", tokens.get(0).getText());
	assertEquals(" ", tokens.get(1).getText());
	assertEquals("'incomplete string with bad escape sequence \\", tokens.get(2).getText());
	assertEquals(0, tokens.get(2).getType());
}
 

@Test public void testLexerError_05() throws Exception {
	String model = "a 'incomplete string \\'";
	InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
	lexer.setCharStream(new ANTLRStringStream(model));
	CommonTokenStream stream = new CommonTokenStream(lexer);
	@SuppressWarnings("unchecked")
	List<CommonToken> tokens = stream.getTokens();
	assertEquals(tokens.toString(), 3, tokens.size());
	assertEquals("a", tokens.get(0).getText());
	assertEquals(" ", tokens.get(1).getText());
	assertEquals("'incomplete string \\'", tokens.get(2).getText());
	assertEquals(0, tokens.get(2).getType());
}
 

@Test public void testLexerError_06() throws Exception {
	String model = "a '";
	InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
	lexer.setCharStream(new ANTLRStringStream(model));
	CommonTokenStream stream = new CommonTokenStream(lexer);
	@SuppressWarnings("unchecked")
	List<CommonToken> tokens = stream.getTokens();
	assertEquals(tokens.toString(), 3, tokens.size());
	assertEquals("a", tokens.get(0).getText());
	assertEquals(" ", tokens.get(1).getText());
	assertEquals("'", tokens.get(2).getText());
	assertEquals(InternalSimpleExpressionsTestLanguageParser.RULE_ANY_OTHER, tokens.get(2).getType());
}
 

/**
 * Apply a lexer to a source and check that the token stream produced is as
 * expected.
 * 
 * @param source the source code
 * @param expected the expected token stream
 */
public void lexAndCheck(final String source, final String expected) {
    try {
        CommonTokenStream ts = lex(source);
        StringBuilder sb = new StringBuilder();
        for (Object token : ts.getTokens()) {
            sb.append(toString((Token) token));
        }
        assertEquals(expected, sb.toString());
    } catch (RecognizerException e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}