org.antlr.runtime.Lexer Java Examples

The following examples show how to use org.antlr.runtime.Lexer. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AbstractLexerTest.java    From xtext-eclipse with Eclipse Public License 2.0 6 votes vote down vote up
private String lexerResult(Lexer lexer, Map<Integer, String> tokenNames, CharSequence text) {
	lexer.setCharStream(new ANTLRStringStream(text.toString()));
	List<String> result = new ArrayList<>();
	while (true) {
		Token token = lexer.nextToken();
		if (token == Token.EOF_TOKEN) {
			return Joiner.on(System.lineSeparator()).join(result);
		}
		Object nameOrType = null;
		String tokenName = tokenNames.get(token.getType());
		if (tokenName != null) {
			nameOrType = tokenName;
		} else {
			nameOrType = Integer.valueOf(token.getType());
		}
		result.add(nameOrType + " \'" + token.getText() + "\'");
	}
}
 
Example #2
Source File: Css3LexerTest.java    From netbeans with Apache License 2.0 6 votes vote down vote up
public void testSimpleSupportsStatementTokens() throws Exception {
    String source = "@supports not (text-align: center) {} ";
    
    Lexer lexer = createLexer(source);
    
    assertANTLRToken("@supports", Css3Lexer.SUPPORTS_SYM, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
    assertANTLRToken("not" ,Css3Lexer.NOT, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
    assertANTLRToken("(" ,Css3Lexer.LPAREN, lexer.nextToken());
    assertANTLRToken("text-align" ,Css3Lexer.IDENT, lexer.nextToken());
    assertANTLRToken(":", Css3Lexer.COLON, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
    assertANTLRToken("center" ,Css3Lexer.IDENT, lexer.nextToken());
    assertANTLRToken(")" ,Css3Lexer.RPAREN, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
    assertANTLRToken("{" ,Css3Lexer.LBRACE, lexer.nextToken());
    assertANTLRToken("}" ,Css3Lexer.RBRACE, lexer.nextToken());
    
}
 
Example #3
Source File: Css3LexerTest.java    From netbeans with Apache License 2.0 5 votes vote down vote up
public void testLexingOfPercentageWithoutNumberPrefix() throws Exception {
    String source = "font: %/20 ";
    Lexer lexer = createLexer(source);
    assertANTLRToken("font" ,Css3Lexer.IDENT, lexer.nextToken());
    assertANTLRToken(":" ,Css3Lexer.COLON, lexer.nextToken());
    assertANTLRToken(" " ,Css3Lexer.WS, lexer.nextToken());
    assertANTLRToken("%" , Css3Lexer.PERCENTAGE_SYMBOL, lexer.nextToken());
    assertANTLRToken("/" ,Css3Lexer.SOLIDUS, lexer.nextToken());
    assertANTLRToken("20" , Css3Lexer.NUMBER, lexer.nextToken());
    assertANTLRToken(" " , Css3Lexer.WS, lexer.nextToken());
}
 
Example #4
Source File: Css3LexerTest.java    From netbeans with Apache License 2.0 5 votes vote down vote up
public void testLineComment() throws Exception {
    String source = "//comment\na";
    Lexer lexer = createLexer(source);
    assertANTLRToken("//comment" ,Css3Lexer.LINE_COMMENT, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.NL, lexer.nextToken());
    assertANTLRToken("a",Css3Lexer.IDENT, lexer.nextToken());
}
 
Example #5
Source File: Css3LexerTest.java    From netbeans with Apache License 2.0 5 votes vote down vote up
public void testSASS_Else() throws Exception {
    String source = "@else cau";
    Lexer lexer = createLexer(source);
    assertANTLRToken(null ,Css3Lexer.SASS_ELSE, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.IDENT, lexer.nextToken());
}
 
Example #6
Source File: Css3LexerTest.java    From netbeans with Apache License 2.0 5 votes vote down vote up
public void testExtendOnlySelector2() throws Exception {
    String source = "#context a%extreme {";
    Lexer lexer = createLexer(source);
    assertANTLRToken(null ,Css3Lexer.HASH, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.IDENT, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.SASS_EXTEND_ONLY_SELECTOR, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.LBRACE, lexer.nextToken());
}
 
Example #7
Source File: Css3LexerTest.java    From netbeans with Apache License 2.0 5 votes vote down vote up
public void testCaseInsensivityOfSomeAtTokens() throws Exception {
    String source = "@FONT-face @charset @CHARSET @charSeT ";

    Lexer lexer = createLexer(source);

    assertANTLRToken("@FONT-face" ,Css3Lexer.FONT_FACE_SYM, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
    assertANTLRToken("@charset" ,Css3Lexer.CHARSET_SYM, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
    assertANTLRToken("@CHARSET" ,Css3Lexer.CHARSET_SYM, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
    assertANTLRToken("@charSeT" ,Css3Lexer.CHARSET_SYM, lexer.nextToken());

}
 
Example #8
Source File: Block.java    From jssembly with MIT License 5 votes vote down vote up
public Block(Architecture arch) {
	this.architecture = arch;
	Lexer lex = null;
	Assembler assembler = null;
	
	switch (architecture) {
		// no parsing necessary if we're writing bytes to memory
		case raw:
			return;
		// arm architectures trickle down
		case armv7:
		case armv9:
			// lex = new armLexer(new ANTLRStringStream(program));
			// assembler = new armAssembler(new CommonTokenStream(lex));
			break;
		case x64:
			lex = new x64Lexer(new ANTLRStringStream(program));
			assembler = new x64Assembler(new CommonTokenStream(lex));
			break;
		case x86:
			break;
		default:
			break;
	}
	
	try {
		if (assembler != null) {
			assembler.start();
			this.instructions = assembler.getMachineCode();
		} else {
			throw new JssemblyException("Assembler not found for architecture: " + this.architecture.name());
		}
	} catch (RecognitionException e) {
		// TODO Auto-generated catch block
		e.printStackTrace();
	}
}
 
Example #9
Source File: XtextTokenStream.java    From xtext-core with Eclipse Public License 2.0 4 votes vote down vote up
public String getLexerErrorMessage(Token invalidToken) {
	if (tokenSource instanceof org.eclipse.xtext.parser.antlr.Lexer) {
		return ((org.eclipse.xtext.parser.antlr.Lexer) tokenSource).getErrorMessage(invalidToken);
	}
	return (invalidToken.getType() == Token.INVALID_TOKEN_TYPE) ? "Invalid token " + invalidToken.getText() : null;
}
 
Example #10
Source File: LexerProvider.java    From xtext-core with Eclipse Public License 2.0 4 votes vote down vote up
public static <T extends Lexer> LexerProvider<T> create(Class<T> clazz) {
	return new LexerProvider<T>(clazz);
}
 
Example #11
Source File: AbstractLexerTest.java    From xtext-eclipse with Eclipse Public License 2.0 4 votes vote down vote up
protected String lex(Lexer lexer, InputStream tokensStream, CharSequence text) throws Exception {
	InputStreamReader _inputStreamReader = new InputStreamReader(tokensStream);
	LinkedHashMap<Integer, String> tokenNames = tokenNames(_inputStreamReader);
	return lexerResult(lexer, tokenNames, text);
}
 
Example #12
Source File: Css3LexerTest.java    From netbeans with Apache License 2.0 4 votes vote down vote up
public void testIssue236649() throws Exception {
    String source = "url(http://fonts.googleapis.com/css?family=Josefin+Sans|Sigmar+One|Maven+Pro)";
    Lexer lexer = createLexer(source);
    assertANTLRToken("url(http://fonts.googleapis.com/css?family=Josefin+Sans|Sigmar+One|Maven+Pro)", Css3Lexer.URI, lexer.nextToken());
}
 
Example #13
Source File: Css3LexerTest.java    From netbeans with Apache License 2.0 4 votes vote down vote up
public void testURLWithAtSign() throws Exception {
    String source = "url([email protected])";
    Lexer lexer = createLexer(source);
    assertANTLRToken("url([email protected])" ,Css3Lexer.URI, lexer.nextToken());
}
 
Example #14
Source File: Css3LexerTest.java    From netbeans with Apache License 2.0 4 votes vote down vote up
public void testLESS_JS_STRING() throws Exception {
    String source = "`\"hello\".toUpperCase() + '!'`;";
    Lexer lexer = createLexer(source);
    assertANTLRToken("`\"hello\".toUpperCase() + '!'`" ,Css3Lexer.LESS_JS_STRING, lexer.nextToken());
    assertANTLRToken(";" ,Css3Lexer.SEMI, lexer.nextToken());
}
 
Example #15
Source File: CTFLexer.java    From tracecompass with Eclipse Public License 2.0 4 votes vote down vote up
public Lexer[] getDelegates() {
	return new Lexer[] {};
}
 
Example #16
Source File: FastSimpleGenericEdifactDirectXMLLexer.java    From hop with Apache License 2.0 4 votes vote down vote up
public Lexer[] getDelegates() {
  return new Lexer[] {};
}
 
Example #17
Source File: FastSimpleGenericEdifactDirectXMLLexer.java    From pentaho-kettle with Apache License 2.0 4 votes vote down vote up
public Lexer[] getDelegates() {
  return new Lexer[] {};
}
 
Example #18
Source File: Css3LexerTest.java    From netbeans with Apache License 2.0 4 votes vote down vote up
public void testLineCommentAtTheFileEnd() throws Exception {
    String source = "//comment";
    Lexer lexer = createLexer(source);
    assertANTLRToken("//comment" ,Css3Lexer.LINE_COMMENT, lexer.nextToken());
}
 
Example #19
Source File: Css3LexerTest.java    From netbeans with Apache License 2.0 4 votes vote down vote up
public void testExtendOnlySelector() throws Exception {
    String source = "body%my";
    Lexer lexer = createLexer(source);
    assertANTLRToken(null ,Css3Lexer.IDENT, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.SASS_EXTEND_ONLY_SELECTOR, lexer.nextToken());
}
 
Example #20
Source File: Css3LexerTest.java    From netbeans with Apache License 2.0 4 votes vote down vote up
public void testCPLineComment() throws Exception {
    String source = "//line comment\n";
    Lexer lexer = createLexer(source);
    assertANTLRToken(null ,Css3Lexer.LINE_COMMENT, lexer.nextToken());

}
 
Example #21
Source File: Css3LexerTest.java    From netbeans with Apache License 2.0 4 votes vote down vote up
public void testSassVar() throws Exception {
    String source = "$var ";
    Lexer lexer = createLexer(source);
    assertANTLRToken(null ,Css3Lexer.SASS_VAR, lexer.nextToken());
}
 
Example #22
Source File: Css3LexerTest.java    From netbeans with Apache License 2.0 4 votes vote down vote up
public void testLexingURLToken() throws Exception {
    String source = "url(http://fonts.googleapis.com/css?family=Syncopate) ";
    Lexer lexer = createLexer(source);
    assertANTLRToken(null ,Css3Lexer.URI, lexer.nextToken());
}
 
Example #23
Source File: OutlineXpectMethod.java    From n4js with Eclipse Public License 1.0 4 votes vote down vote up
@Override
public Lexer get() {
	return new InternalN4JSLexer(null);
}
 
Example #24
Source File: AbstractSmokeTester.java    From n4js with Eclipse Public License 1.0 4 votes vote down vote up
@Override
public Lexer get() {
	return new InternalN4JSLexer(null);
}
 
Example #25
Source File: DSLMapLexer.java    From kogito-runtimes with Apache License 2.0 4 votes vote down vote up
public Lexer[] getDelegates() {
	return new Lexer[] {};
}
 
Example #26
Source File: JavaLexer.java    From kogito-runtimes with Apache License 2.0 4 votes vote down vote up
public Lexer[] getDelegates() {
	return new Lexer[] {};
}
 
Example #27
Source File: Css3LexerTest.java    From netbeans with Apache License 2.0 3 votes vote down vote up
public void testRemUnit() throws Exception {
    String source = "10rad 20rem ";

    Lexer lexer = createLexer(source);

    assertANTLRToken(null ,Css3Lexer.ANGLE, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());

    assertANTLRToken(null ,Css3Lexer.REM, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());

}
 
Example #28
Source File: Css3LexerTest.java    From netbeans with Apache License 2.0 3 votes vote down vote up
public void testSupportsToken() throws Exception {
    String source = "@supports ";
    
    Lexer lexer = createLexer(source);
    
    assertANTLRToken("@supports", Css3Lexer.SUPPORTS_SYM, lexer.nextToken());
    
}
 
Example #29
Source File: AbstractLexerTest.java    From xtext-eclipse with Eclipse Public License 2.0 votes vote down vote up
protected abstract Lexer lexer();