org.antlr.runtime.CommonToken Java Examples

The following examples show how to use org.antlr.runtime.CommonToken. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestTrees.java    From jFuzzyLogic with GNU Lesser General Public License v3.0 6 votes vote down vote up
public void testDupTree() throws Exception {
	// ^(101 ^(102 103 ^(106 107) ) 104 105)
	CommonTree r0 = new CommonTree(new CommonToken(101));
	CommonTree r1 = new CommonTree(new CommonToken(102));
	r0.addChild(r1);
	r1.addChild(new CommonTree(new CommonToken(103)));
	Tree r2 = new CommonTree(new CommonToken(106));
	r2.addChild(new CommonTree(new CommonToken(107)));
	r1.addChild(r2);
	r0.addChild(new CommonTree(new CommonToken(104)));
	r0.addChild(new CommonTree(new CommonToken(105)));

	CommonTree dup = (CommonTree)(new CommonTreeAdaptor()).dupTree(r0);

	assertNull(dup.parent);
	assertEquals(-1, dup.childIndex);
	dup.sanityCheckParentAndChildIndexes();
}
 
Example #2
Source File: TestTrees.java    From jFuzzyLogic with GNU Lesser General Public License v3.0 6 votes vote down vote up
public void testList2() throws Exception {
	// Add child ^(nil 101 102 103) to root 5
	// should pull 101 102 103 directly to become 5's child list
	CommonTree root = new CommonTree(new CommonToken(5));

	// child tree
	CommonTree r0 = new CommonTree((Token)null);
	CommonTree c0, c1, c2;
	r0.addChild(c0=new CommonTree(new CommonToken(101)));
	r0.addChild(c1=new CommonTree(new CommonToken(102)));
	r0.addChild(c2=new CommonTree(new CommonToken(103)));

	root.addChild(r0);

	assertNull(root.parent);
	assertEquals(-1, root.childIndex);
	// check children of root all point at root
	assertEquals(root, c0.parent);
	assertEquals(0, c0.childIndex);
	assertEquals(root, c0.parent);
	assertEquals(1, c1.childIndex);
	assertEquals(root, c0.parent);
	assertEquals(2, c2.childIndex);
}
 
Example #3
Source File: InputPanel.java    From intellij-plugin-v4 with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
public void setCursorToGrammarElement(Project project, PreviewState previewState, int offset) {
	Token tokenUnderCursor = ParsingUtils.getTokenUnderCursor(previewState, offset);
	if ( tokenUnderCursor==null ) {
		return;
	}

	PreviewParser parser = (PreviewParser) previewState.parsingResult.parser;
	Integer atnState = parser.inputTokenToStateMap.get(tokenUnderCursor);
	if ( atnState==null ) { // likely an error token
		//LOG.error("no ATN state for input token " + tokenUnderCursor);
		return;
	}

	Interval region = previewState.g.getStateToGrammarRegion(atnState);
	CommonToken token =
		(CommonToken) previewState.g.tokenStream.get(region.a);
	jumpToGrammarPosition(project, token.getStartIndex());
}
 
Example #4
Source File: STGroup.java    From codebuff with BSD 2-Clause "Simplified" License 6 votes vote down vote up
/** for testing */

    public CompiledST defineTemplate(String templateName, String template) {
        if ( templateName.charAt(0) !='/' ) templateName = "/"+templateName;
        try {
            CompiledST impl = defineTemplate(templateName,
                                             new CommonToken(GroupParser.ID, templateName),
                                             null,
                                             template,
                                             null);
            return impl;
        }
        catch (STException se) {
            // we have reported the error; the exception just blasts us
            // out of parsing this template
        }
        return null;
    }
 
Example #5
Source File: InternalHighlightingParser.java    From n4js with Eclipse Public License 1.0 6 votes vote down vote up
@Override
protected void announce(Token start, Token stop, AbstractElement element) {
	if (start != null && start != Token.EOF_TOKEN) {
		if (start == stop) {
			announce(start, element);
		} else {
			CommonToken castedStart = (CommonToken) start;
			if (stop == null) { // possible error condition
				if (start.getTokenIndex() == state.lastErrorIndex) {
					return;
				}
			}
			CommonToken castedEnd = (CommonToken) stop;
			Integer newType = rewriter.rewrite(castedStart, element);
			if (newType != null && castedEnd != null && castedEnd != Token.EOF_TOKEN) {
				LazyTokenStream castedInput = (LazyTokenStream) this.input;
				for (int i = castedStart.getTokenIndex() + 1; i < castedEnd.getTokenIndex(); i++) {
					Token token = castedInput.get(i);
					if (token.getChannel() != Token.HIDDEN_CHANNEL)
						token.setType(newType);
				}
				castedEnd.setType(newType);
			}
		}
	}
}
 
Example #6
Source File: InputPanel.java    From intellij-plugin-v4 with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
public void setCursorToGrammarRule(Project project, PreviewState previewState, int offset) {
	Token tokenUnderCursor = ParsingUtils.getTokenUnderCursor(previewState, offset);
	if ( tokenUnderCursor==null ) {
		return;
	}

	ParseTree tree = previewState.parsingResult.tree;
	TerminalNode nodeWithToken =
		(TerminalNode) ParsingUtils.getParseTreeNodeWithToken(tree, tokenUnderCursor);
	if ( nodeWithToken==null ) {
		// hidden token
		return;
	}

	ParserRuleContext parent = (ParserRuleContext) nodeWithToken.getParent();
	int ruleIndex = parent.getRuleIndex();
	Rule rule = previewState.g.getRule(ruleIndex);
	GrammarAST ruleNameNode = (GrammarAST) rule.ast.getChild(0);
	int start = ((CommonToken) ruleNameNode.getToken()).getStartIndex();

	jumpToGrammarPosition(project, start);
}
 
Example #7
Source File: STGroup.java    From codebuff with BSD 2-Clause "Simplified" License 6 votes vote down vote up
/** for testing */

    public CompiledST defineTemplate(String templateName, String template) {
        if ( templateName.charAt(0) !='/' ) templateName = "/"+templateName;
        try {
            CompiledST impl = defineTemplate(templateName,
                                             new CommonToken(GroupParser.ID, templateName),
                                             null,
                                             template,
                                             null);
            return impl;
        }
        catch (STException se) {
            // we have reported the error; the exception just blasts us
            // out of parsing this template
        }
        return null;
    }
 
Example #8
Source File: AbstractSplittingTokenSource.java    From xtext-core with Eclipse Public License 2.0 6 votes vote down vote up
/**
 * Create a new token from the given prototype. Any argument besides the prototype is optional and
 * will be ignored if its value is <code>null</code>.
 */
protected CommonToken createToken(CommonToken prototype, String text, 
		Integer charPosInLine, Integer channel, Integer start, Integer stop, Integer type) {
	if (prototype == null)
		throw new IllegalArgumentException("Prototype may not be null.");
	CommonToken result = new CommonToken(prototype);
	if (text != null)
		result.setText(text);
	if (charPosInLine != null)
		result.setCharPositionInLine(charPosInLine.intValue());
	if (channel != null)
		result.setChannel(channel.intValue());
	if (start != null)
		result.setStartIndex(start.intValue());
	if (stop != null)
		result.setStopIndex(stop.intValue());
	if (type != null)
		result.setType(type.intValue());
	return result;
}
 
Example #9
Source File: TestTrees.java    From jFuzzyLogic with GNU Lesser General Public License v3.0 5 votes vote down vote up
public void testReplaceOneWithTwoAtLeft() throws Exception {
	CommonTree t = new CommonTree(new CommonToken(99, "a"));
	t.addChild(new CommonTree(new CommonToken(99, "b")));
	t.addChild(new CommonTree(new CommonToken(99, "c")));
	t.addChild(new CommonTree(new CommonToken(99, "d")));

	CommonTree newChildren = (CommonTree)adaptor.nil();
	newChildren.addChild(new CommonTree(new CommonToken(99,"x")));
	newChildren.addChild(new CommonTree(new CommonToken(99,"y")));

	t.replaceChildren(0, 0, newChildren);
	String expecting = "(a x y c d)";
	assertEquals(expecting, t.toStringTree());
	t.sanityCheckParentAndChildIndexes();
}
 
Example #10
Source File: AntlrDatatypeRuleTokenTest.java    From xtext-core with Eclipse Public License 2.0 5 votes vote down vote up
@Test public void testMergeFirstToken() {
	CommonToken commonToken = new CommonToken(TOKEN_TYPE, "text");
	commonToken.setStartIndex(4);
	AntlrDatatypeRuleToken token = new AntlrDatatypeRuleToken();
	token.merge(commonToken);
	assertEquals("text", token.getText());
	assertEquals(8, token.getExpectedOffset());
}
 
Example #11
Source File: STRawGroupDir.java    From codebuff with BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Override
public CompiledST loadTemplateFile(String prefix, String unqualifiedFileName, CharStream templateStream) {
    String template = templateStream.substring(0, templateStream.size()- 1);
    String templateName = Misc.getFileNameNoSuffix(unqualifiedFileName);
    String fullyQualifiedTemplateName = prefix+templateName;
    CompiledST impl = new Compiler(this).compile(fullyQualifiedTemplateName, template);
    CommonToken nameT = new CommonToken(STLexer.SEMI); // Seems like a hack, best I could come up with.
    nameT.setInputStream(templateStream);
    rawDefineTemplate(fullyQualifiedTemplateName, impl, nameT);
    impl.defineImplicitlyDefinedTemplates(this);
    return impl;
}
 
Example #12
Source File: STGroup.java    From codebuff with BSD 2-Clause "Simplified" License 5 votes vote down vote up
/** for testing */

    public CompiledST defineTemplate(String templateName, String template) {
        if ( templateName.charAt(0)!='/' ) templateName = "/"+templateName;
        try {
            CompiledST impl = defineTemplate(templateName, new CommonToken(GroupParser.ID, templateName), null, template, null);
            return impl;
        }
        catch (STException se) {
            // we have reported the error; the exception just blasts us
            // out of parsing this template
        }
        return null;
    }
 
Example #13
Source File: STGroup.java    From codebuff with BSD 2-Clause "Simplified" License 5 votes vote down vote up
/** for testing */

    public CompiledST defineTemplate(String templateName, String template) {
        if ( templateName.charAt(0)!='/' ) templateName = "/"+templateName;
        try {
            CompiledST impl = defineTemplate(templateName, new CommonToken(GroupParser.ID, templateName), null, template, null);
            return impl;
        }
        catch (STException se) {
            // we have reported the error; the exception just blasts us
            // out of parsing this template
        }
        return null;
    }
 
Example #14
Source File: LexerSLComment.java    From xtext-core with Eclipse Public License 2.0 5 votes vote down vote up
/**
 * see BUG 234135: Comments on EOF not detected
 * https://bugs.eclipse.org/bugs/show_bug.cgi?id=234135
 */
@Test public void testSlCommentEOF() {
	String model = "a\n//sl comment";
	InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
	lexer.setCharStream(new ANTLRStringStream(model));
	CommonTokenStream stream = new CommonTokenStream(lexer);
	Object eofLineComment = stream.getTokens().get(2);
	assertTrue(eofLineComment instanceof CommonToken);
	CommonToken commonToken = (CommonToken) eofLineComment;
	int type = commonToken.getType();
	assertEquals(InternalSimpleExpressionsTestLanguageLexer.RULE_SL_COMMENT, type);
}
 
Example #15
Source File: TestTrees.java    From jFuzzyLogic with GNU Lesser General Public License v3.0 5 votes vote down vote up
public void testBecomeRoot6() throws Exception {
	// emulates construction of ^(5 6)
	CommonTree root_0 = (CommonTree)adaptor.nil();
	CommonTree root_1 = (CommonTree)adaptor.nil();
	root_1 = (CommonTree)adaptor.becomeRoot(new CommonTree(new CommonToken(5)), root_1);

	adaptor.addChild(root_1, new CommonTree(new CommonToken(6)));

	adaptor.addChild(root_0, root_1);

	root_0.sanityCheckParentAndChildIndexes();
}
 
Example #16
Source File: STGroup.java    From codebuff with BSD 2-Clause "Simplified" License 5 votes vote down vote up
/** for testing */

    public CompiledST defineTemplate(String templateName, String template) {
        if ( templateName.charAt(0)!='/' ) templateName = "/"+templateName;
        try {
            CompiledST impl = defineTemplate(templateName, new CommonToken(GroupParser.ID, templateName), null, template, null);
            return impl;
        }
        catch (STException se) {
            // we have reported the error; the exception just blasts us
            // out of parsing this template
        }
        return null;
    }
 
Example #17
Source File: TestTrees.java    From jFuzzyLogic with GNU Lesser General Public License v3.0 5 votes vote down vote up
public void testReplaceTwoWithOneAtRight() throws Exception {
	CommonTree t = new CommonTree(new CommonToken(99, "a"));
	t.addChild(new CommonTree(new CommonToken(99, "b")));
	t.addChild(new CommonTree(new CommonToken(99, "c")));
	t.addChild(new CommonTree(new CommonToken(99, "d")));

	CommonTree newChild = new CommonTree(new CommonToken(99,"x"));

	t.replaceChildren(1, 2, newChild);
	String expecting = "(a b x)";
	assertEquals(expecting, t.toStringTree());
	t.sanityCheckParentAndChildIndexes();
}
 
Example #18
Source File: STGroup.java    From codebuff with BSD 2-Clause "Simplified" License 5 votes vote down vote up
/** for testing */

    public CompiledST defineTemplate(String name, String argsS, String template) {
        if ( name.charAt(0) !='/' ) name = "/"+name;
        String[] args = argsS.split(",");
        List<FormalArgument> a = new ArrayList<FormalArgument>();
        for (String arg : args) {
            a.add(
                new FormalArgument(arg)
            );
        }
        return defineTemplate(name, new CommonToken(GroupParser.ID, name), a, template, null);
    }
 
Example #19
Source File: TestTreeNodeStream.java    From jFuzzyLogic with GNU Lesser General Public License v3.0 5 votes vote down vote up
public void testMarkRewindEntire() throws Exception {
	// ^(101 ^(102 103 ^(106 107) ) 104 105)
	// stream has 7 real + 6 nav nodes
	// Sequence of types: 101 DN 102 DN 103 106 DN 107 UP UP 104 105 UP EOF
	Tree r0 = new CommonTree(new CommonToken(101));
	Tree r1 = new CommonTree(new CommonToken(102));
	r0.addChild(r1);
	r1.addChild(new CommonTree(new CommonToken(103)));
	Tree r2 = new CommonTree(new CommonToken(106));
	r2.addChild(new CommonTree(new CommonToken(107)));
	r1.addChild(r2);
	r0.addChild(new CommonTree(new CommonToken(104)));
	r0.addChild(new CommonTree(new CommonToken(105)));

	CommonTreeNodeStream stream = new CommonTreeNodeStream(r0);
	int m = stream.mark(); // MARK
	for (int k=1; k<=13; k++) { // consume til end
		stream.LT(1);
		stream.consume();
	}
	assertEquals(Token.EOF, ((Tree)stream.LT(1)).getType());
	assertEquals(Token.UP, ((Tree)stream.LT(-1)).getType());
	stream.rewind(m);      // REWIND

	// consume til end again :)
	for (int k=1; k<=13; k++) { // consume til end
		stream.LT(1);
		stream.consume();
	}
	assertEquals(Token.EOF, ((Tree)stream.LT(1)).getType());
	assertEquals(Token.UP, ((Tree)stream.LT(-1)).getType());
}
 
Example #20
Source File: STGroup.java    From codebuff with BSD 2-Clause "Simplified" License 5 votes vote down vote up
/** for testing */

    public CompiledST defineTemplate(String name, String argsS, String template) {
        if ( name.charAt(0)!='/' ) name = "/"+name;
        String[] args = argsS.split(",");
        List<FormalArgument> a = new ArrayList<FormalArgument>();
        for (String arg : args) {
            a.add(new FormalArgument(arg));
        }
        return defineTemplate(name, new CommonToken(GroupParser.ID, name), a, template, null);
    }
 
Example #21
Source File: Lexer.java    From gef with Eclipse Public License 2.0 5 votes vote down vote up
@Override
public Token nextToken() {
	while (true) {
		this.state.token = null;
		this.state.channel = Token.DEFAULT_CHANNEL;
		this.state.tokenStartCharIndex = input.index();
		this.state.tokenStartCharPositionInLine = input
				.getCharPositionInLine();
		this.state.tokenStartLine = input.getLine();
		this.state.text = null;
		if (input.LA(1) == CharStream.EOF) {
			return Token.EOF_TOKEN;
		}
		try {
			mTokens();
			if (this.state.token == null) {
				emit();
			} else if (this.state.token == Token.SKIP_TOKEN) {
				continue;
			}
			return this.state.token;
		} catch (RecognitionException re) {
			reportError(re);
			if (re instanceof NoViableAltException
					|| re instanceof FailedPredicateException) {
				recover(re);
			}
			// create token that holds mismatched char
			Token t = new CommonToken(input, Token.INVALID_TOKEN_TYPE,
					Token.HIDDEN_CHANNEL, this.state.tokenStartCharIndex,
					getCharIndex() - 1);
			t.setLine(this.state.tokenStartLine);
			t.setCharPositionInLine(
					this.state.tokenStartCharPositionInLine);
			tokenErrorMap.put(t, getErrorMessage(re, this.getTokenNames()));
			emit(t);
			return this.state.token;
		}
	}
}
 
Example #22
Source File: SemanticException.java    From AppTroy with Apache License 2.0 5 votes vote down vote up
SemanticException(IntStream input, Token token, String errorMessage, Object... messageArguments) {
    super();
    this.input = input;
    this.token = token;
    this.index = ((CommonToken)token).getStartIndex();
    this.line = token.getLine();
 this.charPositionInLine = token.getCharPositionInLine();
    this.errorMessage = String.format(errorMessage, messageArguments);
}
 
Example #23
Source File: SemanticException.java    From zjdroid with Apache License 2.0 5 votes vote down vote up
SemanticException(IntStream input, Token token, String errorMessage, Object... messageArguments) {
    super();
    this.input = input;
    this.token = token;
    this.index = ((CommonToken)token).getStartIndex();
    this.line = token.getLine();
 this.charPositionInLine = token.getCharPositionInLine();
    this.errorMessage = String.format(errorMessage, messageArguments);
}
 
Example #24
Source File: TestTrees.java    From jFuzzyLogic with GNU Lesser General Public License v3.0 5 votes vote down vote up
public void testBecomeRoot() throws Exception {
	// 5 becomes new root of ^(nil 101 102 103)
	CommonTree newRoot = new CommonTree(new CommonToken(5));

	CommonTree oldRoot = new CommonTree((Token)null);
	oldRoot.addChild(new CommonTree(new CommonToken(101)));
	oldRoot.addChild(new CommonTree(new CommonToken(102)));
	oldRoot.addChild(new CommonTree(new CommonToken(103)));

	TreeAdaptor adaptor = new CommonTreeAdaptor();
	adaptor.becomeRoot(newRoot, oldRoot);
	newRoot.sanityCheckParentAndChildIndexes();
}
 
Example #25
Source File: TestTrees.java    From jFuzzyLogic with GNU Lesser General Public License v3.0 5 votes vote down vote up
public void testReplaceAllWithOne() throws Exception {
	CommonTree t = new CommonTree(new CommonToken(99, "a"));
	t.addChild(new CommonTree(new CommonToken(99, "b")));
	t.addChild(new CommonTree(new CommonToken(99, "c")));
	t.addChild(new CommonTree(new CommonToken(99, "d")));

	CommonTree newChild = new CommonTree(new CommonToken(99,"x"));

	t.replaceChildren(0, 2, newChild);
	String expecting = "(a x)";
	assertEquals(expecting, t.toStringTree());
	t.sanityCheckParentAndChildIndexes();
}
 
Example #26
Source File: TestTreeNodeStream.java    From jFuzzyLogic with GNU Lesser General Public License v3.0 5 votes vote down vote up
public void testAoverB() throws Exception {
	Tree t = new CommonTree(new CommonToken(101));
	t.addChild(new CommonTree(new CommonToken(102)));

	TreeNodeStream stream = newStream(t);
	String expecting = " 101 102";
	String found = toNodesOnlyString(stream);
	assertEquals(expecting, found);

	expecting = " 101 2 102 3";
	found = stream.toString();
	assertEquals(expecting, found);
}
 
Example #27
Source File: NbParseTreeBuilder.java    From netbeans with Apache License 2.0 5 votes vote down vote up
@Override
public void consumeToken(Token token) {
    if (backtracking > 0 || resync) {
        return;
    }

    if (debug_tokens) {
        CommonToken ct = (CommonToken) token;
        int[] ctr = CommonTokenUtil.getCommonTokenOffsetRange(ct);
        System.out.println(token + "(" + ctr[0] + "-" + ctr[1] + ")");
    }

    //ignore the closing EOF token, we do not want it
    //it the parse tree
    if (token.getType() == Css3Lexer.EOF) {
        return;
    }

    //also ignore error tokens - they are added as children of ErrorNode-s in the recognitionException(...) method
    if (token.getType() == Token.INVALID_TOKEN_TYPE) {
        return;
    }

    lastConsumedToken = (CommonToken) token;

    RuleNode ruleNode = callStack.peek();
    TokenNode elementNode = new TokenNode(source, (CommonToken) token);
    elementNode.hiddenTokens = this.hiddenTokens;
    hiddenTokens.clear();
    ruleNode.addChild(elementNode);

    updateFirstTokens(ruleNode, lastConsumedToken);
}
 
Example #28
Source File: STGroup.java    From codebuff with BSD 2-Clause "Simplified" License 5 votes vote down vote up
/** for testing */

    public CompiledST defineTemplate(String templateName, String template) {
        if ( templateName.charAt(0)!='/' ) templateName = "/"+templateName;
        try {
            CompiledST impl = defineTemplate(templateName, new CommonToken(GroupParser.ID, templateName), null, template, null);
            return impl;
        }
        catch (STException se) {
            // we have reported the error; the exception just blasts us
            // out of parsing this template
        }
        return null;
    }
 
Example #29
Source File: TestTrees.java    From jFuzzyLogic with GNU Lesser General Public License v3.0 5 votes vote down vote up
public void testBecomeRoot5() throws Exception {
	// ^(nil 5) becomes new root of ^(101 102 103)
	CommonTree newRoot = new CommonTree((Token)null);
	newRoot.addChild(new CommonTree(new CommonToken(5)));

	CommonTree oldRoot = new CommonTree(new CommonToken(101));
	oldRoot.addChild(new CommonTree(new CommonToken(102)));
	oldRoot.addChild(new CommonTree(new CommonToken(103)));

	TreeAdaptor adaptor = new CommonTreeAdaptor();
	adaptor.becomeRoot(newRoot, oldRoot);
	newRoot.sanityCheckParentAndChildIndexes();
}
 
Example #30
Source File: LexerErrorTest.java    From xtext-core with Eclipse Public License 2.0 5 votes vote down vote up
@Test public void testLexerError_04() throws Exception {
	String model = "a 'incomplete string with bad escape sequence \\";
	InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
	lexer.setCharStream(new ANTLRStringStream(model));
	CommonTokenStream stream = new CommonTokenStream(lexer);
	@SuppressWarnings("unchecked")
	List<CommonToken> tokens = stream.getTokens();
	assertEquals(tokens.toString(), 3, tokens.size());
	assertEquals("a", tokens.get(0).getText());
	assertEquals(" ", tokens.get(1).getText());
	assertEquals("'incomplete string with bad escape sequence \\", tokens.get(2).getText());
	assertEquals(0, tokens.get(2).getType());
}