Java Code Examples for org.antlr.runtime.Token

The following are top voted examples for showing how to use org.antlr.runtime.Token. These examples are extracted from open source projects. You can vote up the examples you like and your votes will be used in our system to generate more good examples.
Example 1
Project: n4js   File: HighlightingParserTester.java   View source code 6 votes vote down vote up
/**
 * Parses the input and returns a list of lexer tokens. Asserts that the produced tokens are equal to the tokens
 * that the production parser produced.
 *
 * @return the tokens for the highlighting.
 */
public List<Token> getTokens(CharSequence input) {
	List<Token> result;
	IParseResult parseResult = parser.parse(new StringReader(input.toString()));
	if (!parseResult.hasSyntaxErrors()) {
		result = throwingHighlightingParser.getTokens(input);
	} else {
		result = highlightingParser.getTokens(input);
	}
	// assert equal tokens
	Iterator<Token> iter = result.iterator();
	for (ILeafNode leaf : parseResult.getRootNode().getLeafNodes()) {
		Assert.assertTrue("hasNext at index " + leaf.getTotalOffset() + " for leaf '" + leaf.getText() + "'",
				iter.hasNext());
		Token token = iter.next();
		// TODO: assert token type
		Assert.assertEquals(leaf.getText(), token.getText());
	}
	return result;
}
 
Example 2
Project: n4js   File: ContentAssistContextFactory.java   View source code 6 votes vote down vote up
@Override
protected void createContextsForLastCompleteNode(EObject previousModel, boolean strict) {
	String currentNodePrefix = getPrefix(currentNode);
	if (!Strings.isEmpty(currentNodePrefix) && !currentNode.getText().equals(currentNodePrefix)) {
		lexer.setCharStream(new ANTLRStringStream(currentNodePrefix));
		Token token = lexer.nextToken();
		if (token == Token.EOF_TOKEN) { // error case - nothing could be parsed
			return;
		}
		while (token != Token.EOF_TOKEN) {
			if (isErrorToken(token))
				return;
			token = lexer.nextToken();
		}
	}
	String prefix = "";
	Collection<FollowElement> followElements = getParser().getFollowElements(rootNode, 0, completionOffset, strict);
	doCreateContexts(lastCompleteNode, currentNode, prefix, previousModel, followElements);
}
 
Example 3
Project: n4js   File: ParserBasedDocumentTokenSource.java   View source code 6 votes vote down vote up
@Override
protected TokenSource createTokenSource(String string) {
	List<Token> tokens = highlightingParser.getTokens(string);
	Iterator<Token> iter = tokens.iterator();
	return new TokenSource() {

		@Override
		public Token nextToken() {
			if (iter.hasNext()) {
				return iter.next();
			}
			return Token.EOF_TOKEN;
		}

		@Override
		public String getSourceName() {
			return "Text: " + string;
		}
	};
}
 
Example 4
Project: n4js   File: InternalHighlightingParser.java   View source code 6 votes vote down vote up
@Override
protected void announce(Token start, Token stop, AbstractElement element) {
	if (start != null && start != Token.EOF_TOKEN) {
		if (start == stop) {
			announce(start, element);
		} else {
			CommonToken castedStart = (CommonToken) start;
			if (stop == null) { // possible error condition
				if (start.getTokenIndex() == state.lastErrorIndex) {
					return;
				}
			}
			CommonToken castedEnd = (CommonToken) stop;
			Integer newType = rewriter.rewrite(castedStart, element);
			if (newType != null && castedEnd != null && castedEnd != Token.EOF_TOKEN) {
				LazyTokenStream castedInput = (LazyTokenStream) this.input;
				for (int i = castedStart.getTokenIndex() + 1; i < castedEnd.getTokenIndex(); i++) {
					Token token = castedInput.get(i);
					if (token.getChannel() != Token.HIDDEN_CHANNEL)
						token.setType(newType);
				}
				castedEnd.setType(newType);
			}
		}
	}
}
 
Example 5
Project: n4js   File: SemicolonInjectionHelper.java   View source code 6 votes vote down vote up
/**
 * Returns {@code true} if the set of expected follow-states includes an implicit or explicit semicolon.
 */
private static boolean followedBySemicolon(RecognizerSharedState state, Callback.RecoverySets recoverySets,
		int currentIndex) {
	int top = state._fsp;
	if (currentIndex != state.lastErrorIndex) {
		long[] array = state.following[top].toPackedArray();
		if (array.length == 1 && array[0] == (1L << Token.EOR_TOKEN_TYPE)) {
			return false;
		}
	}
	for (int i = top; i >= 0; i--) {
		BitSet localFollowSet = state.following[i];
		if (recoverySets.matches(localFollowSet)) {
			return true;
		}

	}
	return false;
}
 
Example 6
Project: n4js   File: SemicolonInjectionHelper.java   View source code 6 votes vote down vote up
/**
 * Returns true if there was an unexpected EOL.
 */
public static boolean hasDisallowedEOL(Callback callback) {
	TokenStream input = callback.getInput();
	Token lt = input.LT(1);

	// Start on the position before the current token and scan backwards off channel tokens until the previous on
	// channel token.
	for (int ix = lt.getTokenIndex() - 1; ix > 0; ix--) {
		lt = input.get(ix);
		if (lt.getChannel() == Token.DEFAULT_CHANNEL) {
			// On channel token found: stop scanning.
			break;
		} else if (isSemicolonEquivalent(lt)) {
			return true;
		}
	}
	return false;
}
 
Example 7
Project: n4js   File: SemicolonInjectionHelper.java   View source code 6 votes vote down vote up
/**
 * Returns true if the given token is considered to be semicolon equivalent.
 */
static boolean isSemicolonEquivalent(Token lt) {
	if (lt.getType() == InternalN4JSParser.RULE_EOL) {
		return true;
	}
	if (lt.getType() == InternalN4JSParser.RULE_ML_COMMENT) {
		String tokenText = lt.getText();
		for (int i = 2; i < tokenText.length() - 2; i++) {
			char c = tokenText.charAt(i);
			if (c == '\n' || c == '\r') {
				return true;
			}
		}
	}
	return false;
}
 
Example 8
Project: incubator-netbeans   File: ExtCss3Parser.java   View source code 6 votes vote down vote up
/** Consume tokens until one matches the given token set */
    @Override
    public void consumeUntil(IntStream i, BitSet set) {
//        System.out.println("consumeUntil(" + set.toString(getTokenNames()) + ")");
        Token ttype;
        List<Token> skipped = new ArrayList<>();
        beginResync();
        try {
            while ((ttype = input.LT(1)) != null && ttype.getType() != Token.EOF && !set.member(ttype.getType())) {
//            System.out.println("consume during recover LA(1)=" + getTokenNames()[input.LA(1)]);
                input.consume();
                skipped.add(ttype);
            }
        } finally {
            endResync();
        }
        ((NbParseTreeBuilder) dbg).consumeSkippedTokens(skipped);
    }
 
Example 9
Project: JavaGraph   File: CtrlHelper.java   View source code 6 votes vote down vote up
/** Creates a qualified name tree by extending an existing one
 * with an additional fragment in front.
 * @param init token holding the additional level text
 * @param subTree the existing tree; may be {@code null}, in which case
 * the result is calculated using {@link #toQualName(Token,Token)} with first
 * argument {@code null}
 */
CommonTree toQualName(Token init, CtrlTree subTree) {
    CtrlTree result;
    if (subTree == null || this.namespace.hasErrors()) {
        result = toQualName(null, init);
    } else {
        Token subTop = subTree.getToken();
        QualName qualName = subTree.getQualName()
            .nest(getText(init));
        CommonToken top = new CommonToken(subTop.getType(), qualName.toString());
        top.setLine(init.getLine());
        top.setTokenIndex(init.getTokenIndex());
        result = new CtrlTree(top);
        result.setQualName(qualName);
        result.addChild(subTree.getChild(0));
    }
    return result;
}
 
Example 10
Project: codebuff   File: STGroup.java   View source code 6 votes vote down vote up
public CompiledST defineRegion(String enclosingTemplateName, Token regionT, String template, Token templateToken) {
    String name = regionT.getText();
    template = Misc.trimOneStartingNewline(template);
    template = Misc.trimOneTrailingNewline(template);
    CompiledST code = compile(getFileName(), enclosingTemplateName, null, template, templateToken);
    String mangled = getMangledRegionName(enclosingTemplateName, name);
    if ( lookupTemplate(mangled)==null ) {
        errMgr.compileTimeError(ErrorType.NO_SUCH_REGION, templateToken, regionT, enclosingTemplateName, name);
        return new CompiledST();
    }
    code.name = mangled;
    code.isRegion = true;
    code.regionDefType = ST.RegionType.EXPLICIT;
    code.templateDefStartToken = regionT;
    rawDefineTemplate(mangled, code, regionT);
    code.defineArgDefaultValueTemplates(this);
    code.defineImplicitlyDefinedTemplates(this);
    return code;
}
 
Example 11
Project: codebuff   File: STGroup.java   View source code 6 votes vote down vote up
public CompiledST defineTemplate(String fullyQualifiedTemplateName,
                                 Token nameT,
                                 List<FormalArgument> args,
                                 String template, Token templateToken) {
    if ( verbose ) System.out.println("defineTemplate("+fullyQualifiedTemplateName+")");
    if ( fullyQualifiedTemplateName==null || fullyQualifiedTemplateName.length()==0 ) {
        throw new IllegalArgumentException("empty template name");
    }
    if ( fullyQualifiedTemplateName.indexOf('.')>=0 ) {
        throw new IllegalArgumentException("cannot have '.' in template names");
    }
    template = Misc.trimOneStartingNewline(template);
    template = Misc.trimOneTrailingNewline(template);
    // compile, passing in templateName as enclosing name for any embedded regions
    CompiledST code = compile(getFileName(),
                              fullyQualifiedTemplateName,
                              args,
                              template,
                              templateToken);
    code.name = fullyQualifiedTemplateName;
    rawDefineTemplate(fullyQualifiedTemplateName, code, nameT);
    code.defineArgDefaultValueTemplates(this);
    code.defineImplicitlyDefinedTemplates(this); // define any anonymous subtemplates
    return code;
}
 
Example 12
Project: xtext-core   File: PartialContentAssistContextFactory.java   View source code 6 votes vote down vote up
@Override
protected void createContextsForLastCompleteNode(EObject previousModel, boolean strict) {
	String currentNodePrefix = getPrefix(currentNode);
	if (!Strings.isEmpty(currentNodePrefix) && !currentNode.getText().equals(currentNodePrefix)) {
		lexer.setCharStream(new ANTLRStringStream(currentNodePrefix));
		Token token = lexer.nextToken();
		if (token == Token.EOF_TOKEN) {
			return;
		}
		while (token != Token.EOF_TOKEN) {
			if (isErrorToken(token)) {
				return;
			}
			token = lexer.nextToken();
		}
	}
	String prefix = "";
	Collection<FollowElement> followElements = parseFollowElements(completionOffset, strict);
	doCreateContexts(lastCompleteNode, currentNode, prefix, previousModel, followElements);
}
 
Example 13
Project: codebuff   File: STLexer.java   View source code 6 votes vote down vote up
Token COMMENT() {
    match('!');
    while ( !(c=='!' && input.LA(2)==delimiterStopChar) ) {
        if ( c==EOF ) {
            RecognitionException re = new MismatchedTokenException((int)'!', input);
            re.line = input.getLine();
            re.charPositionInLine = input.getCharPositionInLine();
            errMgr.lexerError(input.getSourceName(), "Nonterminated comment starting at "+startLine+":"+startCharPositionInLine+": '!"+delimiterStopChar+"' missing", templateToken, re);
            break;
        }
        consume();
    }
    consume();
    consume(); // grab !>
    return newToken(COMMENT);
}
 
Example 14
Project: codebuff   File: STLexer.java   View source code 6 votes vote down vote up
Token COMMENT() {
    match('!');
    while ( !(c=='!' && input.LA(2)==delimiterStopChar) ) {
        if ( c==EOF ) {
            RecognitionException re = new MismatchedTokenException((int)'!', input);
            re.line = input.getLine();
            re.charPositionInLine = input.getCharPositionInLine();
            errMgr.lexerError(input.getSourceName(), "Nonterminated comment starting at "+startLine+":"+startCharPositionInLine+": '!"+delimiterStopChar+"' missing", templateToken, re);
            break;
        }
        consume();
    }
    consume();
    consume(); // grab !>
    return newToken(COMMENT);
}
 
Example 15
Project: codebuff   File: STLexer.java   View source code 6 votes vote down vote up
Token COMMENT() {
    match('!');
    while ( !(c=='!' && input.LA(2)==delimiterStopChar) ) {
        if ( c==EOF ) {
            RecognitionException re = new MismatchedTokenException((int)'!', input);
            re.line = input.getLine();
            re.charPositionInLine = input.getCharPositionInLine();
            errMgr.lexerError(input.getSourceName(), "Nonterminated comment starting at "+startLine+":"+startCharPositionInLine+": '!"+delimiterStopChar+"' missing", templateToken, re);
            break;
        }
        consume();
    }
    consume();
    consume(); // grab !>
    return newToken(COMMENT);
}
 
Example 16
Project: codebuff   File: STGroup.java   View source code 6 votes vote down vote up
public CompiledST defineTemplate(String fullyQualifiedTemplateName, Token nameT, List<FormalArgument> args, String template, Token templateToken) {
    if ( verbose ) System.out.println("defineTemplate("+fullyQualifiedTemplateName+")");
    if ( fullyQualifiedTemplateName==null || fullyQualifiedTemplateName.length()==0 ) {
        throw new IllegalArgumentException("empty template name");
    }
    if ( fullyQualifiedTemplateName.indexOf('.')>=0 ) {
        throw new IllegalArgumentException("cannot have '.' in template names");
    }
    template = Misc.trimOneStartingNewline(template);
    template = Misc.trimOneTrailingNewline(template);
    // compile, passing in templateName as enclosing name for any embedded regions
    CompiledST code = compile(getFileName(), fullyQualifiedTemplateName, args, template, templateToken);
    code.name = fullyQualifiedTemplateName;
    rawDefineTemplate(fullyQualifiedTemplateName, code, nameT);
    code.defineArgDefaultValueTemplates(this);
    code.defineImplicitlyDefinedTemplates(this); // define any anonymous subtemplates
    return code;
}
 
Example 17
Project: codebuff   File: STLexer.java   View source code 6 votes vote down vote up
Token ESCAPE() {
startCharIndex = input.index();
startCharPositionInLine = input.getCharPositionInLine();
consume(); // kill \\
if ( c=='u') return UNICODE();
String text;
      switch ( c ) {
          case '\\' : LINEBREAK(); return SKIP;
	case 'n'  : text = "\n"; break;
	case 't'  : text = "\t"; break;
	case ' '  : text = " "; break;
          default :
              NoViableAltException e = new NoViableAltException("",0,0,input);
              errMgr.lexerError(input.getSourceName(), "invalid escaped char: '"+str(c)+"'", templateToken, e);
		consume();
		match(delimiterStopChar);
		return SKIP;
      }
      consume();
Token t = newToken(TEXT, text, input.getCharPositionInLine()-2);
      match(delimiterStopChar);
      return t;
  }
 
Example 18
Project: codebuff   File: STLexer.java   View source code 6 votes vote down vote up
Token COMMENT() {
    match('!');
    while ( !(c=='!' && input.LA(2)==delimiterStopChar) ) {
        if ( c==EOF ) {
            RecognitionException re = new MismatchedTokenException((int)'!', input);
            re.line = input.getLine();
            re.charPositionInLine = input.getCharPositionInLine();
            errMgr.lexerError(input.getSourceName(),
                              "Nonterminated comment starting at " +
                              startLine +":"+startCharPositionInLine +
                              ": '!" +
                              delimiterStopChar +
                              "' missing", templateToken, re);
            break;
        }
        consume();
    }
    consume();
    consume(); // grab !>
    return newToken(COMMENT);
}
 
Example 19
Project: sstable-adaptor   File: ErrorCollector.java   View source code 5 votes vote down vote up
/**
 * Appends a query snippet to the message to help the user to understand the problem.
 *
 * @param parser the parser used to parse the query
 * @param builder the <code>StringBuilder</code> used to build the error message
 */
private void appendQuerySnippet(Parser parser, StringBuilder builder)
{
    TokenStream tokenStream = parser.getTokenStream();
    int index = tokenStream.index();
    int size = tokenStream.size();

    Token from = tokenStream.get(getSnippetFirstTokenIndex(index));
    Token to = tokenStream.get(getSnippetLastTokenIndex(index, size));
    Token offending = tokenStream.get(getOffendingTokenIndex(index, size));

    appendSnippet(builder, from, to, offending);
}
 
Example 20
Project: sstable-adaptor   File: ErrorCollector.java   View source code 5 votes vote down vote up
/**
 * Appends a query snippet to the message to help the user to understand the problem.
 *
 * @param from the first token to include within the snippet
 * @param to the last token to include within the snippet
 * @param offending the token which is responsible for the error
 */
final void appendSnippet(StringBuilder builder,
                         Token from,
                         Token to,
                         Token offending)
{
    if (!areTokensValid(from, to, offending))
        return;

    String[] lines = query.split("\n");

    boolean includeQueryStart = (from.getLine() == 1) && (from.getCharPositionInLine() == 0);
    boolean includeQueryEnd = (to.getLine() == lines.length)
            && (getLastCharPositionInLine(to) == lines[lines.length - 1].length());

    builder.append(" (");

    if (!includeQueryStart)
        builder.append("...");

    String toLine = lines[lineIndex(to)];
    int toEnd = getLastCharPositionInLine(to);
    lines[lineIndex(to)] = toEnd >= toLine.length() ? toLine : toLine.substring(0, toEnd);
    lines[lineIndex(offending)] = highlightToken(lines[lineIndex(offending)], offending);
    lines[lineIndex(from)] = lines[lineIndex(from)].substring(from.getCharPositionInLine());

    for (int i = lineIndex(from), m = lineIndex(to); i <= m; i++)
        builder.append(lines[i]);

    if (!includeQueryEnd)
        builder.append("...");

    builder.append(")");
}
 
Example 21
Project: sstable-adaptor   File: ErrorCollector.java   View source code 5 votes vote down vote up
/**
 * Checks if the specified tokens are valid.
 *
 * @param tokens the tokens to check
 * @return <code>true</code> if all the specified tokens are valid ones,
 * <code>false</code> otherwise.
 */
private static boolean areTokensValid(Token... tokens)
{
    for (Token token : tokens)
    {
        if (!isTokenValid(token))
            return false;
    }
    return true;
}
 
Example 22
Project: sstable-adaptor   File: CqlLexer.java   View source code 5 votes vote down vote up
public Token nextToken()
{
    super.nextToken();
    if (tokens.size() == 0)
        return new CommonToken(Token.EOF);
    return tokens.remove(0);
}
 
Example 23
Project: sstable-adaptor   File: Cql_Parser.java   View source code 5 votes vote down vote up
public final String basic_unreserved_keyword() throws RecognitionException {
	String str = null;


	Token k=null;

	try {
		// Parser.g:1628:5: (k= ( K_KEYS | K_AS | K_CLUSTERING | K_COMPACT | K_STORAGE | K_TYPE | K_VALUES | K_MAP | K_LIST | K_FILTERING | K_PERMISSION | K_PERMISSIONS | K_KEYSPACES | K_ALL | K_USER | K_USERS | K_ROLE | K_ROLES | K_SUPERUSER | K_NOSUPERUSER | K_LOGIN | K_NOLOGIN | K_OPTIONS | K_PASSWORD | K_EXISTS | K_CUSTOM | K_TRIGGER | K_CONTAINS | K_STATIC | K_FROZEN | K_TUPLE | K_FUNCTION | K_FUNCTIONS | K_AGGREGATE | K_SFUNC | K_STYPE | K_FINALFUNC | K_INITCOND | K_RETURNS | K_LANGUAGE | K_CALLED | K_INPUT | K_LIKE | K_PER | K_PARTITION | K_GROUP ) )
		// Parser.g:1628:7: k= ( K_KEYS | K_AS | K_CLUSTERING | K_COMPACT | K_STORAGE | K_TYPE | K_VALUES | K_MAP | K_LIST | K_FILTERING | K_PERMISSION | K_PERMISSIONS | K_KEYSPACES | K_ALL | K_USER | K_USERS | K_ROLE | K_ROLES | K_SUPERUSER | K_NOSUPERUSER | K_LOGIN | K_NOLOGIN | K_OPTIONS | K_PASSWORD | K_EXISTS | K_CUSTOM | K_TRIGGER | K_CONTAINS | K_STATIC | K_FROZEN | K_TUPLE | K_FUNCTION | K_FUNCTIONS | K_AGGREGATE | K_SFUNC | K_STYPE | K_FINALFUNC | K_INITCOND | K_RETURNS | K_LANGUAGE | K_CALLED | K_INPUT | K_LIKE | K_PER | K_PARTITION | K_GROUP )
		{
			k=input.LT(1);
			if ( (input.LA(1) >= K_AGGREGATE && input.LA(1) <= K_ALL)||input.LA(1)==K_AS||input.LA(1)==K_CALLED||input.LA(1)==K_CLUSTERING||(input.LA(1) >= K_COMPACT && input.LA(1) <= K_CONTAINS)||input.LA(1)==K_CUSTOM||(input.LA(1) >= K_EXISTS && input.LA(1) <= K_FINALFUNC)||input.LA(1)==K_FROZEN||(input.LA(1) >= K_FUNCTION && input.LA(1) <= K_FUNCTIONS)||input.LA(1)==K_GROUP||(input.LA(1) >= K_INITCOND && input.LA(1) <= K_INPUT)||input.LA(1)==K_KEYS||(input.LA(1) >= K_KEYSPACES && input.LA(1) <= K_LIKE)||(input.LA(1) >= K_LIST && input.LA(1) <= K_MAP)||input.LA(1)==K_NOLOGIN||input.LA(1)==K_NOSUPERUSER||input.LA(1)==K_OPTIONS||(input.LA(1) >= K_PARTITION && input.LA(1) <= K_PERMISSIONS)||input.LA(1)==K_RETURNS||(input.LA(1) >= K_ROLE && input.LA(1) <= K_ROLES)||input.LA(1)==K_SFUNC||(input.LA(1) >= K_STATIC && input.LA(1) <= K_SUPERUSER)||input.LA(1)==K_TRIGGER||(input.LA(1) >= K_TUPLE && input.LA(1) <= K_TYPE)||(input.LA(1) >= K_USER && input.LA(1) <= K_USERS)||input.LA(1)==K_VALUES ) {
				input.consume();
				state.errorRecovery=false;
			}
			else {
				MismatchedSetException mse = new MismatchedSetException(null,input);
				throw mse;
			}
			str = (k!=null?k.getText():null);
		}

	}
	catch (RecognitionException re) {
		reportError(re);
		recover(input,re);
	}
	finally {
		// do for sure before leaving
	}
	return str;
}
 
Example 24
Project: n4js   File: CustomN4JSParser.java   View source code 5 votes vote down vote up
/**
 * The second pass over the given input. If the input ends with an ASI, the semicolon is removed and it is parsed
 * again since the production may have skipped the ASI if more input was present. Same applies for the opposite
 * direction, e.g if it does not end with an ASI but the prev token suggests that there may have been a semicolon,
 * it is inserted.
 */
private void adjustASIAndCollectFollowElements(CustomInternalN4JSParser previousParser, boolean strict,
		Set<FollowElement> result) {
	ObservableXtextTokenStream tokens = (ObservableXtextTokenStream) previousParser.getTokenStream();
	int lastTokenIndex = tokens.size() - 1;
	if (lastTokenIndex >= 0) {
		CommonToken lastToken = (CommonToken) tokens.LT(-1);
		@SuppressWarnings("unchecked")
		List<Token> tokenList = tokens.getTokens();
		if (lastToken == null) {
			return; // TODO ask Sebastian why and how this can happen... it happens!
		}

		if (shouldSkipASI(lastToken)) {
			// Some statements may not span multiple lines, e.g between the return keyword
			// and the returned expression, there may not be an ASI. Filter these locations.
			if (maySkipASI(lastToken, tokens)) {
				tokenList.remove(lastTokenIndex);
				result.addAll(resetAndGetFollowElements(tokens, strict));
				// If a postfix operator sneaked into the result, remove it since
				// we'd have produce an ASI before that
				removePostfixOperator(result);
			}
		} else if (shouldAddSyntheticSemicolon(previousParser, lastTokenIndex, lastToken)) {
			CommonToken token = new CommonToken(semi);
			tokenList.add(token);
			result.addAll(resetAndGetFollowElements(tokens, strict));
			// Same here, if we had added an ASI, the postfix operator would be rendered
			// invalid, remove it.
			removePostfixOperator(result);
		}
	}
}
 
Example 25
Project: n4js   File: CustomN4JSParser.java   View source code 5 votes vote down vote up
/**
 * Prevent ASIs to be skipped at certain locations, e.g. after a return keyword.
 */
private boolean maySkipASI(CommonToken lastToken, ObservableXtextTokenStream tokens) {
	int countDownFrom = lastToken.getTokenIndex();
	for (int i = countDownFrom - 1; i >= 0; i--) {
		Token prevToken = tokens.get(i);
		if (prevToken.getChannel() == Token.DEFAULT_CHANNEL) {
			if (mandatoryASI.get(prevToken.getType())) {
				return false;
			}
			return true;
		}
	}
	return true;
}
 
Example 26
Project: n4js   File: NodeModelTokenSource.java   View source code 5 votes vote down vote up
/**
 * Implementation of the {@link TokenSource} interface. Return new tokens as long as there are some, afterwards
 * return {@link Token#EOF_TOKEN}.
 */
@Override
public Token nextToken() {
	if (next != null) {
		Token result = next;
		next = null;
		return result;
	}
	if (!leafNodes.hasNext()) {
		return Token.EOF_TOKEN;
	}
	ILeafNode leaf = leafNodes.next();
	if (leaf.getTotalOffset() >= endOffset) {
		leafNodes = Iterators.emptyIterator();
		return Token.EOF_TOKEN;
	}
	if (leaf.getTotalEndOffset() <= startOffset) {
		return nextToken();
	}
	if (leaf.getTotalEndOffset() > endOffset) {
		return toPrefixToken(leaf);
	}
	SyntaxErrorMessage syntaxErrorMessage = leaf.getSyntaxErrorMessage();
	if (syntaxErrorMessage != null && SEMICOLON_INSERTED.equals(syntaxErrorMessage.getIssueCode())) {
		return toASIToken(leaf);
	}
	if (leaf.isHidden()) {
		return processHiddenToken(leaf);
	}
	int tokenType = tokenTypeMapper.getInternalTokenType(leaf);
	return new CommonToken(tokenType, leaf.getText());
}
 
Example 27
Project: n4js   File: NodeModelTokenSource.java   View source code 5 votes vote down vote up
/**
 * Skips the given leaf as it's hidden. If it was the last token to be returned, a hidden token may be syntesized if
 * would affect the semicolon insertion.
 */
private Token processHiddenToken(ILeafNode leaf) {
	Token result = nextToken();
	if (result == Token.EOF_TOKEN && Strings.countLineBreaks(leaf.getText()) > 0) {
		next = result;
		CommonToken hidden = new CommonToken(tokenTypeMapper.getInternalTokenType(leaf), leaf.getText());
		hidden.setChannel(Token.HIDDEN_CHANNEL);
		return hidden;
	}
	return result;
}
 
Example 28
Project: n4js   File: NodeModelTokenSource.java   View source code 5 votes vote down vote up
/**
 * Produce an Antlr token for the prefix of the given leaf that overlaps the requested region
 *
 * @see #endOffset
 */
private Token toPrefixToken(ILeafNode leaf) {
	Lexer lexer = new InternalN4JSLexer();
	String text = leaf.getText();
	String prefix = text.substring(0, endOffset - leaf.getTotalOffset());
	ANTLRStringStream stream = new ANTLRStringStream(prefix);
	lexer.setCharStream(stream);
	Token nextToken = lexer.nextToken();
	// copy to get rid of the reference to the stream again
	return new CommonToken(nextToken.getType(), nextToken.getText());
}
 
Example 29
Project: n4js   File: HighlightingParser.java   View source code 5 votes vote down vote up
/**
 * Obtain the tokens from the given reader.
 */
public List<Token> getTokens(Reader reader) {
	try {
		return doParse(new AntlrStreamWithToString(reader));
	} catch (IOException e) {
		throw new WrappedException(e);
	}
}
 
Example 30
Project: n4js   File: HighlightingParser.java   View source code 5 votes vote down vote up
private List<Token> doParse(CharStream in) {
	TokenSource tokenSource = createLexer(in);
	LazyTokenStream tokenStream = createTokenStream(tokenSource);
	setInitialHiddenTokens(tokenStream);
	InternalN4JSParser parser = createParser(tokenStream);
	IUnorderedGroupHelper helper = unorderedGroupHelper.get();
	if (!(helper instanceof IUnorderedGroupHelper.Null)) {
		throw new IllegalStateException("Unexpected usage of unordered groups.");
	}
	Stopwatch stopwatch = null;
	boolean debug = LOGGER.isDebugEnabled();
	// boolean debug = true;
	if (debug) {
		stopwatch = Stopwatch.createStarted();
	}
	try {
		parser.entryRuleScript();
		while (tokenStream.LT(1) != Token.EOF_TOKEN) {
			tokenStream.consume();
		}
		@SuppressWarnings("unchecked")
		List<Token> result = tokenStream.getTokens();
		return result;
	} catch (Exception re) {
		throw new ParseException(re.getMessage(), re);
	} finally {
		if (debug) {
			assert stopwatch != null;
			long elapsed = stopwatch.stop().elapsed(TimeUnit.MILLISECONDS);
			if (elapsed > 5) {
				LOGGER.warn("Coloring parser took: " + elapsed);
			}
		}
	}
}
 
Example 31
Project: n4js   File: JSTokenList.java   View source code 5 votes vote down vote up
/**
 * Adds token and adjust channel.
 */
@Override
public boolean add(Token tok) {
	super.add(tok);
	int type = tok.getType();
	if (type == InternalN4JSParser.EqualsSignGreaterThanSign) {
		// The arrow expression may not follow a semicolon thus we promote those here
		// to he default channel if they precede the arrow => operator
		for (int i = size() - 2; i >= 0; i--) {
			Token prev = get(i);
			if (prev.getChannel() == Token.HIDDEN_CHANNEL) {
				if (SemicolonInjectionHelper.isSemicolonEquivalent(prev)) {
					prev.setChannel(Token.DEFAULT_CHANNEL);
					break;
				}
			} else {
				break;
			}
		}
	} else if (type == InternalN4JSParser.RULE_EOL
			|| type == InternalN4JSParser.RULE_ML_COMMENT
			|| type == InternalN4JSParser.RULE_WS
			|| type == InternalN4JSParser.RULE_SL_COMMENT) {
		tok.setChannel(Token.HIDDEN_CHANNEL);
	} else {
		tok.setChannel(Token.DEFAULT_CHANNEL);
	}
	return true;
}
 
Example 32
Project: n4js   File: InternalSemicolonInjectingParser.java   View source code 5 votes vote down vote up
private void exhaustTokenSource() {
	LazyTokenStream casted = (LazyTokenStream) this.input;
	int marked = casted.mark();
	try {
		while (casted.LT(1) != Token.EOF_TOKEN) {
			casted.consume();
		}
	} finally {
		casted.rewind(marked);
	}
}
 
Example 33
Project: n4js   File: SemicolonInjectionHelper.java   View source code 5 votes vote down vote up
/**
 * <p>
 * Promotes EOL which may lead to an automatically inserted semicolon. This is probably the most important method
 * for automatic semicolon insertion, as it is only possible to insert a semicolon in case of line breaks (even if
 * they are hidden in a multi-line comment!).
 * </p>
 */
public static void promoteEOL(Callback callback) {
	RecognizerSharedState state = callback.getState();
	TokenStream input = callback.getInput();
	// Don't promote EOL if there was a syntax error at EOF
	if (state.lastErrorIndex == input.size()) {
		return;
	}
	// Get current token and its type (the possibly offending token).
	Token prev = input.LT(-1);
	Token next = input.LT(1);
	int la = next.getType();

	// Promoting an EOL means switching it from off channel to on channel.
	// A ML_COMMENT gets promoted when it contains an EOL.
	for (int idx = prev == null ? 0 : prev.getTokenIndex() + 1, max = la == Token.EOF ? input.size()
			: next.getTokenIndex(); idx < max; idx++) {
		Token lt = input.get(idx);
		if (lt.getChannel() == Token.DEFAULT_CHANNEL) {
			// On channel token found: stop scanning (previously promoted)
			break;
		} else if (isSemicolonEquivalent(lt)) {
			// We found our EOL: promote the token to on channel, position the input on it and reset the rule
			// start.
			lt.setChannel(Token.DEFAULT_CHANNEL);
			input.seek(idx);
			break;
		}
	}
}
 
Example 34
Project: n4js   File: SemicolonInjectionHelper.java   View source code 5 votes vote down vote up
/**
 * A "," cannot be followed by an automatically inserted semicolon. This is in particular true in case of variable
 * statements, in which the last declaration is ended with a comma (which might easily happen in case of copying the
 * initializer from a list or object literal (cf. IDEBUG-214).
 */
private static boolean findCommaBeforeEOL(TokenStream casted, int startIndex) {
	for (int ix = startIndex - 1; ix > 0; ix--) {
		Token lt = casted.get(ix);
		if (lt.getType() == InternalN4JSParser.Comma) {
			// System.out.println("Found Comma, EOL is not valid");
			return true;
		}
		if (lt.getChannel() == Token.DEFAULT_CHANNEL) { // any other real char ends this search
			break;
		}
	}
	return false;
}
 
Example 35
Project: n4js   File: RegExLiteralAwareLexer.java   View source code 5 votes vote down vote up
private void clearAndResetTokenState() {
	this.state.token = null;
	this.state.channel = Token.DEFAULT_CHANNEL;
	this.state.tokenStartCharIndex = input.index();
	this.state.tokenStartCharPositionInLine = input.getCharPositionInLine();
	this.state.tokenStartLine = input.getLine();
	this.state.text = null;
}
 
Example 36
Project: incubator-netbeans   File: SwitchNode.java   View source code 5 votes vote down vote up
public SwitchNode(Token t) {
    super(t);
    if (t != null) {
        setName(t.getText());
        setValue((OptionValue.SwitchOnly) OptionValue.createSwitch());
    }
}
 
Example 37
Project: incubator-netbeans   File: UnknownOption.java   View source code 5 votes vote down vote up
public UnknownOption(Token t) {
    super(t);
    if (t != null) {
        setName(t.getText());
        setValue(new OptionValue.SwitchOnly(true));
    }
}
 
Example 38
Project: incubator-netbeans   File: ParametrizedNode.java   View source code 5 votes vote down vote up
public ParametrizedNode(Token token, int splitIndex) {
    super(token);
    final String string = token.getText();
    if (string != null) {
        setName(string.substring(0, splitIndex));
        setValue(new OptionValue.SimpleString(string.substring(splitIndex)));
        delimiter = "";
        setValid(true);
    } else {
        setName("");
        setValid(false);
    }
}
 
Example 39
Project: incubator-netbeans   File: ParametrizedNode.java   View source code 5 votes vote down vote up
public ParametrizedNode(Token name, String delimiter, String parameter, boolean isValid) {
    super(name);
    setName(name.getText());
    this.delimiter = delimiter;
    if (parameter != null) {
        setValue(new OptionValue.SimpleString(parameter));
    }
    setValid(isValid);
}
 
Example 40
Project: incubator-netbeans   File: UnrecognizedOption.java   View source code 5 votes vote down vote up
public UnrecognizedOption(TokenStream input, Token start, Token stop, RecognitionException e) {
    super(start);        
    this.input = input;
    this.start = start;
    this.stop = stop;
    this.e = e;
    if (start != null) {
        setName(start.getText());
    }
    setValue(new OptionValue.SwitchOnly(true));
}