Java Code Examples for org.antlr.runtime.CommonToken#getTokenIndex()

The following examples show how to use org.antlr.runtime.CommonToken#getTokenIndex() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: InternalHighlightingParser.java    From n4js with Eclipse Public License 1.0 6 votes vote down vote up 
@Override
protected void announce(Token start, Token stop, AbstractElement element) {
	if (start != null && start != Token.EOF_TOKEN) {
		if (start == stop) {
			announce(start, element);
		} else {
			CommonToken castedStart = (CommonToken) start;
			if (stop == null) { // possible error condition
				if (start.getTokenIndex() == state.lastErrorIndex) {
					return;
				}
			}
			CommonToken castedEnd = (CommonToken) stop;
			Integer newType = rewriter.rewrite(castedStart, element);
			if (newType != null && castedEnd != null && castedEnd != Token.EOF_TOKEN) {
				LazyTokenStream castedInput = (LazyTokenStream) this.input;
				for (int i = castedStart.getTokenIndex() + 1; i < castedEnd.getTokenIndex(); i++) {
					Token token = castedInput.get(i);
					if (token.getChannel() != Token.HIDDEN_CHANNEL)
						token.setType(newType);
				}
				castedEnd.setType(newType);
			}
		}
	}
}
Example 2
Source File: CustomN4JSParser.java    From n4js with Eclipse Public License 1.0 5 votes vote down vote up 
/**
 * Prevent ASIs to be skipped at certain locations, e.g. after a return keyword.
 */
private boolean maySkipASI(CommonToken lastToken, ObservableXtextTokenStream tokens) {
	int countDownFrom = lastToken.getTokenIndex();
	for (int i = countDownFrom - 1; i >= 0; i--) {
		Token prevToken = tokens.get(i);
		if (prevToken.getChannel() == Token.DEFAULT_CHANNEL) {
			if (mandatoryASI.get(prevToken.getType())) {
				return false;
			}
			return true;
		}
	}
	return true;
}
Example 3
Source File: CustomN4JSParser.java    From n4js with Eclipse Public License 1.0 4 votes vote down vote up 
/**
 * Returns true if a synthetic ASI token should be added for a second pass. This is usually the case if the input
 * ends with a NL token.
 */
private boolean shouldAddSyntheticSemicolon(CustomInternalN4JSParser previousParser, int lastTokenIndex,
		CommonToken lastNonHiddenToken) {
	return lastTokenIndex != lastNonHiddenToken.getTokenIndex()
			&& previousParser.getState().lastErrorIndex != lastNonHiddenToken.getTokenIndex();
}