Java Code Examples for org.antlr.runtime.Token#getChannel()

@Override
protected void announce(Token start, Token stop, AbstractElement element) {
	if (start != null && start != Token.EOF_TOKEN) {
		if (start == stop) {
			announce(start, element);
		} else {
			CommonToken castedStart = (CommonToken) start;
			if (stop == null) { // possible error condition
				if (start.getTokenIndex() == state.lastErrorIndex) {
					return;
				}
			}
			CommonToken castedEnd = (CommonToken) stop;
			Integer newType = rewriter.rewrite(castedStart, element);
			if (newType != null && castedEnd != null && castedEnd != Token.EOF_TOKEN) {
				LazyTokenStream castedInput = (LazyTokenStream) this.input;
				for (int i = castedStart.getTokenIndex() + 1; i < castedEnd.getTokenIndex(); i++) {
					Token token = castedInput.get(i);
					if (token.getChannel() != Token.HIDDEN_CHANNEL)
						token.setType(newType);
				}
				castedEnd.setType(newType);
			}
		}
	}
}
 

/**
 * Returns true if there was an unexpected EOL.
 */
public static boolean hasDisallowedEOL(Callback callback) {
	TokenStream input = callback.getInput();
	Token lt = input.LT(1);

	// Start on the position before the current token and scan backwards off channel tokens until the previous on
	// channel token.
	for (int ix = lt.getTokenIndex() - 1; ix > 0; ix--) {
		lt = input.get(ix);
		if (lt.getChannel() == Token.DEFAULT_CHANNEL) {
			// On channel token found: stop scanning.
			break;
		} else if (isSemicolonEquivalent(lt)) {
			return true;
		}
	}
	return false;
}
 

@Test
public void TestLexer() throws IOException {
    CharStream input = new QueryParserFileStream( "test/org/apache/pig/parser/TestLexer.pig" );
    QueryLexer lexer = new QueryLexer( input );
    int tokenCount = 0;
    Token token;
    while( ( token = lexer.nextToken() ).getType() != Token.EOF ) {
        if( token.getChannel() == Token.HIDDEN_CHANNEL )
            continue;
        tokenCount++;
        if( token.getText().equals( ";" ) ) {
            System.out.println( token.getText() );
        } else {
            System.out.print( token.getText() + "(" + token.getType() + ") " );
        }
    }

    // While we can check more conditions, such as type of each token, for now I think the following
    // is enough. If the token type is wrong, it will be most likely caught by the parser.
    Assert.assertEquals( 455, tokenCount );
    Assert.assertEquals( 0, lexer.getNumberOfSyntaxErrors() );
}
 

@Test
public void test2() throws IOException {
    String query = "A = load 'input' using PigStorage(';');" +
                   "B = foreach ^ A generate string.concatsep( ';', $1, $2 );";
    CharStream input = new QueryParserStringStream( query, null );
    QueryLexer lexer = new QueryLexer( input );
    Token token;
    try {
        while( ( token = lexer.nextToken() ).getType() != Token.EOF ) {
            if( token.getChannel() == Token.HIDDEN_CHANNEL )
                continue;
            if( token.getText().equals( ";" ) ) {
                System.out.println( token.getText() );
            } else {
                System.out.print( token.getText() + "(" + token.getType() + ") " );
            }
        }
    } catch(Exception ex) {
        Assert.assertTrue( ex.getMessage().contains( "Unexpected character" ) );
        return;
    }
    Assert.fail( "Query should fail." );
}
 

private String getDoc(CommonTree ast, String prefix) {
    CommonTokenStream ts = ((TarsRoot) ast.getAncestor(TarsParser.TARS_ROOT)).getTokenStream();
    Token t = ts.get(ast.getTokenStartIndex() - 1);
    if (t != null && t.getChannel() == Token.HIDDEN_CHANNEL) {
        return prefix + t.getText().replaceAll("\n\\s*\\*", "\n" + prefix + " *");
    } else {
        return "";
    }
}
 

/**
 * Prevent ASIs to be skipped at certain locations, e.g. after a return keyword.
 */
private boolean maySkipASI(CommonToken lastToken, ObservableXtextTokenStream tokens) {
	int countDownFrom = lastToken.getTokenIndex();
	for (int i = countDownFrom - 1; i >= 0; i--) {
		Token prevToken = tokens.get(i);
		if (prevToken.getChannel() == Token.DEFAULT_CHANNEL) {
			if (mandatoryASI.get(prevToken.getType())) {
				return false;
			}
			return true;
		}
	}
	return true;
}
 

/**
 * Adds token and adjust channel.
 */
@Override
public boolean add(Token tok) {
	super.add(tok);
	int type = tok.getType();
	if (type == InternalN4JSParser.EqualsSignGreaterThanSign) {
		// The arrow expression may not follow a semicolon thus we promote those here
		// to the default channel if they precede the arrow => operator
		for (int i = size() - 2; i >= 0; i--) {
			Token prev = get(i);
			if (prev.getChannel() == Token.HIDDEN_CHANNEL) {
				if (SemicolonInjectionHelper.isSemicolonEquivalent(prev)) {
					prev.setChannel(Token.DEFAULT_CHANNEL);
					break;
				}
			} else {
				break;
			}
		}
	} else if (type == InternalN4JSParser.RULE_EOL
			|| type == InternalN4JSParser.RULE_ML_COMMENT
			|| type == InternalN4JSParser.RULE_WS
			|| type == InternalN4JSParser.RULE_SL_COMMENT) {
		tok.setChannel(Token.HIDDEN_CHANNEL);
	} else {
		tok.setChannel(Token.DEFAULT_CHANNEL);
	}
	return true;
}
 

/**
 * <p>
 * Promotes EOL which may lead to an automatically inserted semicolon. This is probably the most important method
 * for automatic semicolon insertion, as it is only possible to insert a semicolon in case of line breaks (even if
 * they are hidden in a multi-line comment!).
 * </p>
 */
public static void promoteEOL(Callback callback) {
	RecognizerSharedState state = callback.getState();
	TokenStream input = callback.getInput();
	// Don't promote EOL if there was a syntax error at EOF
	if (state.lastErrorIndex == input.size()) {
		return;
	}
	// Get current token and its type (the possibly offending token).
	Token prev = input.LT(-1);
	Token next = input.LT(1);
	int la = next.getType();
	if (la == InternalN4JSParser.Semicolon) {
		return;
	}

	// Promoting an EOL means switching it from off channel to on channel.
	// A ML_COMMENT gets promoted when it contains an EOL.
	for (int idx = prev == null ? 0 : prev.getTokenIndex() + 1, max = la == Token.EOF ? input.size()
			: next.getTokenIndex(); idx < max; idx++) {
		Token lt = input.get(idx);
		if (lt.getChannel() == Token.DEFAULT_CHANNEL) {
			// On channel token found: stop scanning (previously promoted)
			break;
		} else if (isSemicolonEquivalent(lt)) {
			// We found our EOL: promote the token to on channel, position the input on it and reset the rule
			// start.
			lt.setChannel(Token.DEFAULT_CHANNEL);
			input.seek(idx);
			break;
		}
	}
}
 

/**
 * A "," cannot be followed by an automatically inserted semicolon. This is in particular true in case of variable
 * statements, in which the last declaration is ended with a comma (which might easily happen in case of copying the
 * initializer from a list or object literal (cf. IDEBUG-214).
 */
private static boolean findCommaBeforeEOL(TokenStream casted, int startIndex) {
	for (int ix = startIndex - 1; ix > 0; ix--) {
		Token lt = casted.get(ix);
		if (lt.getType() == InternalN4JSParser.Comma) {
			// System.out.println("Found Comma, EOL is not valid");
			return true;
		}
		if (lt.getChannel() == Token.DEFAULT_CHANNEL) { // any other real char ends this search
			break;
		}
	}
	return false;
}
 

private ILeafNode createLeafNode(Token token, EObject grammarElement) {
	boolean isHidden = token.getChannel() == HIDDEN;
	SyntaxErrorMessage error = null;
	if (!isHidden) {
		if (currentError != null) {
			error = currentError;
			currentError = null;
		}
	}
	if (token.getType() == Token.INVALID_TOKEN_TYPE) {
		if (error == null) {
			String lexerErrorMessage = ((XtextTokenStream) input).getLexerErrorMessage(token);
			LexerErrorContext errorContext = new LexerErrorContext(lexerErrorMessage);
			error = syntaxErrorProvider.getSyntaxErrorMessage(errorContext);
		}
	}
	if (grammarElement == null) {
		String ruleName = antlrTypeToLexerName.get(token.getType());
		grammarElement = allRules.get(ruleName);
	}
	CommonToken commonToken = (CommonToken) token;
	if (error != null)
		hadErrors = true;
	return nodeBuilder.newLeafNode(
			commonToken.getStartIndex(), 
			commonToken.getStopIndex() - commonToken.getStartIndex() + 1, 
			grammarElement, 
			isHidden, 
			error, 
			currentNode);
}
 

protected void skipHiddenTokens() {
	if (hiddenTokens == null || hiddenTokens.isEmpty())
		return;
	Token token = LT(1);
	while(token.getChannel() == Token.HIDDEN_CHANNEL) {
		p++;
		token = LT(1);
	}
}
 

/**
 * Recover from an error found on the input stream. This is for {@link NoViableAltException} and
 * {@link MismatchedTokenException}. If you enable single token insertion and deletion, this will usually not handle
 * mismatched symbol exceptions but there could be a mismatched token that the
 * {@link Parser#match(IntStream, int, BitSet) match} routine could not recover from.
 */
public static void recover(IntStream inputStream, RecognitionException re, Callback callback) {
	RecognizerSharedState state = callback.getState();
	if (re instanceof MismatchedTokenException) {
		// We expected a specific token
		// if that is not a semicolon, ASI is pointless, perform normal recovery
		int expecting = ((MismatchedTokenException) re).expecting;
		if (expecting != InternalN4JSParser.Semicolon) {

			callback.discardError(); // delete ASI message, a previously added ASI may fix too much! cf.
			// IDEBUG-215
			callback.recoverBase(inputStream, re);
			return;
		}
	}

	// System.out.println("Line: " + re.line + ":" + re.index);

	int unexpectedTokenType = re.token.getType();
	if (!followedBySemicolon(state, callback.getRecoverySets(), re.index)
			|| isOffendingToken(unexpectedTokenType)) {
		callback.recoverBase(inputStream, re);
	} else {
		int la = inputStream.LA(1);
		TokenStream casted = (TokenStream) inputStream;
		if (!isOffendingToken(la)) {
			// Start on the position before the current token and scan backwards off channel tokens until the
			// previous on channel token.
			for (int ix = re.token.getTokenIndex() - 1; ix > 0; ix--) {
				Token lt = casted.get(ix);
				if (lt.getChannel() == Token.DEFAULT_CHANNEL) {
					// On channel token found: stop scanning.
					callback.recoverBase(inputStream, re);
					return;
				} else if (lt.getType() == InternalN4JSParser.RULE_EOL) {
					// We found our EOL: everything's good, no need to do additional recovering
					// rule start.
					if (!callback.allowASI(re)) {
						callback.recoverBase(inputStream, re);
						return;
					}
					if (!findCommaBeforeEOL(casted, ix)) {
						callback.addASIMessage();
						return;
					}
				} else if (lt.getType() == InternalN4JSParser.RULE_ML_COMMENT) {
					String tokenText = lt.getText();
					if (!findCommaBeforeEOL(casted, ix)
							&& (tokenText.indexOf('\n', 2) >= 2 || tokenText.indexOf('\r', 2) >= 2)) {
						callback.addASIMessage();
						return;
					}
				}
			}
			callback.recoverBase(inputStream, re);
		}
	}
}
 

/**
 * Fills the buffer but stops on a div or div-equals token.
 */
@SuppressWarnings("unchecked")
@Override
protected void fillBuffer() {
	int oldP = p;
	int index = tokens.size();
	Token t = tokenSource.nextToken();
	while (t != null && t.getType() != CharStream.EOF) {
		// stop on div, div-equal and right curly brace tokens.
		int type = t.getType();
		if (type == InternalN4JSLexer.Solidus || type == InternalN4JSLexer.SolidusEqualsSign
				|| type == InternalN4JSLexer.RightCurlyBracket) {
			t.setTokenIndex(index);
			tokens.add(t);
			index++;
			break;
		}
		boolean discard = false;
		// is there a channel override for token type?
		if (channelOverrideMap != null) {
			Integer channelI = (Integer) channelOverrideMap.get(Integer.valueOf(type));
			if (channelI != null) {
				t.setChannel(channelI.intValue());
			}
		}
		if (discardSet != null &&
				discardSet.contains(Integer.valueOf(type))) {
			discard = true;
		} else if (discardOffChannelTokens && t.getChannel() != this.channel) {
			discard = true;
		}
		if (!discard) {
			t.setTokenIndex(index);
			tokens.add(t);
			index++;
		}
		t = tokenSource.nextToken();
	}
	// leave p pointing at first token on channel
	p = oldP == -1 ? 0 : oldP;
	p = skipOffTokenChannels(p);
}