Java Code Examples for org.antlr.v4.runtime.Token#DEFAULT_CHANNEL
The following examples show how to use
org.antlr.v4.runtime.Token#DEFAULT_CHANNEL .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: PSITokenSource.java From antlr4-intellij-adaptor with BSD 2-Clause "Simplified" License | 6 votes |
|
/** Create an ANTLR Token from the current token type of the builder
* then advance the builder to next token (which ultimately calls an
* ANTLR lexer). The {@link ANTLRLexerAdaptor} creates tokens via
* an ANTLR lexer but converts to {@link TokenIElementType} and here
* we have to convert back to an ANTLR token using what info we
* can get from the builder. We lose info such as the original channel.
* So, whitespace and comments (typically hidden channel) will look like
* real tokens. Jetbrains uses {@link ParserDefinition#getWhitespaceTokens()}
* and {@link ParserDefinition#getCommentTokens()} to strip these before
* our ANTLR parser sees them.
*/
@Override
public Token nextToken() {
ProgressIndicatorProvider.checkCanceled();
TokenIElementType ideaTType = (TokenIElementType)builder.getTokenType();
int type = ideaTType!=null ? ideaTType.getANTLRTokenType() : Token.EOF;
int channel = Token.DEFAULT_CHANNEL;
Pair<TokenSource, CharStream> source = new Pair<TokenSource, CharStream>(this, null);
String text = builder.getTokenText();
int start = builder.getCurrentOffset();
int length = text != null ? text.length() : 0;
int stop = start + length - 1;
// PsiBuilder doesn't provide line, column info
int line = 0;
int charPositionInLine = 0;
Token t = tokenFactory.create(source, type, text, channel, start, stop, line, charPositionInLine);
builder.advanceLexer();
// System.out.println("TOKEN: "+t);
return t;
}
Example 2
| Source File: ECMAScriptLexer.java From rockscript with Apache License 2.0 | 5 votes |
|
/**
* Return the next token from the character stream and records this last
* token in case it resides on the default channel. This recorded token
* is used to determine when the lexer could possibly match a regex
* literal.
*
* @return the next token from the character stream.
*/
@Override
public Token nextToken() {
// Get the next token.
Token next = super.nextToken();
if (next.getChannel() == Token.DEFAULT_CHANNEL) {
// Keep track of the last token on the default channel.
this.lastToken = next;
}
return next;
}
Example 3
| Source File: JavaScriptBaseLexer.java From MSPaintIDE with MIT License | 5 votes |
|
/**
* Return the next token from the character stream and records this last
* token in case it resides on the default channel. This recorded token
* is used to determine when the lexer could possibly match a regex
* literal. Also changes scopeStrictModes stack if tokenize special
* string 'use strict';
*
* @return the next token from the character stream.
*/
@Override
public Token nextToken() {
Token next = super.nextToken();
if (next.getChannel() == Token.DEFAULT_CHANNEL) {
// Keep track of the last token on the default channel.
this.lastToken = next;
}
return next;
}
Example 4
| Source File: BatfishLexerRecoveryStrategy.java From batfish with Apache License 2.0 | 5 votes |
|
/**
* Wrap current unmatchable char up to next char in provided separator chars in a {@link
* BatfishLexer#UNMATCHABLE_TOKEN} and emit it.
*/
public void recover() {
// Always recover in the default mode -- otherwise, the parser can get stuck in an infinite
// loop, e.g. if separator is not valid in the current mode.
_lexer._mode = Lexer.DEFAULT_MODE;
int tokenStartMarker = _lexer._input.mark();
try {
_lexer._token = null;
_lexer._channel = Token.DEFAULT_CHANNEL;
_lexer._tokenStartCharIndex = _lexer._input.index();
_lexer._tokenStartCharPositionInLine = _lexer.getInterpreter().getCharPositionInLine();
_lexer._tokenStartLine = _lexer.getInterpreter().getLine();
_lexer._text = null;
_lexer._type = BatfishLexer.UNMATCHABLE_TOKEN;
for (int nextChar = _lexer._input.LA(1);
!_separatorChars.contains(nextChar);
nextChar = _lexer._input.LA(1)) {
if (nextChar == IntStream.EOF) {
_lexer._hitEOF = true;
_lexer.emitEOF();
return;
}
_lexer.getInterpreter().consume(_lexer._input);
}
_lexer.emit();
} finally {
// make sure we release marker after match or
// unbuffered char stream will keep buffering
_lexer._input.release(tokenStartMarker);
}
}
Example 5
| Source File: DelimiterLexer.java From presto with Apache License 2.0 | 4 votes |
|
@Override
public Token nextToken()
{
if (_input == null) {
throw new IllegalStateException("nextToken requires a non-null input stream.");
}
// Mark start location in char stream so unbuffered streams are
// guaranteed at least have text of current token
int tokenStartMarker = _input.mark();
try {
outer:
while (true) {
if (_hitEOF) {
emitEOF();
return _token;
}
_token = null;
_channel = Token.DEFAULT_CHANNEL;
_tokenStartCharIndex = _input.index();
_tokenStartCharPositionInLine = getInterpreter().getCharPositionInLine();
_tokenStartLine = getInterpreter().getLine();
_text = null;
do {
_type = Token.INVALID_TYPE;
int ttype = -1;
// This entire method is copied from org.antlr.v4.runtime.Lexer, with the following bit
// added to match the delimiters before we attempt to match the token
boolean found = false;
for (String terminator : delimiters) {
if (match(terminator)) {
ttype = SqlBaseParser.DELIMITER;
found = true;
break;
}
}
if (!found) {
try {
ttype = getInterpreter().match(_input, _mode);
}
catch (LexerNoViableAltException e) {
notifyListeners(e); // report error
recover(e);
ttype = SKIP;
}
}
if (_input.LA(1) == IntStream.EOF) {
_hitEOF = true;
}
if (_type == Token.INVALID_TYPE) {
_type = ttype;
}
if (_type == SKIP) {
continue outer;
}
}
while (_type == MORE);
if (_token == null) {
emit();
}
return _token;
}
}
finally {
// make sure we release marker after match or
// unbuffered char stream will keep buffering
_input.release(tokenStartMarker);
}
}
Example 6
| Source File: DelimiterLexer.java From rainbow with Apache License 2.0 | 4 votes |
|
@Override
public Token nextToken()
{
if (_input == null) {
throw new IllegalStateException("nextToken requires a non-null input stream.");
}
// Mark start location in char stream so unbuffered streams are
// guaranteed at least have text of current token
int tokenStartMarker = _input.mark();
try {
outer:
while (true) {
if (_hitEOF) {
emitEOF();
return _token;
}
_token = null;
_channel = Token.DEFAULT_CHANNEL;
_tokenStartCharIndex = _input.index();
_tokenStartCharPositionInLine = getInterpreter().getCharPositionInLine();
_tokenStartLine = getInterpreter().getLine();
_text = null;
do {
_type = Token.INVALID_TYPE;
int ttype = -1;
// This entire method is copied from org.antlr.v4.runtime.Lexer, with the following bit
// added to match the delimiters before we attempt to match the token
boolean found = false;
for (String terminator : delimiters) {
if (match(terminator)) {
ttype = SqlBaseParser.DELIMITER;
found = true;
break;
}
}
if (!found) {
try {
ttype = getInterpreter().match(_input, _mode);
}
catch (LexerNoViableAltException e) {
notifyListeners(e); // report error
recover(e);
ttype = SKIP;
}
}
if (_input.LA(1) == IntStream.EOF) {
_hitEOF = true;
}
if (_type == Token.INVALID_TYPE) {
_type = ttype;
}
if (_type == SKIP) {
continue outer;
}
}
while (_type == MORE);
if (_token == null) {
emit();
}
return _token;
}
}
finally {
// make sure we release marker after match or
// unbuffered char stream will keep buffering
_input.release(tokenStartMarker);
}
}
Example 7
| Source File: DelimiterLexer.java From macrobase with Apache License 2.0 | 4 votes |
|
@Override
public Token nextToken() {
if (_input == null) {
throw new IllegalStateException("nextToken requires a non-null input stream.");
}
// Mark start location in char stream so unbuffered streams are
// guaranteed at least have text of current token
int tokenStartMarker = _input.mark();
try {
outer:
while (true) {
if (_hitEOF) {
emitEOF();
return _token;
}
_token = null;
_channel = Token.DEFAULT_CHANNEL;
_tokenStartCharIndex = _input.index();
_tokenStartCharPositionInLine = getInterpreter().getCharPositionInLine();
_tokenStartLine = getInterpreter().getLine();
_text = null;
do {
_type = Token.INVALID_TYPE;
int ttype = -1;
// This entire method is copied from org.antlr.v4.runtime.Lexer, with the following bit
// added to match the delimiters before we attempt to match the token
boolean found = false;
for (String terminator : delimiters) {
if (match(terminator)) {
ttype = SqlBaseParser.DELIMITER;
found = true;
break;
}
}
if (!found) {
try {
ttype = getInterpreter().match(_input, _mode);
} catch (LexerNoViableAltException e) {
notifyListeners(e); // report error
recover(e);
ttype = SKIP;
}
}
if (_input.LA(1) == IntStream.EOF) {
_hitEOF = true;
}
if (_type == Token.INVALID_TYPE) {
_type = ttype;
}
if (_type == SKIP) {
continue outer;
}
}
while (_type == MORE);
if (_token == null) {
emit();
}
return _token;
}
} finally {
// make sure we release marker after match or
// unbuffered char stream will keep buffering
_input.release(tokenStartMarker);
}
}
