org.eclipse.jdt.core.compiler.InvalidInputException Java Examples

protected Object syntaxRecoverQualifiedName(int primitiveToken) throws InvalidInputException {
	if (this.cursorLocation == ((int)this.identifierPositionStack[this.identifierPtr])) {
		// special case of completion just before the dot.
		return createTypeReference(primitiveToken);
	}
	int idLength = this.identifierLengthStack[this.identifierLengthPtr];
	char[][] tokens = new char[idLength][];
	int startPtr = this.identifierPtr-idLength+1;
	System.arraycopy(this.identifierStack, startPtr, tokens, 0, idLength);
	long[] positions = new long[idLength+1];
	System.arraycopy(this.identifierPositionStack, startPtr, positions, 0, idLength);
	positions[idLength] = (((long)this.tokenPreviousPosition)<<32) + this.tokenPreviousPosition;
	this.completionNode = new CompletionOnJavadocQualifiedTypeReference(tokens, CharOperation.NO_CHAR, positions, this.tagSourceStart, this.tagSourceEnd);

	if (CompletionEngine.DEBUG) {
		System.out.println("	completion partial qualified type="+this.completionNode); //$NON-NLS-1$
	}
	return this.completionNode;
}
 

@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
	final PublicScanner scanner = prepareScanner();
	final SortedMap<Integer, String> tokens = Maps.newTreeMap();
	tokens.put(-1, SENTENCE_START);
	tokens.put(Integer.MAX_VALUE, SENTENCE_END);
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				}
				final String nxtToken = transformToken(token,
						scanner.getCurrentTokenString());
				final int position = scanner.getCurrentTokenStartPosition();
				tokens.put(position, stripTokenIfNeeded(nxtToken));
			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}
 

protected boolean parseTag(int previousPosition) throws InvalidInputException {
	int startPosition = this.inlineTagStarted ? this.inlineTagStart : previousPosition;
	boolean newLine = !this.lineStarted;
	boolean valid = super.parseTag(previousPosition);
	boolean inCompletion = (this.tagSourceStart <= (this.cursorLocation+1) && this.cursorLocation <= this.tagSourceEnd) // completion cursor is between first and last stacked identifiers
		|| ((this.tagSourceStart == (this.tagSourceEnd+1) && this.tagSourceEnd == this.cursorLocation)); // or it's a completion on empty token
	if (inCompletion) {
		int end = this.tagSourceEnd;
		if (this.inlineTagStarted && this.scanner.currentCharacter == '}') {
			end = this.scanner.currentPosition;
		}
		long position = (((long)startPosition)<<32) + end;
		int length = this.cursorLocation+1-this.tagSourceStart;
		char[] tag = new char[length];
		System.arraycopy(this.source, this.tagSourceStart, tag, 0, length);
		char[][][] tags = possibleTags(tag, newLine);
		if (tags != null) {
			this.completionNode = new CompletionOnJavadocTag(tag, position, startPosition, end, tags, this.allPossibleTags);
		}
	}
	return valid;
}
 

@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
	final PublicScanner scanner = prepareScanner();
	final SortedMap<Integer, String> tokens = Maps.newTreeMap();
	tokens.put(-1, SENTENCE_START);
	tokens.put(Integer.MAX_VALUE, SENTENCE_END);
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				}
				final String nxtToken = transformToken(token,
						scanner.getCurrentTokenString());
				final int position = scanner.getCurrentTokenStartPosition();
				tokens.put(position, stripTokenIfNeeded(nxtToken));
			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}
 

/**
 * Sets the string value of this literal node. The value is the sequence
 * of characters that would appear in the source program, including
 * enclosing single quotes and embedded escapes. For example,
 * <ul>
 * <li><code>'a'</code> <code>setEscapedValue("\'a\'")</code></li>
 * <li><code>'\n'</code> <code>setEscapedValue("\'\\n\'")</code></li>
 * </ul>
 *
 * @param value the string value, including enclosing single quotes
 *    and embedded escapes
 * @exception IllegalArgumentException if the argument is incorrect
 */
public void setEscapedValue(String value) {
	// check setInternalEscapedValue(String) if this method is changed
	if (value == null) {
		throw new IllegalArgumentException();
	}
	Scanner scanner = this.ast.scanner;
	char[] source = value.toCharArray();
	scanner.setSource(source);
	scanner.resetTo(0, source.length);
	try {
		int tokenType = scanner.getNextToken();
		switch(tokenType) {
			case TerminalTokens.TokenNameCharacterLiteral:
				break;
			default:
				throw new IllegalArgumentException();
		}
	} catch(InvalidInputException e) {
		throw new IllegalArgumentException();
	}
	preValueChange(ESCAPED_VALUE_PROPERTY);
	this.escapedValue = value;
	postValueChange(ESCAPED_VALUE_PROPERTY);
}
 

@Override
public List<String> tokenListFromCode(final char[] code) {
	final List<String> tokens = Lists.newArrayList();
	tokens.add(SENTENCE_START);
	final PublicScanner scanner = prepareScanner(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			}
			tokens.addAll(getConvertedToken(scanner, token));
		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(SENTENCE_END);
	return tokens;
}
 

/**
 * Removes comments and whitespace
 * @param reference the type reference
 * @return the reference only consisting of dots and java identifier characters
 */
public static String normalizeReference(String reference) {
	IScanner scanner= ToolFactory.createScanner(false, false, false, false);
	scanner.setSource(reference.toCharArray());
	StringBuffer sb= new StringBuffer();
	try {
		int tokenType= scanner.getNextToken();
		while (tokenType != ITerminalSymbols.TokenNameEOF) {
			sb.append(scanner.getRawTokenSource());
			tokenType= scanner.getNextToken();
		}
	} catch (InvalidInputException e) {
		Assert.isTrue(false, reference);
	}
	reference= sb.toString();
	return reference;
}
 

public void printNextToken(int[] expectedTokenTypes, boolean considerSpaceIfAny){
	printComment(CodeFormatter.K_UNKNOWN, NO_TRAILING_COMMENT);
	try {
		this.currentToken = this.scanner.getNextToken();
		if (Arrays.binarySearch(expectedTokenTypes, this.currentToken) < 0) {
			StringBuffer expectations = new StringBuffer(5);
			for (int i = 0; i < expectedTokenTypes.length; i++){
				if (i > 0) {
					expectations.append(',');
				}
				expectations.append(expectedTokenTypes[i]);
			}
			throw new AbortFormatting("unexpected token type, expecting:["+expectations.toString()+"], actual:"+this.currentToken);//$NON-NLS-1$//$NON-NLS-2$
		}
		print(this.scanner.currentPosition - this.scanner.startPosition, considerSpaceIfAny);
	} catch (InvalidInputException e) {
		throw new AbortFormatting(e);
	}
}
 

protected Object createFieldReference(Object receiver) throws InvalidInputException {
	int refStart = (int) (this.identifierPositionStack[0] >>> 32);
	int refEnd = (int) this.identifierPositionStack[0];
	boolean inCompletion = (refStart <= (this.cursorLocation+1) && this.cursorLocation <= refEnd) // completion cursor is between first and last stacked identifiers
		|| ((refStart == (refEnd+1) && refEnd == this.cursorLocation)) // or it's a completion on empty token
		|| (this.memberStart == this.cursorLocation); // or it's a completion just after the member separator with an identifier after the cursor
	if (inCompletion) {
		JavadocFieldReference fieldRef = (JavadocFieldReference) super.createFieldReference(receiver);
		char[] name = this.sourceParser.compilationUnit.getMainTypeName();
		TypeDeclaration typeDecl = getParsedTypeDeclaration();
		if (typeDecl != null) {
			name = typeDecl.name;
		}
		this.completionNode = new CompletionOnJavadocFieldReference(fieldRef, this.memberStart, name);
		if (CompletionEngine.DEBUG) {
			System.out.println("	completion field="+this.completionNode); //$NON-NLS-1$
		}
		return this.completionNode;
	}
	return super.createFieldReference(receiver);
}
 

protected boolean parseThrows() {
	int start = this.scanner.currentPosition;
	try {
		Object typeRef = parseQualifiedName(true);
		if (this.abort) return false; // May be aborted by specialized parser
		if (typeRef == null) {
			if (this.reportProblems)
				this.sourceParser.problemReporter().javadocMissingThrowsClassName(this.tagSourceStart, this.tagSourceEnd, this.sourceParser.modifiers);
		} else {
			return pushThrowName(typeRef);
		}
	} catch (InvalidInputException ex) {
		if (this.reportProblems) this.sourceParser.problemReporter().javadocInvalidThrowsClass(start, getTokenEndPosition());
	}
	return false;
}
 

@Override
public List<String> tokenListFromCode(final char[] code) {
	final List<String> tokens = Lists.newArrayList();
	tokens.add(SENTENCE_START);
	final PublicScanner scanner = prepareScanner(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			}
			tokens.addAll(getConvertedToken(scanner, token));
		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(SENTENCE_END);
	return tokens;
}
 

@Override
public List<FullToken> getTokenListFromCode(final char[] code) {
	final List<FullToken> tokens = Lists.newArrayList();
	tokens.add(new FullToken(SENTENCE_START, SENTENCE_START));
	final PublicScanner scanner = prepareScanner(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			}
			for (final String cToken : getConvertedToken(scanner, token)) {
				tokens.add(new FullToken(cToken, ""));
			}
		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(new FullToken(SENTENCE_END, SENTENCE_END));
	return tokens;
}
 

/**
 * @param source
 *            must be not null
 * @param range
 *            can be null
 * @return may return null, otherwise an initialized scanner which may
 *         answer which source offset index belongs to which source line
 * @throws JavaModelException
 */
private static IScanner initScanner(IType source, ISourceRange range) throws JavaModelException {
    if (range == null) {
        return null;
    }
    char[] charContent = getContent(source);
    if (charContent == null) {
        return null;
    }
    IScanner scanner = ToolFactory.createScanner(false, false, false, true);
    scanner.setSource(charContent);
    int offset = range.getOffset();
    try {
        while (scanner.getNextToken() != ITerminalSymbols.TokenNameEOF) {
            // do nothing, just wait for the end of stream
            if (offset <= scanner.getCurrentTokenEndPosition()) {
                break;
            }
        }
    } catch (InvalidInputException e) {
        FindbugsPlugin.getDefault().logException(e, "Could not init scanner for type: " + source);
    }
    return scanner;
}
 

protected Object createArgumentReference(char[] name, int dim, boolean isVarargs, Object typeRef, long[] dimPositions, long argNamePos) throws InvalidInputException {
	// Create argument as we may need it after
	char[] argName = name==null ? CharOperation.NO_CHAR : name;
	Expression expression = (Expression) super.createArgumentReference(argName, dim, isVarargs, typeRef, dimPositions, argNamePos);
	// See if completion location is in argument
	int refStart = ((TypeReference)typeRef).sourceStart;
	int refEnd = ((TypeReference)typeRef).sourceEnd;
	boolean inCompletion = (refStart <= this.cursorLocation && this.cursorLocation <= refEnd) // completion cursor is between first and last stacked identifiers
		|| ((refStart == (refEnd+1) && refEnd == this.cursorLocation)); // or it's a completion on empty token
	if (this.completionNode == null && inCompletion) {
		JavadocArgumentExpression javadocArgument = (JavadocArgumentExpression) expression;
		TypeReference expressionType = javadocArgument.argument.type;
		if (expressionType instanceof JavadocSingleTypeReference) {
			this.completionNode = new CompletionOnJavadocSingleTypeReference((JavadocSingleTypeReference) expressionType);
		} else if (expressionType instanceof JavadocQualifiedTypeReference) {
			this.completionNode = new CompletionOnJavadocQualifiedTypeReference((JavadocQualifiedTypeReference) expressionType);
		}
		if (CompletionEngine.DEBUG) {
			System.out.println("	completion argument="+this.completionNode); //$NON-NLS-1$
		}
		return this.completionNode;
	}
	return expression;
}
 

private final int consumeDigits0(int radix, int usingUnderscore, int invalidPosition, boolean expectingDigitFirst) throws InvalidInputException {
	int kind = 0;
	if (getNextChar('_')) {
		if (expectingDigitFirst) {
			return invalidPosition;
		}
		kind = usingUnderscore;
		while (getNextChar('_')) {/*empty */}
	}
	if (getNextCharAsDigit(radix)) {
		// continue to read digits or underscore
		while (getNextCharAsDigit(radix)) {/*empty */}
		int kind2 = consumeDigits0(radix, usingUnderscore, invalidPosition, false);
		if (kind2 == 0) {
			return kind;
		}
		return kind2;
	}
	if (kind == usingUnderscore) return invalidPosition;
	return kind;
}
 

@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
	final PublicScanner scanner = prepareScanner();
	final SortedMap<Integer, String> tokens = Maps.newTreeMap();
	tokens.put(-1, SENTENCE_START);
	tokens.put(Integer.MAX_VALUE, SENTENCE_END);
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				}
				final String nxtToken = transformToken(token,
						scanner.getCurrentTokenString());
				final int position = scanner.getCurrentTokenStartPosition();
				tokens.put(position, stripTokenIfNeeded(nxtToken));
			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}
 

private int getLastSimpleNameStart(String reference) {
	fScanner.setSource(reference.toCharArray());
	int lastIdentifierStart = -1;
	try {
		int tokenType = fScanner.getNextToken();
		while (tokenType != ITerminalSymbols.TokenNameEOF) {
			if (tokenType == ITerminalSymbols.TokenNameIdentifier) {
				lastIdentifierStart = fScanner.getCurrentTokenStartPosition();
			}
			tokenType = fScanner.getNextToken();
		}
	} catch (InvalidInputException e) {
		JavaLanguageServerPlugin.logException(e.getMessage(), e);
	}
	return lastIdentifierStart;
}
 

private int getSurroundingComment(IScanner scanner) {
	try {
		int start= fLocation.getOffset();
		int end= start + fLocation.getLength();

		int token= scanner.getNextToken();
		while (token != ITerminalSymbols.TokenNameEOF) {
			if (TokenScanner.isComment(token)) {
				int currStart= scanner.getCurrentTokenStartPosition();
				int currEnd= scanner.getCurrentTokenEndPosition() + 1;
				if (currStart <= start && end <= currEnd) {
					return token;
				}
			}
			token= scanner.getNextToken();
		}

	} catch (InvalidInputException e) {
		// ignore
	}
	return ITerminalSymbols.TokenNameEOF;
}
 

protected Object syntaxRecoverEmptyArgumentType(Object methodRef) throws InvalidInputException {
	if (methodRef instanceof JavadocMessageSend) {
		JavadocMessageSend msgSend = (JavadocMessageSend) methodRef;
		if (this.index > this.cursorLocation) {
			msgSend.sourceEnd = this.tokenPreviousPosition-1;
		}
		this.completionNode = new CompletionOnJavadocMessageSend(msgSend, this.memberStart);
	} else if (methodRef instanceof JavadocAllocationExpression) {
		JavadocAllocationExpression allocExp = (JavadocAllocationExpression) methodRef;
		if (this.index > this.cursorLocation) {
			allocExp.sourceEnd = this.tokenPreviousPosition-1;
		}
		this.completionNode = new CompletionOnJavadocAllocationExpression(allocExp, this.memberStart);
	}
	if (CompletionEngine.DEBUG) {
		System.out.println("	completion method="+this.completionNode); //$NON-NLS-1$
	}
	return this.completionNode;
}
 

private final int consumeDigits0(int radix, int usingUnderscore, int invalidPosition, boolean expectingDigitFirst) throws InvalidInputException {
	int kind = 0;
	if (getNextChar('_')) {
		if (expectingDigitFirst) {
			return invalidPosition;
		}
		kind = usingUnderscore;
		while (getNextChar('_')) {/*empty */}
	}
	if (getNextCharAsDigit(radix)) {
		// continue to read digits or underscore
		while (getNextCharAsDigit(radix)) {/*empty */}
		int kind2 = consumeDigits0(radix, usingUnderscore, invalidPosition, false);
		if (kind2 == 0) {
			return kind;
		}
		return kind2;
	}
	if (kind == usingUnderscore) return invalidPosition;
	return kind;
}
 

private boolean isValidComment(String template) {
	IScanner scanner= ToolFactory.createScanner(true, false, false, false);
	scanner.setSource(template.toCharArray());
	try {
		int next= scanner.getNextToken();
		while (TokenScanner.isComment(next)) {
			next= scanner.getNextToken();
		}
		return next == ITerminalSymbols.TokenNameEOF;
	} catch (InvalidInputException e) {
	}
	return false;
}
 

/**
 * Reads the next token.
 * @param ignoreComments If set, comments will be overread.
 * @return Return the token id.
 * @exception CoreException Thrown when the end of the file has been reached (code END_OF_FILE)
 * or a lexical error was detected while scanning (code LEXICAL_ERROR)
 */
private int readNextWithEOF(boolean ignoreComments) throws CoreException {
	int curr= 0;
	do {
		try {
			curr= fScanner.getNextToken();
		} catch (InvalidInputException e) {
			throw new CoreException(createError(LEXICAL_ERROR, e.getMessage(), e));
		}
	} while (ignoreComments && isComment(curr));
	return curr;
}
 

protected Object createMethodReference(Object receiver, List arguments) throws InvalidInputException {
	int memberPtr = this.identifierLengthStack[0] - 1;	// may be > 0 for inner class constructor reference
	int start = (int) (this.identifierPositionStack[memberPtr] >>> 32);
	int end = (int) this.identifierPositionStack[memberPtr];
	if (start <= this.selectionStart && this.selectionEnd <= end) {
		this.selectedNode = (ASTNode) super.createMethodReference(receiver, arguments);
		this.abort = true;
		if (SelectionEngine.DEBUG) {
			System.out.println("	selected method="+this.selectedNode); //$NON-NLS-1$
		}
	}
	return null;
}
 

private boolean isValidComment(String template) {
	IScanner scanner= ToolFactory.createScanner(true, false, false, false);
	scanner.setSource(template.toCharArray());
	try {
		int next= scanner.getNextToken();
		while (TokenScanner.isComment(next)) {
			next= scanner.getNextToken();
		}
		return next == ITerminalSymbols.TokenNameEOF;
	} catch (InvalidInputException e) {
	}
	return false;
}
 

protected Object createFieldReference(Object receiver) throws InvalidInputException {
	int start = (int) (this.identifierPositionStack[0] >>> 32);
	int end = (int) this.identifierPositionStack[0];
	if (start <= this.selectionStart && this.selectionEnd <= end) {
		this.selectedNode = (ASTNode) super.createFieldReference(receiver);
		this.abort = true;
		if (SelectionEngine.DEBUG) {
			System.out.println("	selected field="+this.selectedNode); //$NON-NLS-1$
		}
	}
	return null;
}
 

/**
 * Creates a token comparator for the given string.
 *
 * @param text the text to be tokenized
 */
public JavaTokenComparator(String text) {
	Assert.isLegal(text != null);

	fText= text;

	int length= fText.length();
	fStarts= new int[length];
	fLengths= new int[length];
	fCount= 0;

	IScanner scanner= ToolFactory.createScanner(true, true, false, false); // returns comments & whitespace
	scanner.setSource(fText.toCharArray());
	int endPos= 0;
	try {
		int tokenType;
		while ((tokenType= scanner.getNextToken()) != ITerminalSymbols.TokenNameEOF) {
			int start= scanner.getCurrentTokenStartPosition();
			int end= scanner.getCurrentTokenEndPosition()+1;
			// Comments and strings should not be treated as a single token, see https://bugs.eclipse.org/78063
			if (TokenScanner.isComment(tokenType) || tokenType == ITerminalSymbols.TokenNameStringLiteral) {
				// Line comments are often commented code, so lets treat them as code. See https://bugs.eclipse.org/216707
				boolean parseAsJava= tokenType == ITerminalSymbols.TokenNameCOMMENT_LINE;
				int dl= parseAsJava ? getCommentStartTokenLength(tokenType) : 0;
				if (dl > 0)
					recordTokenRange(start, dl);
				parseSubrange(start + dl, text.substring(start + dl, end), parseAsJava);
			} else {
				recordTokenRange(start, end - start);
			}
			endPos= end;
		}
	} catch (InvalidInputException ex) {
		// We couldn't parse part of the input. Fall through and make the rest a single token
	}
	// Workaround for https://bugs.eclipse.org/bugs/show_bug.cgi?id=13907
	if (endPos < length) {
		recordTokenRange(endPos, length - endPos);
	}
}
 

protected boolean parseThrows() {
	try {
		Object typeRef = parseQualifiedName(true);
		if (this.completionNode != null) {
			this.completionNode.addCompletionFlags(CompletionOnJavadoc.EXCEPTION);
		}
		return pushThrowName(typeRef);
	} catch (InvalidInputException ex) {
		// ignore
	}
	return false;
}
 

protected int readToken() throws InvalidInputException {
	int token = super.readToken();
	if (token == TerminalTokens.TokenNameIdentifier && this.scanner.currentPosition == this.scanner.startPosition) {
		// Scanner is looping on empty token => read it...
		this.scanner.getCurrentIdentifierSource();
	}
	return token;
}
 

@Override
public SortedMap<Integer, FullToken> fullTokenListWithPos(final char[] code) {
	// TODO Duplicate Code
	final PublicScanner scanner = prepareScanner();
	final SortedMap<Integer, FullToken> tokens = Maps.newTreeMap();
	tokens.put(-1, new FullToken(SENTENCE_START, SENTENCE_START));
	tokens.put(Integer.MAX_VALUE, new FullToken(SENTENCE_END, SENTENCE_END));
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				}
				final String nxtToken = transformToken(token,
						scanner.getCurrentTokenString());
				final int position = scanner.getCurrentTokenStartPosition();
				tokens.put(position,
						new FullToken(nxtToken, Integer.toString(token)));
			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}
 

/**
 * @param tokens
 * @param scanner
 * @param token
 * @return
 * @throws InvalidInputException
 */
private List<String> getConvertedToken(final PublicScanner scanner,
		final int token) throws InvalidInputException {
	final List<String> tokens = Lists.newArrayList();
	if (token == ITerminalSymbols.TokenNameEOF) {
		return Collections.emptyList();
	}
	final String tokenString = scanner.getCurrentTokenString();

	if (token == ITerminalSymbols.TokenNameWHITESPACE) {
		tokens.add(whitespaceConverter.toWhiteSpaceSymbol(tokenString));
	} else if (token == ITerminalSymbols.TokenNameIdentifier) {
		tokens.add(toIdentifierSymbol(tokenString));
	} else if (JavaTokenTypeTokenizer.isLiteralToken(token)) {
		tokens.add(toLiteralSymbol(tokenString));
	} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
		tokens.add(JavaTokenTypeTokenizer.COMMENT_BLOCK);
	} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
		tokens.add(JavaTokenTypeTokenizer.COMMENT_LINE);
		final int nextToken = scanner.getNextToken();
		if (nextToken == ITerminalSymbols.TokenNameWHITESPACE) {
			tokens.add(whitespaceConverter.toWhiteSpaceSymbol("\n"
					+ scanner.getCurrentTokenString()));
		} else {
			tokens.add(whitespaceConverter.toWhiteSpaceSymbol("\n"));
			tokens.addAll(getConvertedToken(scanner, nextToken));
		}
	} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
		tokens.add(JavaTokenTypeTokenizer.COMMENT_JAVADOC);
	} else {
		tokens.add(tokenString);
	}

	return tokens;
}