Java Code Examples for org.eclipse.jdt.core.compiler.ITerminalSymbols#TokenNameCOMMENT_BLOCK

The following examples show how to use org.eclipse.jdt.core.compiler.ITerminalSymbols#TokenNameCOMMENT_BLOCK . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: RefactoringScanner.java    From Eclipse-Postfix-Code-Completion with Eclipse Public License 1.0 6 votes vote down vote up
private void doScan() {
	try{
		int token = fScanner.getNextToken();
		while (token != ITerminalSymbols.TokenNameEOF) {
			switch (token) {
				case ITerminalSymbols.TokenNameStringLiteral :
				case ITerminalSymbols.TokenNameCOMMENT_JAVADOC :
				case ITerminalSymbols.TokenNameCOMMENT_LINE :
				case ITerminalSymbols.TokenNameCOMMENT_BLOCK :
					parseCurrentToken();
			}
			token = fScanner.getNextToken();
		}
	} catch (InvalidInputException e){
		//ignore
	}
}
 
Example 2
Source File: RefactoringScanner.java    From eclipse.jdt.ls with Eclipse Public License 2.0 6 votes vote down vote up
private void doScan() {
	try{
		int token = fScanner.getNextToken();
		while (token != ITerminalSymbols.TokenNameEOF) {
			switch (token) {
				case ITerminalSymbols.TokenNameStringLiteral :
				case ITerminalSymbols.TokenNameCOMMENT_JAVADOC :
				case ITerminalSymbols.TokenNameCOMMENT_LINE :
				case ITerminalSymbols.TokenNameCOMMENT_BLOCK :
					parseCurrentToken();
			}
			token = fScanner.getNextToken();
		}
	} catch (InvalidInputException e){
		//ignore
	}
}
 
Example 3
Source File: JavaTokenTypeTokenizer.java    From tassal with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
	final SortedMap<Integer, String> tokens = Maps.newTreeMap();
	tokens.put(-1, SENTENCE_START);
	tokens.put(Integer.MAX_VALUE, SENTENCE_END);
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				final int position = scanner.getCurrentTokenStartPosition();

				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				} else if (token == ITerminalSymbols.TokenNameIdentifier) {
					tokens.put(position, IDENTIFIER_TOKEN);
				} else if (isLiteralToken(token)) {
					tokens.put(position, LITERAL_TOKEN);
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
					tokens.put(position, COMMENT_BLOCK);
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
					tokens.put(position, COMMENT_JAVADOC);
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
					tokens.put(position, COMMENT_LINE);
				} else {
					tokens.put(position, scanner.getCurrentTokenString());
				}

			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}
 
Example 4
Source File: JavaTokenTypeTokenizer.java    From codemining-core with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
@Override
public List<FullToken> getTokenListFromCode(final char[] code) {
	final List<FullToken> tokens = Lists.newArrayList();
	tokens.add(new FullToken(SENTENCE_START, SENTENCE_START));
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			} else if (token == ITerminalSymbols.TokenNameIdentifier) {
				tokens.add(new FullToken(IDENTIFIER_TOKEN, ""));
			} else if (isLiteralToken(token)) {
				tokens.add(new FullToken(LITERAL_TOKEN, ""));
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
				tokens.add(new FullToken(COMMENT_BLOCK, ""));
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
				tokens.add(new FullToken(COMMENT_JAVADOC, ""));
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
				tokens.add(new FullToken(COMMENT_LINE, ""));
			} else {
				tokens.add(new FullToken(scanner.getCurrentTokenString(),
						""));
			}

		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(new FullToken(SENTENCE_END, SENTENCE_END));
	return tokens;
}
 
Example 5
Source File: JavaTokenTypeTokenizer.java    From codemining-core with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
@Override
public SortedMap<Integer, FullToken> fullTokenListWithPos(final char[] code) {
	final SortedMap<Integer, FullToken> tokens = Maps.newTreeMap();
	tokens.put(-1, new FullToken(SENTENCE_START, SENTENCE_START));
	tokens.put(Integer.MAX_VALUE, new FullToken(SENTENCE_END, SENTENCE_END));
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				final int position = scanner.getCurrentTokenStartPosition();

				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				} else if (token == ITerminalSymbols.TokenNameIdentifier) {
					tokens.put(position,
							new FullToken(IDENTIFIER_TOKEN, ""));
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
					tokens.put(position, new FullToken(COMMENT_BLOCK, ""));
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
					tokens.put(position, new FullToken(COMMENT_JAVADOC, ""));
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
					tokens.put(position, new FullToken(COMMENT_LINE, ""));
				} else if (isLiteralToken(token)) {
					tokens.put(position, new FullToken(LITERAL_TOKEN, ""));
				} else {
					tokens.put(position,
							new FullToken(scanner.getCurrentTokenString(),
									""));
				}

			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}
 
Example 6
Source File: JavaTokenTypeTokenizer.java    From codemining-core with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
@Override
public List<String> tokenListFromCode(final char[] code) {
	final List<String> tokens = Lists.newArrayList();
	tokens.add(SENTENCE_START);
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			} else if (token == ITerminalSymbols.TokenNameIdentifier) {
				tokens.add(IDENTIFIER_TOKEN);
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
				tokens.add(COMMENT_BLOCK);
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
				tokens.add(COMMENT_LINE);
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
				tokens.add(COMMENT_JAVADOC);
			} else if (isLiteralToken(token)) {
				tokens.add(LITERAL_TOKEN);
			} else {
				tokens.add(scanner.getCurrentTokenString());
			}

		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(SENTENCE_END);
	return tokens;
}
 
Example 7
Source File: CommentAnalyzer.java    From Eclipse-Postfix-Code-Completion with Eclipse Public License 1.0 5 votes vote down vote up
private void check(RefactoringStatus result, Selection selection, IScanner scanner, int start, int end) {
	char[] characters= scanner.getSource();
	selection= adjustSelection(characters, selection, end);
	scanner.resetTo(start, end);

	int token= 0;
	try {
		loop: while (token != ITerminalSymbols.TokenNameEOF) {
			token= scanner.getNextToken();
			switch(token) {
				case ITerminalSymbols.TokenNameCOMMENT_LINE:
				case ITerminalSymbols.TokenNameCOMMENT_BLOCK:
				case ITerminalSymbols.TokenNameCOMMENT_JAVADOC:
					if (checkStart(scanner, selection.getOffset())) {
						result.addFatalError(RefactoringCoreMessages.CommentAnalyzer_starts_inside_comment);
						break loop;
					}
					if (checkEnd(scanner, selection.getInclusiveEnd())) {
						result.addFatalError(RefactoringCoreMessages.CommentAnalyzer_ends_inside_comment);
						break loop;
					}
					break;
			}
		}
	} catch (InvalidInputException e) {
		result.addFatalError(RefactoringCoreMessages.CommentAnalyzer_internal_error);
	}
}
 
Example 8
Source File: JavaWhitespaceTokenizer.java    From tassal with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
/**
 * @param tokens
 * @param scanner
 * @param token
 * @return
 * @throws InvalidInputException
 */
private List<String> getConvertedToken(final PublicScanner scanner,
		final int token) throws InvalidInputException {
	final List<String> tokens = Lists.newArrayList();
	if (token == ITerminalSymbols.TokenNameEOF) {
		return Collections.emptyList();
	}
	final String tokenString = scanner.getCurrentTokenString();

	if (token == ITerminalSymbols.TokenNameWHITESPACE) {
		tokens.add(whitespaceConverter.toWhiteSpaceSymbol(tokenString));
	} else if (token == ITerminalSymbols.TokenNameIdentifier) {
		tokens.add(toIdentifierSymbol(tokenString));
	} else if (JavaTokenTypeTokenizer.isLiteralToken(token)) {
		tokens.add(toLiteralSymbol(tokenString));
	} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
		tokens.add(JavaTokenTypeTokenizer.COMMENT_BLOCK);
	} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
		tokens.add(JavaTokenTypeTokenizer.COMMENT_LINE);
		final int nextToken = scanner.getNextToken();
		if (nextToken == ITerminalSymbols.TokenNameWHITESPACE) {
			tokens.add(whitespaceConverter.toWhiteSpaceSymbol("\n"
					+ scanner.getCurrentTokenString()));
		} else {
			tokens.add(whitespaceConverter.toWhiteSpaceSymbol("\n"));
			tokens.addAll(getConvertedToken(scanner, nextToken));
		}
	} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
		tokens.add(JavaTokenTypeTokenizer.COMMENT_JAVADOC);
	} else {
		tokens.add(tokenString);
	}

	return tokens;
}
 
Example 9
Source File: JavaTokenTypeTokenizer.java    From tassal with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
@Override
public List<String> tokenListFromCode(final char[] code) {
	final List<String> tokens = Lists.newArrayList();
	tokens.add(SENTENCE_START);
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			} else if (token == ITerminalSymbols.TokenNameIdentifier) {
				tokens.add(IDENTIFIER_TOKEN);
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
				tokens.add(COMMENT_BLOCK);
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
				tokens.add(COMMENT_LINE);
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
				tokens.add(COMMENT_JAVADOC);
			} else if (isLiteralToken(token)) {
				tokens.add(LITERAL_TOKEN);
			} else {
				tokens.add(scanner.getCurrentTokenString());
			}

		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(SENTENCE_END);
	return tokens;
}
 
Example 10
Source File: JavaTokenTypeTokenizer.java    From tassal with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
@Override
public List<FullToken> getTokenListFromCode(final char[] code) {
	final List<FullToken> tokens = Lists.newArrayList();
	tokens.add(new FullToken(SENTENCE_START, SENTENCE_START));
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			} else if (token == ITerminalSymbols.TokenNameIdentifier) {
				tokens.add(new FullToken(IDENTIFIER_TOKEN, ""));
			} else if (isLiteralToken(token)) {
				tokens.add(new FullToken(LITERAL_TOKEN, ""));
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
				tokens.add(new FullToken(COMMENT_BLOCK, ""));
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
				tokens.add(new FullToken(COMMENT_JAVADOC, ""));
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
				tokens.add(new FullToken(COMMENT_LINE, ""));
			} else {
				tokens.add(new FullToken(scanner.getCurrentTokenString(),
						""));
			}

		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(new FullToken(SENTENCE_END, SENTENCE_END));
	return tokens;
}
 
Example 11
Source File: JavaWhitespaceTokenizer.java    From codemining-core with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
/**
 * @param tokens
 * @param scanner
 * @param token
 * @return
 * @throws InvalidInputException
 */
private List<String> getConvertedToken(final PublicScanner scanner,
		final int token) throws InvalidInputException {
	final List<String> tokens = Lists.newArrayList();
	if (token == ITerminalSymbols.TokenNameEOF) {
		return Collections.emptyList();
	}
	final String tokenString = scanner.getCurrentTokenString();

	if (token == ITerminalSymbols.TokenNameWHITESPACE) {
		tokens.add(whitespaceConverter.toWhiteSpaceSymbol(tokenString));
	} else if (token == ITerminalSymbols.TokenNameIdentifier) {
		tokens.add(toIdentifierSymbol(tokenString));
	} else if (JavaTokenTypeTokenizer.isLiteralToken(token)) {
		tokens.add(toLiteralSymbol(tokenString));
	} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
		tokens.add(JavaTokenTypeTokenizer.COMMENT_BLOCK);
	} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
		tokens.add(JavaTokenTypeTokenizer.COMMENT_LINE);
		final int nextToken = scanner.getNextToken();
		if (nextToken == ITerminalSymbols.TokenNameWHITESPACE) {
			tokens.add(whitespaceConverter.toWhiteSpaceSymbol("\n"
					+ scanner.getCurrentTokenString()));
		} else {
			tokens.add(whitespaceConverter.toWhiteSpaceSymbol("\n"));
			tokens.addAll(getConvertedToken(scanner, nextToken));
		}
	} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
		tokens.add(JavaTokenTypeTokenizer.COMMENT_JAVADOC);
	} else {
		tokens.add(tokenString);
	}

	return tokens;
}
 
Example 12
Source File: JavaWhitespaceTokenizer.java    From api-mining with GNU General Public License v3.0 5 votes vote down vote up
/**
 * @param tokens
 * @param scanner
 * @param token
 * @return
 * @throws InvalidInputException
 */
private List<String> getConvertedToken(final PublicScanner scanner,
		final int token) throws InvalidInputException {
	final List<String> tokens = Lists.newArrayList();
	if (token == ITerminalSymbols.TokenNameEOF) {
		return Collections.emptyList();
	}
	final String tokenString = scanner.getCurrentTokenString();

	if (token == ITerminalSymbols.TokenNameWHITESPACE) {
		tokens.add(whitespaceConverter.toWhiteSpaceSymbol(tokenString));
	} else if (token == ITerminalSymbols.TokenNameIdentifier) {
		tokens.add(toIdentifierSymbol(tokenString));
	} else if (JavaTokenTypeTokenizer.isLiteralToken(token)) {
		tokens.add(toLiteralSymbol(tokenString));
	} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
		tokens.add(JavaTokenTypeTokenizer.COMMENT_BLOCK);
	} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
		tokens.add(JavaTokenTypeTokenizer.COMMENT_LINE);
		final int nextToken = scanner.getNextToken();
		if (nextToken == ITerminalSymbols.TokenNameWHITESPACE) {
			tokens.add(whitespaceConverter.toWhiteSpaceSymbol("\n"
					+ scanner.getCurrentTokenString()));
		} else {
			tokens.add(whitespaceConverter.toWhiteSpaceSymbol("\n"));
			tokens.addAll(getConvertedToken(scanner, nextToken));
		}
	} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
		tokens.add(JavaTokenTypeTokenizer.COMMENT_JAVADOC);
	} else {
		tokens.add(tokenString);
	}

	return tokens;
}
 
Example 13
Source File: JavaTokenTypeTokenizer.java    From api-mining with GNU General Public License v3.0 5 votes vote down vote up
@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
	final SortedMap<Integer, String> tokens = Maps.newTreeMap();
	tokens.put(-1, SENTENCE_START);
	tokens.put(Integer.MAX_VALUE, SENTENCE_END);
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				final int position = scanner.getCurrentTokenStartPosition();

				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				} else if (token == ITerminalSymbols.TokenNameIdentifier) {
					tokens.put(position, IDENTIFIER_TOKEN);
				} else if (isLiteralToken(token)) {
					tokens.put(position, LITERAL_TOKEN);
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
					tokens.put(position, COMMENT_BLOCK);
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
					tokens.put(position, COMMENT_JAVADOC);
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
					tokens.put(position, COMMENT_LINE);
				} else {
					tokens.put(position, scanner.getCurrentTokenString());
				}

			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}
 
Example 14
Source File: JavaTokenTypeTokenizer.java    From api-mining with GNU General Public License v3.0 5 votes vote down vote up
@Override
public List<String> tokenListFromCode(final char[] code) {
	final List<String> tokens = Lists.newArrayList();
	tokens.add(SENTENCE_START);
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			} else if (token == ITerminalSymbols.TokenNameIdentifier) {
				tokens.add(IDENTIFIER_TOKEN);
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
				tokens.add(COMMENT_BLOCK);
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
				tokens.add(COMMENT_LINE);
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
				tokens.add(COMMENT_JAVADOC);
			} else if (isLiteralToken(token)) {
				tokens.add(LITERAL_TOKEN);
			} else {
				tokens.add(scanner.getCurrentTokenString());
			}

		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(SENTENCE_END);
	return tokens;
}
 
Example 15
Source File: JavaTokenTypeTokenizer.java    From api-mining with GNU General Public License v3.0 5 votes vote down vote up
@Override
public List<FullToken> getTokenListFromCode(final char[] code) {
	final List<FullToken> tokens = Lists.newArrayList();
	tokens.add(new FullToken(SENTENCE_START, SENTENCE_START));
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			} else if (token == ITerminalSymbols.TokenNameIdentifier) {
				tokens.add(new FullToken(IDENTIFIER_TOKEN, ""));
			} else if (isLiteralToken(token)) {
				tokens.add(new FullToken(LITERAL_TOKEN, ""));
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
				tokens.add(new FullToken(COMMENT_BLOCK, ""));
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
				tokens.add(new FullToken(COMMENT_JAVADOC, ""));
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
				tokens.add(new FullToken(COMMENT_LINE, ""));
			} else {
				tokens.add(new FullToken(scanner.getCurrentTokenString(),
						""));
			}

		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(new FullToken(SENTENCE_END, SENTENCE_END));
	return tokens;
}
 
Example 16
Source File: JavaTokenTypeTokenizer.java    From api-mining with GNU General Public License v3.0 5 votes vote down vote up
@Override
public SortedMap<Integer, FullToken> fullTokenListWithPos(final char[] code) {
	final SortedMap<Integer, FullToken> tokens = Maps.newTreeMap();
	tokens.put(-1, new FullToken(SENTENCE_START, SENTENCE_START));
	tokens.put(Integer.MAX_VALUE, new FullToken(SENTENCE_END, SENTENCE_END));
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				final int position = scanner.getCurrentTokenStartPosition();

				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				} else if (token == ITerminalSymbols.TokenNameIdentifier) {
					tokens.put(position,
							new FullToken(IDENTIFIER_TOKEN, ""));
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
					tokens.put(position, new FullToken(COMMENT_BLOCK, ""));
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
					tokens.put(position, new FullToken(COMMENT_JAVADOC, ""));
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
					tokens.put(position, new FullToken(COMMENT_LINE, ""));
				} else if (isLiteralToken(token)) {
					tokens.put(position, new FullToken(LITERAL_TOKEN, ""));
				} else {
					tokens.put(position,
							new FullToken(scanner.getCurrentTokenString(),
									""));
				}

			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}
 
Example 17
Source File: TokenScanner.java    From Eclipse-Postfix-Code-Completion with Eclipse Public License 1.0 4 votes vote down vote up
public static boolean isComment(int token) {
	return token == ITerminalSymbols.TokenNameCOMMENT_BLOCK || token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC
		|| token == ITerminalSymbols.TokenNameCOMMENT_LINE;
}
 
Example 18
Source File: DefaultJavaFoldingStructureProvider.java    From Eclipse-Postfix-Code-Completion with Eclipse Public License 1.0 4 votes vote down vote up
private IRegion computeHeaderComment(FoldingStructureComputationContext ctx) throws JavaModelException {
	// search at most up to the first type
	ISourceRange range= ctx.getFirstType().getSourceRange();
	if (range == null)
		return null;
	int start= 0;
	int end= range.getOffset();


	/* code adapted from CommentFormattingStrategy:
	 * scan the header content up to the first type. Once a comment is
	 * found, accumulate any additional comments up to the stop condition.
	 * The stop condition is reaching a package declaration, import container,
	 * or the end of the input.
	 */
	IScanner scanner= ctx.getScanner();
	scanner.resetTo(start, end);

	int headerStart= -1;
	int headerEnd= -1;
	try {
		boolean foundComment= false;
		int terminal= scanner.getNextToken();
		while (terminal != ITerminalSymbols.TokenNameEOF && !(terminal == ITerminalSymbols.TokenNameclass || terminal == ITerminalSymbols.TokenNameinterface || terminal == ITerminalSymbols.TokenNameenum || (foundComment && (terminal == ITerminalSymbols.TokenNameimport || terminal == ITerminalSymbols.TokenNamepackage)))) {

			if (terminal == ITerminalSymbols.TokenNameCOMMENT_JAVADOC || terminal == ITerminalSymbols.TokenNameCOMMENT_BLOCK || terminal == ITerminalSymbols.TokenNameCOMMENT_LINE) {
				if (!foundComment)
					headerStart= scanner.getCurrentTokenStartPosition();
				headerEnd= scanner.getCurrentTokenEndPosition();
				foundComment= true;
			}
			terminal= scanner.getNextToken();
		}


	} catch (InvalidInputException ex) {
		return null;
	}

	if (headerEnd != -1) {
		return new Region(headerStart, headerEnd - headerStart);
	}
	return null;
}
 
Example 19
Source File: Member.java    From Eclipse-Postfix-Code-Completion with Eclipse Public License 1.0 4 votes vote down vote up
public ISourceRange getJavadocRange() throws JavaModelException {
	ISourceRange range= getSourceRange();
	if (range == null) return null;
	IBuffer buf= null;
	if (isBinary()) {
		buf = getClassFile().getBuffer();
	} else {
		ICompilationUnit compilationUnit = getCompilationUnit();
		if (!compilationUnit.isConsistent()) {
			return null;
		}
		buf = compilationUnit.getBuffer();
	}
	final int start= range.getOffset();
	final int length= range.getLength();
	if (length > 0 && buf.getChar(start) == '/') {
		IScanner scanner= ToolFactory.createScanner(true, false, false, false);
		try {
			scanner.setSource(buf.getText(start, length).toCharArray());
			int docOffset= -1;
			int docEnd= -1;

			int terminal= scanner.getNextToken();
			loop: while (true) {
				switch(terminal) {
					case ITerminalSymbols.TokenNameCOMMENT_JAVADOC :
						docOffset= scanner.getCurrentTokenStartPosition();
						docEnd= scanner.getCurrentTokenEndPosition() + 1;
						terminal= scanner.getNextToken();
						break;
					case ITerminalSymbols.TokenNameCOMMENT_LINE :
					case ITerminalSymbols.TokenNameCOMMENT_BLOCK :
						terminal= scanner.getNextToken();
						continue loop;
					default :
						break loop;
				}
			}
			if (docOffset != -1) {
				return new SourceRange(docOffset + start, docEnd - docOffset);
			}
		} catch (InvalidInputException ex) {
			// try if there is inherited Javadoc
		} catch (IndexOutOfBoundsException e) {
			// https://bugs.eclipse.org/bugs/show_bug.cgi?id=305001
		}
	}
	return null;
}
 
Example 20
Source File: FoldingRangeHandler.java    From eclipse.jdt.ls with Eclipse Public License 2.0 4 votes vote down vote up
private void computeFoldingRanges(List<FoldingRange> foldingRanges, ITypeRoot unit, IProgressMonitor monitor) {
	try {
		ISourceRange range = unit.getSourceRange();
		if (!SourceRange.isAvailable(range)) {
			return;
		}

		String contents = unit.getSource();
		if (StringUtils.isBlank(contents)) {
			return;
		}

		final int shift = range.getOffset();
		IScanner scanner = getScanner();
		scanner.setSource(contents.toCharArray());
		scanner.resetTo(shift, shift + range.getLength());

		int start = shift;
		int token = 0;
		Stack<Integer> regionStarts = new Stack<>();
		while (token != ITerminalSymbols.TokenNameEOF) {
			start = scanner.getCurrentTokenStartPosition();
			switch (token) {
				case ITerminalSymbols.TokenNameCOMMENT_JAVADOC:
				case ITerminalSymbols.TokenNameCOMMENT_BLOCK:
					int end = scanner.getCurrentTokenEndPosition();
					FoldingRange commentFoldingRange = new FoldingRange(scanner.getLineNumber(start) - 1, scanner.getLineNumber(end) - 1);
					commentFoldingRange.setKind(FoldingRangeKind.Comment);
					foldingRanges.add(commentFoldingRange);
					break;
				case ITerminalSymbols.TokenNameCOMMENT_LINE:
					String currentSource = String.valueOf(scanner.getCurrentTokenSource());
					if (REGION_START_PATTERN.matcher(currentSource).lookingAt()) {
						regionStarts.push(start);
					} else if (REGION_END_PATTERN.matcher(currentSource).lookingAt()) {
						if (regionStarts.size() > 0) {
							FoldingRange regionFolding = new FoldingRange(scanner.getLineNumber(regionStarts.pop()) - 1, scanner.getLineNumber(start) - 1);
							regionFolding.setKind(FoldingRangeKind.Region);
							foldingRanges.add(regionFolding);
						}
					}
					break;
				default:
					break;
			}
			token = getNextToken(scanner);
		}
		computeTypeRootRanges(foldingRanges, unit, scanner);
	} catch (CoreException e) {
		JavaLanguageServerPlugin.logException("Problem with folding range for " + unit.getPath().toPortableString(), e);
		monitor.setCanceled(true);
	}
}