Java Code Examples for org.eclipse.jdt.core.compiler.ITerminalSymbols#TokenNameCOMMENT_JAVADOC

The following examples show how to use org.eclipse.jdt.core.compiler.ITerminalSymbols#TokenNameCOMMENT_JAVADOC . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: RefactoringScanner.java From eclipse.jdt.ls with Eclipse Public License 2.0

6 votes

private void doScan() {
	try{
		int token = fScanner.getNextToken();
		while (token != ITerminalSymbols.TokenNameEOF) {
			switch (token) {
				case ITerminalSymbols.TokenNameStringLiteral :
				case ITerminalSymbols.TokenNameCOMMENT_JAVADOC :
				case ITerminalSymbols.TokenNameCOMMENT_LINE :
				case ITerminalSymbols.TokenNameCOMMENT_BLOCK :
					parseCurrentToken();
			}
			token = fScanner.getNextToken();
		}
	} catch (InvalidInputException e){
		//ignore
	}
}

Example 2

Source File: RefactoringScanner.java From Eclipse-Postfix-Code-Completion with Eclipse Public License 1.0

6 votes

private void doScan() {
	try{
		int token = fScanner.getNextToken();
		while (token != ITerminalSymbols.TokenNameEOF) {
			switch (token) {
				case ITerminalSymbols.TokenNameStringLiteral :
				case ITerminalSymbols.TokenNameCOMMENT_JAVADOC :
				case ITerminalSymbols.TokenNameCOMMENT_LINE :
				case ITerminalSymbols.TokenNameCOMMENT_BLOCK :
					parseCurrentToken();
			}
			token = fScanner.getNextToken();
		}
	} catch (InvalidInputException e){
		//ignore
	}
}

Example 3

Source File: JavaTokenTypeTokenizer.java From codemining-core with BSD 3-Clause "New" or "Revised" License

5 votes

@Override
public List<FullToken> getTokenListFromCode(final char[] code) {
	final List<FullToken> tokens = Lists.newArrayList();
	tokens.add(new FullToken(SENTENCE_START, SENTENCE_START));
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			} else if (token == ITerminalSymbols.TokenNameIdentifier) {
				tokens.add(new FullToken(IDENTIFIER_TOKEN, ""));
			} else if (isLiteralToken(token)) {
				tokens.add(new FullToken(LITERAL_TOKEN, ""));
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
				tokens.add(new FullToken(COMMENT_BLOCK, ""));
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
				tokens.add(new FullToken(COMMENT_JAVADOC, ""));
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
				tokens.add(new FullToken(COMMENT_LINE, ""));
			} else {
				tokens.add(new FullToken(scanner.getCurrentTokenString(),
						""));
			}

		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(new FullToken(SENTENCE_END, SENTENCE_END));
	return tokens;
}

Example 4

Source File: JavaTokenTypeTokenizer.java From codemining-core with BSD 3-Clause "New" or "Revised" License

5 votes

@Override
public List<String> tokenListFromCode(final char[] code) {
	final List<String> tokens = Lists.newArrayList();
	tokens.add(SENTENCE_START);
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			} else if (token == ITerminalSymbols.TokenNameIdentifier) {
				tokens.add(IDENTIFIER_TOKEN);
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
				tokens.add(COMMENT_BLOCK);
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
				tokens.add(COMMENT_LINE);
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
				tokens.add(COMMENT_JAVADOC);
			} else if (isLiteralToken(token)) {
				tokens.add(LITERAL_TOKEN);
			} else {
				tokens.add(scanner.getCurrentTokenString());
			}

		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(SENTENCE_END);
	return tokens;
}

Example 5

Source File: JavaWhitespaceTokenizer.java From codemining-core with BSD 3-Clause "New" or "Revised" License

5 votes

/**
 * @param tokens
 * @param scanner
 * @param token
 * @return
 * @throws InvalidInputException
 */
private List<String> getConvertedToken(final PublicScanner scanner,
		final int token) throws InvalidInputException {
	final List<String> tokens = Lists.newArrayList();
	if (token == ITerminalSymbols.TokenNameEOF) {
		return Collections.emptyList();
	}
	final String tokenString = scanner.getCurrentTokenString();

	if (token == ITerminalSymbols.TokenNameWHITESPACE) {
		tokens.add(whitespaceConverter.toWhiteSpaceSymbol(tokenString));
	} else if (token == ITerminalSymbols.TokenNameIdentifier) {
		tokens.add(toIdentifierSymbol(tokenString));
	} else if (JavaTokenTypeTokenizer.isLiteralToken(token)) {
		tokens.add(toLiteralSymbol(tokenString));
	} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
		tokens.add(JavaTokenTypeTokenizer.COMMENT_BLOCK);
	} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
		tokens.add(JavaTokenTypeTokenizer.COMMENT_LINE);
		final int nextToken = scanner.getNextToken();
		if (nextToken == ITerminalSymbols.TokenNameWHITESPACE) {
			tokens.add(whitespaceConverter.toWhiteSpaceSymbol("\n"
					+ scanner.getCurrentTokenString()));
		} else {
			tokens.add(whitespaceConverter.toWhiteSpaceSymbol("\n"));
			tokens.addAll(getConvertedToken(scanner, nextToken));
		}
	} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
		tokens.add(JavaTokenTypeTokenizer.COMMENT_JAVADOC);
	} else {
		tokens.add(tokenString);
	}

	return tokens;
}

Example 6

Source File: CommentAnalyzer.java From Eclipse-Postfix-Code-Completion with Eclipse Public License 1.0

5 votes

private void check(RefactoringStatus result, Selection selection, IScanner scanner, int start, int end) {
	char[] characters= scanner.getSource();
	selection= adjustSelection(characters, selection, end);
	scanner.resetTo(start, end);

	int token= 0;
	try {
		loop: while (token != ITerminalSymbols.TokenNameEOF) {
			token= scanner.getNextToken();
			switch(token) {
				case ITerminalSymbols.TokenNameCOMMENT_LINE:
				case ITerminalSymbols.TokenNameCOMMENT_BLOCK:
				case ITerminalSymbols.TokenNameCOMMENT_JAVADOC:
					if (checkStart(scanner, selection.getOffset())) {
						result.addFatalError(RefactoringCoreMessages.CommentAnalyzer_starts_inside_comment);
						break loop;
					}
					if (checkEnd(scanner, selection.getInclusiveEnd())) {
						result.addFatalError(RefactoringCoreMessages.CommentAnalyzer_ends_inside_comment);
						break loop;
					}
					break;
			}
		}
	} catch (InvalidInputException e) {
		result.addFatalError(RefactoringCoreMessages.CommentAnalyzer_internal_error);
	}
}

Example 7

Source File: JavaWhitespaceTokenizer.java From tassal with BSD 3-Clause "New" or "Revised" License

5 votes

/**
 * @param tokens
 * @param scanner
 * @param token
 * @return
 * @throws InvalidInputException
 */
private List<String> getConvertedToken(final PublicScanner scanner,
		final int token) throws InvalidInputException {
	final List<String> tokens = Lists.newArrayList();
	if (token == ITerminalSymbols.TokenNameEOF) {
		return Collections.emptyList();
	}
	final String tokenString = scanner.getCurrentTokenString();

	if (token == ITerminalSymbols.TokenNameWHITESPACE) {
		tokens.add(whitespaceConverter.toWhiteSpaceSymbol(tokenString));
	} else if (token == ITerminalSymbols.TokenNameIdentifier) {
		tokens.add(toIdentifierSymbol(tokenString));
	} else if (JavaTokenTypeTokenizer.isLiteralToken(token)) {
		tokens.add(toLiteralSymbol(tokenString));
	} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
		tokens.add(JavaTokenTypeTokenizer.COMMENT_BLOCK);
	} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
		tokens.add(JavaTokenTypeTokenizer.COMMENT_LINE);
		final int nextToken = scanner.getNextToken();
		if (nextToken == ITerminalSymbols.TokenNameWHITESPACE) {
			tokens.add(whitespaceConverter.toWhiteSpaceSymbol("\n"
					+ scanner.getCurrentTokenString()));
		} else {
			tokens.add(whitespaceConverter.toWhiteSpaceSymbol("\n"));
			tokens.addAll(getConvertedToken(scanner, nextToken));
		}
	} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
		tokens.add(JavaTokenTypeTokenizer.COMMENT_JAVADOC);
	} else {
		tokens.add(tokenString);
	}

	return tokens;
}

Example 8

Source File: JavaTokenTypeTokenizer.java From tassal with BSD 3-Clause "New" or "Revised" License

5 votes

@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
	final SortedMap<Integer, String> tokens = Maps.newTreeMap();
	tokens.put(-1, SENTENCE_START);
	tokens.put(Integer.MAX_VALUE, SENTENCE_END);
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				final int position = scanner.getCurrentTokenStartPosition();

				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				} else if (token == ITerminalSymbols.TokenNameIdentifier) {
					tokens.put(position, IDENTIFIER_TOKEN);
				} else if (isLiteralToken(token)) {
					tokens.put(position, LITERAL_TOKEN);
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
					tokens.put(position, COMMENT_BLOCK);
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
					tokens.put(position, COMMENT_JAVADOC);
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
					tokens.put(position, COMMENT_LINE);
				} else {
					tokens.put(position, scanner.getCurrentTokenString());
				}

			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}

Example 9

Source File: JavaTokenTypeTokenizer.java From tassal with BSD 3-Clause "New" or "Revised" License

5 votes

@Override
public List<String> tokenListFromCode(final char[] code) {
	final List<String> tokens = Lists.newArrayList();
	tokens.add(SENTENCE_START);
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			} else if (token == ITerminalSymbols.TokenNameIdentifier) {
				tokens.add(IDENTIFIER_TOKEN);
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
				tokens.add(COMMENT_BLOCK);
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
				tokens.add(COMMENT_LINE);
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
				tokens.add(COMMENT_JAVADOC);
			} else if (isLiteralToken(token)) {
				tokens.add(LITERAL_TOKEN);
			} else {
				tokens.add(scanner.getCurrentTokenString());
			}

		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(SENTENCE_END);
	return tokens;
}

Example 10

Source File: JavaTokenTypeTokenizer.java From codemining-core with BSD 3-Clause "New" or "Revised" License

5 votes

@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
	final SortedMap<Integer, String> tokens = Maps.newTreeMap();
	tokens.put(-1, SENTENCE_START);
	tokens.put(Integer.MAX_VALUE, SENTENCE_END);
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				final int position = scanner.getCurrentTokenStartPosition();

				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				} else if (token == ITerminalSymbols.TokenNameIdentifier) {
					tokens.put(position, IDENTIFIER_TOKEN);
				} else if (isLiteralToken(token)) {
					tokens.put(position, LITERAL_TOKEN);
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
					tokens.put(position, COMMENT_BLOCK);
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
					tokens.put(position, COMMENT_JAVADOC);
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
					tokens.put(position, COMMENT_LINE);
				} else {
					tokens.put(position, scanner.getCurrentTokenString());
				}

			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}

Example 11

Source File: JavaTokenTypeTokenizer.java From tassal with BSD 3-Clause "New" or "Revised" License

5 votes

@Override
public SortedMap<Integer, FullToken> fullTokenListWithPos(final char[] code) {
	final SortedMap<Integer, FullToken> tokens = Maps.newTreeMap();
	tokens.put(-1, new FullToken(SENTENCE_START, SENTENCE_START));
	tokens.put(Integer.MAX_VALUE, new FullToken(SENTENCE_END, SENTENCE_END));
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				final int position = scanner.getCurrentTokenStartPosition();

				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				} else if (token == ITerminalSymbols.TokenNameIdentifier) {
					tokens.put(position,
							new FullToken(IDENTIFIER_TOKEN, ""));
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
					tokens.put(position, new FullToken(COMMENT_BLOCK, ""));
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
					tokens.put(position, new FullToken(COMMENT_JAVADOC, ""));
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
					tokens.put(position, new FullToken(COMMENT_LINE, ""));
				} else if (isLiteralToken(token)) {
					tokens.put(position, new FullToken(LITERAL_TOKEN, ""));
				} else {
					tokens.put(position,
							new FullToken(scanner.getCurrentTokenString(),
									""));
				}

			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}

Example 12

Source File: JavaWhitespaceTokenizer.java From api-mining with GNU General Public License v3.0

5 votes

/**
 * @param tokens
 * @param scanner
 * @param token
 * @return
 * @throws InvalidInputException
 */
private List<String> getConvertedToken(final PublicScanner scanner,
		final int token) throws InvalidInputException {
	final List<String> tokens = Lists.newArrayList();
	if (token == ITerminalSymbols.TokenNameEOF) {
		return Collections.emptyList();
	}
	final String tokenString = scanner.getCurrentTokenString();

	if (token == ITerminalSymbols.TokenNameWHITESPACE) {
		tokens.add(whitespaceConverter.toWhiteSpaceSymbol(tokenString));
	} else if (token == ITerminalSymbols.TokenNameIdentifier) {
		tokens.add(toIdentifierSymbol(tokenString));
	} else if (JavaTokenTypeTokenizer.isLiteralToken(token)) {
		tokens.add(toLiteralSymbol(tokenString));
	} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
		tokens.add(JavaTokenTypeTokenizer.COMMENT_BLOCK);
	} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
		tokens.add(JavaTokenTypeTokenizer.COMMENT_LINE);
		final int nextToken = scanner.getNextToken();
		if (nextToken == ITerminalSymbols.TokenNameWHITESPACE) {
			tokens.add(whitespaceConverter.toWhiteSpaceSymbol("\n"
					+ scanner.getCurrentTokenString()));
		} else {
			tokens.add(whitespaceConverter.toWhiteSpaceSymbol("\n"));
			tokens.addAll(getConvertedToken(scanner, nextToken));
		}
	} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
		tokens.add(JavaTokenTypeTokenizer.COMMENT_JAVADOC);
	} else {
		tokens.add(tokenString);
	}

	return tokens;
}

Example 13

Source File: JavaTokenTypeTokenizer.java From api-mining with GNU General Public License v3.0

5 votes

@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
	final SortedMap<Integer, String> tokens = Maps.newTreeMap();
	tokens.put(-1, SENTENCE_START);
	tokens.put(Integer.MAX_VALUE, SENTENCE_END);
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				final int position = scanner.getCurrentTokenStartPosition();

				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				} else if (token == ITerminalSymbols.TokenNameIdentifier) {
					tokens.put(position, IDENTIFIER_TOKEN);
				} else if (isLiteralToken(token)) {
					tokens.put(position, LITERAL_TOKEN);
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
					tokens.put(position, COMMENT_BLOCK);
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
					tokens.put(position, COMMENT_JAVADOC);
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
					tokens.put(position, COMMENT_LINE);
				} else {
					tokens.put(position, scanner.getCurrentTokenString());
				}

			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}

Example 14

Source File: JavaTokenTypeTokenizer.java From api-mining with GNU General Public License v3.0

5 votes

@Override
public List<String> tokenListFromCode(final char[] code) {
	final List<String> tokens = Lists.newArrayList();
	tokens.add(SENTENCE_START);
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			} else if (token == ITerminalSymbols.TokenNameIdentifier) {
				tokens.add(IDENTIFIER_TOKEN);
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
				tokens.add(COMMENT_BLOCK);
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
				tokens.add(COMMENT_LINE);
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
				tokens.add(COMMENT_JAVADOC);
			} else if (isLiteralToken(token)) {
				tokens.add(LITERAL_TOKEN);
			} else {
				tokens.add(scanner.getCurrentTokenString());
			}

		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(SENTENCE_END);
	return tokens;
}

Example 15

Source File: JavaTokenTypeTokenizer.java From api-mining with GNU General Public License v3.0

5 votes

@Override
public List<FullToken> getTokenListFromCode(final char[] code) {
	final List<FullToken> tokens = Lists.newArrayList();
	tokens.add(new FullToken(SENTENCE_START, SENTENCE_START));
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			} else if (token == ITerminalSymbols.TokenNameIdentifier) {
				tokens.add(new FullToken(IDENTIFIER_TOKEN, ""));
			} else if (isLiteralToken(token)) {
				tokens.add(new FullToken(LITERAL_TOKEN, ""));
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
				tokens.add(new FullToken(COMMENT_BLOCK, ""));
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
				tokens.add(new FullToken(COMMENT_JAVADOC, ""));
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
				tokens.add(new FullToken(COMMENT_LINE, ""));
			} else {
				tokens.add(new FullToken(scanner.getCurrentTokenString(),
						""));
			}

		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(new FullToken(SENTENCE_END, SENTENCE_END));
	return tokens;
}

Example 16

Source File: JavaTokenTypeTokenizer.java From api-mining with GNU General Public License v3.0

5 votes

@Override
public SortedMap<Integer, FullToken> fullTokenListWithPos(final char[] code) {
	final SortedMap<Integer, FullToken> tokens = Maps.newTreeMap();
	tokens.put(-1, new FullToken(SENTENCE_START, SENTENCE_START));
	tokens.put(Integer.MAX_VALUE, new FullToken(SENTENCE_END, SENTENCE_END));
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				final int position = scanner.getCurrentTokenStartPosition();

				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				} else if (token == ITerminalSymbols.TokenNameIdentifier) {
					tokens.put(position,
							new FullToken(IDENTIFIER_TOKEN, ""));
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
					tokens.put(position, new FullToken(COMMENT_BLOCK, ""));
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
					tokens.put(position, new FullToken(COMMENT_JAVADOC, ""));
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
					tokens.put(position, new FullToken(COMMENT_LINE, ""));
				} else if (isLiteralToken(token)) {
					tokens.put(position, new FullToken(LITERAL_TOKEN, ""));
				} else {
					tokens.put(position,
							new FullToken(scanner.getCurrentTokenString(),
									""));
				}

			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}

Example 17

Source File: SJavaTokenComparator.java From coming with MIT License

5 votes

/**
 * Returns the length of the token that
 * initiates the given comment type.
 *
 * @param tokenType
 * @return the length of the token that start a comment
 * @since 3.3
 */
private static int getCommentStartTokenLength(int tokenType) {
  if (tokenType == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
    return 3;
  } else if (tokenType == ITerminalSymbols.TokenNameStringLiteral) {
    return 1;
  } else {
    return 2;
  }
}

Example 18

Source File: DefaultJavaFoldingStructureProvider.java From Eclipse-Postfix-Code-Completion with Eclipse Public License 1.0

4 votes

private IRegion computeHeaderComment(FoldingStructureComputationContext ctx) throws JavaModelException {
	// search at most up to the first type
	ISourceRange range= ctx.getFirstType().getSourceRange();
	if (range == null)
		return null;
	int start= 0;
	int end= range.getOffset();


	/* code adapted from CommentFormattingStrategy:
	 * scan the header content up to the first type. Once a comment is
	 * found, accumulate any additional comments up to the stop condition.
	 * The stop condition is reaching a package declaration, import container,
	 * or the end of the input.
	 */
	IScanner scanner= ctx.getScanner();
	scanner.resetTo(start, end);

	int headerStart= -1;
	int headerEnd= -1;
	try {
		boolean foundComment= false;
		int terminal= scanner.getNextToken();
		while (terminal != ITerminalSymbols.TokenNameEOF && !(terminal == ITerminalSymbols.TokenNameclass || terminal == ITerminalSymbols.TokenNameinterface || terminal == ITerminalSymbols.TokenNameenum || (foundComment && (terminal == ITerminalSymbols.TokenNameimport || terminal == ITerminalSymbols.TokenNamepackage)))) {

			if (terminal == ITerminalSymbols.TokenNameCOMMENT_JAVADOC || terminal == ITerminalSymbols.TokenNameCOMMENT_BLOCK || terminal == ITerminalSymbols.TokenNameCOMMENT_LINE) {
				if (!foundComment)
					headerStart= scanner.getCurrentTokenStartPosition();
				headerEnd= scanner.getCurrentTokenEndPosition();
				foundComment= true;
			}
			terminal= scanner.getNextToken();
		}


	} catch (InvalidInputException ex) {
		return null;
	}

	if (headerEnd != -1) {
		return new Region(headerStart, headerEnd - headerStart);
	}
	return null;
}

Example 19

Source File: DefaultCodeFormatter.java From Eclipse-Postfix-Code-Completion with Eclipse Public License 1.0

4 votes

private TextEdit probeFormatting(String source, int indentationLevel, String lineSeparator, IRegion[] regions, boolean includeComments) {
	if (PROBING_SCANNER == null) {
		// scanner use to check if the kind could be K_JAVA_DOC, K_MULTI_LINE_COMMENT or K_SINGLE_LINE_COMMENT
		// do not tokenize white spaces to get single comments even with spaces before...
		PROBING_SCANNER = new Scanner(true, false/*do not tokenize whitespaces*/, false/*nls*/, ClassFileConstants.JDK1_6, ClassFileConstants.JDK1_6, null/*taskTags*/, null/*taskPriorities*/, true/*taskCaseSensitive*/);
	}
	PROBING_SCANNER.setSource(source.toCharArray());

	IRegion coveredRegion = getCoveredRegion(regions);
	int offset = coveredRegion.getOffset();
	int length = coveredRegion.getLength();

	PROBING_SCANNER.resetTo(offset, offset + length - 1);
	try {
		int kind = -1;
		switch(PROBING_SCANNER.getNextToken()) {
			case ITerminalSymbols.TokenNameCOMMENT_BLOCK :
				if (PROBING_SCANNER.getNextToken() == TerminalTokens.TokenNameEOF) {
					kind = K_MULTI_LINE_COMMENT;
				}
				break;
			case ITerminalSymbols.TokenNameCOMMENT_LINE :
				if (PROBING_SCANNER.getNextToken() == TerminalTokens.TokenNameEOF) {
					kind = K_SINGLE_LINE_COMMENT;
				}
				break;
			case ITerminalSymbols.TokenNameCOMMENT_JAVADOC :
				if (PROBING_SCANNER.getNextToken() == TerminalTokens.TokenNameEOF) {
					kind = K_JAVA_DOC;
				}
				break;
		}
		if (kind != -1) {
			return formatComment(kind, source, indentationLevel, lineSeparator, regions);
		}
	} catch (InvalidInputException e) {
		// ignore
	}
	PROBING_SCANNER.setSource((char[]) null);

	// probe for expression
	Expression expression = this.codeSnippetParsingUtil.parseExpression(source.toCharArray(), getDefaultCompilerOptions(), true);
	if (expression != null) {
		return internalFormatExpression(source, indentationLevel, lineSeparator, expression, regions, includeComments);
	}

	// probe for body declarations (fields, methods, constructors)
	ASTNode[] bodyDeclarations = this.codeSnippetParsingUtil.parseClassBodyDeclarations(source.toCharArray(), getDefaultCompilerOptions(), true);
	if (bodyDeclarations != null) {
		return internalFormatClassBodyDeclarations(source, indentationLevel, lineSeparator, bodyDeclarations, regions, includeComments);
	}

	// probe for statements
	ConstructorDeclaration constructorDeclaration = this.codeSnippetParsingUtil.parseStatements(source.toCharArray(), getDefaultCompilerOptions(), true, false);
	if (constructorDeclaration.statements != null) {
		return internalFormatStatements(source, indentationLevel, lineSeparator, constructorDeclaration, regions, includeComments);
	}

	// this has to be a compilation unit
	return formatCompilationUnit(source, indentationLevel, lineSeparator, regions, includeComments);
}

Example 20

Source File: Member.java From Eclipse-Postfix-Code-Completion with Eclipse Public License 1.0

4 votes

public ISourceRange getJavadocRange() throws JavaModelException {
	ISourceRange range= getSourceRange();
	if (range == null) return null;
	IBuffer buf= null;
	if (isBinary()) {
		buf = getClassFile().getBuffer();
	} else {
		ICompilationUnit compilationUnit = getCompilationUnit();
		if (!compilationUnit.isConsistent()) {
			return null;
		}
		buf = compilationUnit.getBuffer();
	}
	final int start= range.getOffset();
	final int length= range.getLength();
	if (length > 0 && buf.getChar(start) == '/') {
		IScanner scanner= ToolFactory.createScanner(true, false, false, false);
		try {
			scanner.setSource(buf.getText(start, length).toCharArray());
			int docOffset= -1;
			int docEnd= -1;

			int terminal= scanner.getNextToken();
			loop: while (true) {
				switch(terminal) {
					case ITerminalSymbols.TokenNameCOMMENT_JAVADOC :
						docOffset= scanner.getCurrentTokenStartPosition();
						docEnd= scanner.getCurrentTokenEndPosition() + 1;
						terminal= scanner.getNextToken();
						break;
					case ITerminalSymbols.TokenNameCOMMENT_LINE :
					case ITerminalSymbols.TokenNameCOMMENT_BLOCK :
						terminal= scanner.getNextToken();
						continue loop;
					default :
						break loop;
				}
			}
			if (docOffset != -1) {
				return new SourceRange(docOffset + start, docEnd - docOffset);
			}
		} catch (InvalidInputException ex) {
			// try if there is inherited Javadoc
		} catch (IndexOutOfBoundsException e) {
			// https://bugs.eclipse.org/bugs/show_bug.cgi?id=305001
		}
	}
	return null;
}