Java Code Examples for codemining.js.codeutils.JavascriptTokenizer

The following examples show how to use codemining.js.codeutils.JavascriptTokenizer. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
/**
 * Get the token bindings given the ASTNode bindings and the source code
 * positions.
 *
 * @param sourceCode
 * @param nodeBindings
 * @return
 */
public static List<TokenNameBinding> getTokenBindings(
		final String sourceCode, final Set<Set<ASTNode>> nodeBindings) {
	final JavascriptTokenizer tokenizer = new JavascriptTokenizer();
	final SortedMap<Integer, String> tokenPositions = tokenizer
			.tokenListWithPos(sourceCode.toCharArray());
	final SortedMap<Integer, Integer> positionToIndex = getTokenIndexForPostion(tokenPositions);
	final List<String> tokens = Lists.newArrayList(tokenPositions.values());

	final List<TokenNameBinding> bindings = Lists.newArrayList();

	for (final Set<ASTNode> boundName : nodeBindings) {
		final List<Integer> boundPositions = Lists.newArrayList();
		for (final ASTNode name : boundName) {
			// Convert position to token index and add
			final int tokenIdx = positionToIndex.get(name
					.getStartPosition());
			boundPositions.add(tokenIdx);
		}
		bindings.add(new TokenNameBinding(Sets.newTreeSet(boundPositions),
				tokens, getFeatures(boundName)));
	}

	return bindings;
}
 
Example 2
/**
 * Get the token bindings given the ASTNode bindings and the source code
 * positions.
 *
 * @param sourceCode
 * @param nodeBindings
 * @return
 */
public static List<TokenNameBinding> getTokenBindings(
		final String sourceCode, final Set<Set<ASTNode>> nodeBindings) {
	final JavascriptTokenizer tokenizer = new JavascriptTokenizer();
	final SortedMap<Integer, String> tokenPositions = tokenizer
			.tokenListWithPos(sourceCode.toCharArray());
	final SortedMap<Integer, Integer> positionToIndex = getTokenIndexForPostion(tokenPositions);
	final List<String> tokens = Lists.newArrayList(tokenPositions.values());

	final List<TokenNameBinding> bindings = Lists.newArrayList();

	for (final Set<ASTNode> boundName : nodeBindings) {
		final List<Integer> boundPositions = Lists.newArrayList();
		for (final ASTNode name : boundName) {
			// Convert position to token index and add
			final int tokenIdx = positionToIndex.get(name
					.getStartPosition());
			boundPositions.add(tokenIdx);
		}
		bindings.add(new TokenNameBinding(Sets.newTreeSet(boundPositions),
				tokens, getFeatures(boundName)));
	}

	return bindings;
}
 
Example 3
/**
 * Get the token bindings given the ASTNode bindings and the source code
 * positions.
 *
 * @param sourceCode
 * @param nodeBindings
 * @return
 */
public static List<TokenNameBinding> getTokenBindings(
		final String sourceCode, final Set<Set<ASTNode>> nodeBindings) {
	final JavascriptTokenizer tokenizer = new JavascriptTokenizer();
	final SortedMap<Integer, String> tokenPositions = tokenizer
			.tokenListWithPos(sourceCode.toCharArray());
	final SortedMap<Integer, Integer> positionToIndex = getTokenIndexForPostion(tokenPositions);
	final List<String> tokens = Lists.newArrayList(tokenPositions.values());

	final List<TokenNameBinding> bindings = Lists.newArrayList();

	for (final Set<ASTNode> boundName : nodeBindings) {
		final List<Integer> boundPositions = Lists.newArrayList();
		for (final ASTNode name : boundName) {
			// Convert position to token index and add
			final int tokenIdx = positionToIndex.get(name
					.getStartPosition());
			boundPositions.add(tokenIdx);
		}
		bindings.add(new TokenNameBinding(Sets.newTreeSet(boundPositions),
				tokens, getFeatures(boundName)));
	}

	return bindings;
}
 
Example 4
public static ResolvedSourceCode getResolvedSourceCode(
		final String sourceCode, final Set<Set<ASTNode>> nodeBindings) {
	final JavascriptTokenizer tokenizer = new JavascriptTokenizer();
	final SortedMap<Integer, String> tokenPositions = tokenizer
			.tokenListWithPos(sourceCode.toCharArray());
	final SortedMap<Integer, Integer> positionToIndex = getTokenIndexForPostion(tokenPositions);
	final List<String> tokens = Lists.newArrayList(tokenPositions.values());

	final ArrayListMultimap<String, TokenNameBinding> bindings = ArrayListMultimap
			.create();

	for (final Set<ASTNode> boundName : nodeBindings) {
		if (boundName.isEmpty()) {
			continue;
		}
		final List<Integer> boundPositions = Lists.newArrayList();
		for (final ASTNode name : boundName) {
			// Convert position to token index and add
			final int tokenIdx = positionToIndex.get(name
					.getStartPosition());
			boundPositions.add(tokenIdx);
		}
		bindings.put(tokens.get(boundPositions.get(0)),
				new TokenNameBinding(Sets.newTreeSet(boundPositions),
						tokens, getFeatures(boundName)));
	}

	return new ResolvedSourceCode(tokens, bindings);
}
 
Example 5
public static ResolvedSourceCode getResolvedSourceCode(
		final String sourceCode, final Set<Set<ASTNode>> nodeBindings) {
	final JavascriptTokenizer tokenizer = new JavascriptTokenizer();
	final SortedMap<Integer, String> tokenPositions = tokenizer
			.tokenListWithPos(sourceCode.toCharArray());
	final SortedMap<Integer, Integer> positionToIndex = getTokenIndexForPostion(tokenPositions);
	final List<String> tokens = Lists.newArrayList(tokenPositions.values());

	final ArrayListMultimap<String, TokenNameBinding> bindings = ArrayListMultimap
			.create();

	for (final Set<ASTNode> boundName : nodeBindings) {
		if (boundName.isEmpty()) {
			continue;
		}
		final List<Integer> boundPositions = Lists.newArrayList();
		for (final ASTNode name : boundName) {
			// Convert position to token index and add
			final int tokenIdx = positionToIndex.get(name
					.getStartPosition());
			boundPositions.add(tokenIdx);
		}
		bindings.put(tokens.get(boundPositions.get(0)),
				new TokenNameBinding(Sets.newTreeSet(boundPositions),
						tokens, getFeatures(boundName)));
	}

	return new ResolvedSourceCode(tokens, bindings);
}
 
Example 6
public static ResolvedSourceCode getResolvedSourceCode(
		final String sourceCode, final Set<Set<ASTNode>> nodeBindings) {
	final JavascriptTokenizer tokenizer = new JavascriptTokenizer();
	final SortedMap<Integer, String> tokenPositions = tokenizer
			.tokenListWithPos(sourceCode.toCharArray());
	final SortedMap<Integer, Integer> positionToIndex = getTokenIndexForPostion(tokenPositions);
	final List<String> tokens = Lists.newArrayList(tokenPositions.values());

	final ArrayListMultimap<String, TokenNameBinding> bindings = ArrayListMultimap
			.create();

	for (final Set<ASTNode> boundName : nodeBindings) {
		if (boundName.isEmpty()) {
			continue;
		}
		final List<Integer> boundPositions = Lists.newArrayList();
		for (final ASTNode name : boundName) {
			// Convert position to token index and add
			final int tokenIdx = positionToIndex.get(name
					.getStartPosition());
			boundPositions.add(tokenIdx);
		}
		bindings.put(tokens.get(boundPositions.get(0)),
				new TokenNameBinding(Sets.newTreeSet(boundPositions),
						tokens, getFeatures(boundName)));
	}

	return new ResolvedSourceCode(tokens, bindings);
}