codemining.java.tokenizers.JavaTokenizer Java Examples

/**
 * @param args
 */
public static void main(String[] args) {
	if (args.length < 3) {
		System.err
				.println("Usage <directory> <renamerClass> variable|method");
		return;
	}

	final IScopeExtractor scopeExtractor = ScopesTUI
			.getScopeExtractorByName(args[2]);

	final String renamerClass = args[1];

	final File directory = new File(args[0]);

	final PerturbationEvaluator pe = new PerturbationEvaluator(directory,
			new JavaTokenizer(), scopeExtractor, renamerClass);

	pe.performEvaluation();
	pe.er.printStats();

}
 

/**
 * Extract the bindings from the input folder to the output file, using the
 * bindingExtractor.
 *
 * @param inputFolder
 * @param outputFile
 * @param bindingExtractor
 * @throws IOException
 * @throws JsonIOException
 */
public static void extractBindings(final File inputFolder,
		final File outputFile,
		final AbstractJavaNameBindingsExtractor bindingExtractor)
		throws IOException, JsonIOException {
	final Collection<File> allFiles = FileUtils
			.listFiles(inputFolder, JavaTokenizer.javaCodeFileFilter,
					DirectoryFileFilter.DIRECTORY);
	final List<SerializableResolvedSourceCode> resolvedCode = allFiles
			.parallelStream()
			.map(f -> getResolvedCode(f, bindingExtractor))
			.filter(r -> r != null)
			.map(r -> SerializableResolvedSourceCode
					.fromResolvedSourceCode(r))
			.filter(s -> !s.boundVariables.isEmpty())
			.collect(Collectors.toList());

	final FileWriter writer = new FileWriter(outputFile);
	try {
		final Gson gson = new Gson();
		gson.toJson(resolvedCode, writer);
	} finally {
		writer.close();
	}
}
 

/**
 * @param args
 */
public static void main(final String[] args) {
	if (args.length != 1) {
		System.err.println("Usage <codeFolder>");
		System.exit(-1);
	}
	final File directory = new File(args[0]);

	final Collection<File> allFiles = FileUtils
			.listFiles(directory, JavaTokenizer.javaCodeFileFilter,
					DirectoryFileFilter.DIRECTORY);

	final JavaTypeHierarchyExtractor jthe = new JavaTypeHierarchyExtractor();
	jthe.addFilesToCorpus(allFiles);

	System.out.println(jthe);
}
 

/**
 * @param args
 */
public static void main(final String[] args) {
	if (args.length != 1) {
		System.err.println("Usage <codeFolder>");
		System.exit(-1);
	}
	final File directory = new File(args[0]);

	final Collection<File> allFiles = FileUtils
			.listFiles(directory, JavaTokenizer.javaCodeFileFilter,
					DirectoryFileFilter.DIRECTORY);

	final JavaTypeHierarchyExtractor jthe = new JavaTypeHierarchyExtractor();
	jthe.addFilesToCorpus(allFiles);

	System.out.println(jthe);
}
 

/**
 * Hacky way to compare snippets.
 *
 * @param snippet
 * @return
 */
private String normalizeCode(final char[] snippet) {
	final List<String> tokens = (new JavaTokenizer())
			.tokenListFromCode(snippet);

	final StringBuffer bf = new StringBuffer();
	for (final String token : tokens) {
		if (token.equals(ITokenizer.SENTENCE_START)
				|| token.equals(ITokenizer.SENTENCE_END)) {
			continue;
		} else {
			bf.append(token);
		}
		bf.append(" ");
	}
	return bf.toString();

}
 

/**
 * @param args
 */
public static void main(String[] args) {
	if (args.length < 5) {
		System.err.println("Usage <folder> <junkNames> ...");
		return;
	}

	final File directory = new File(args[0]);

	final ITokenizer tokenizer = new JavaTokenizer();

	final Class<? extends AbstractNGramLM> smoothedNgramClass = StupidBackoff.class;

	final CommonNameRenamingEvaluator evaluator = new CommonNameRenamingEvaluator(
			directory, tokenizer, smoothedNgramClass,
			Sets.newTreeSet(Arrays.asList(args).subList(1, args.length)));
	evaluator.evaluate();
	evaluator.printResults();

}
 

private void doFirstScan(final File repositoryDir, final String sha) {
	for (final File f : FileUtils
			.listFiles(repositoryDir, JavaTokenizer.javaCodeFileFilter,
					DirectoryFileFilter.DIRECTORY)) {
		final String fileInRepo = f.getAbsolutePath().substring(
				(int) (repositoryDir.getAbsolutePath().length() + 1));
		Set<IdentifierInformation> identiferInfos;
		try {
			identiferInfos = infoScanner.scanFile(f, sha);
			identiferInfos
					.forEach(info -> {
						final IdentifierInformationThroughTime iitt = new IdentifierInformationThroughTime();
						iitt.addInformation(info);
						currentStateOfIdentifiers.put(fileInRepo, iitt);
					});
		} catch (final IOException e) {
			LOGGER.severe("Could not find file " + f + "\n"
					+ ExceptionUtils.getFullStackTrace(e));
		}

	}
}
 

/**
 * @param args
 */
public static void main(final String[] args) {
	if (args.length != 1) {
		System.err.println("Usage <codeFolder>");
		System.exit(-1);
	}
	final File directory = new File(args[0]);

	final Collection<File> allFiles = FileUtils
			.listFiles(directory, JavaTokenizer.javaCodeFileFilter,
					DirectoryFileFilter.DIRECTORY);

	final JavaTypeHierarchyExtractor jthe = new JavaTypeHierarchyExtractor();
	jthe.addFilesToCorpus(allFiles);

	System.out.println(jthe);
}
 

/**
 * Hacky way to compare snippets.
 *
 * @param snippet
 * @return
 */
private String normalizeCode(final char[] snippet) {
	final List<String> tokens = (new JavaTokenizer())
			.tokenListFromCode(snippet);

	final StringBuffer bf = new StringBuffer();
	for (final String token : tokens) {
		if (token.equals(ITokenizer.SENTENCE_START)
				|| token.equals(ITokenizer.SENTENCE_END)) {
			continue;
		} else {
			bf.append(token);
		}
		bf.append(" ");
	}
	return bf.toString();

}
 

/**
 * Extract the bindings from the input folder to the output file, using the
 * bindingExtractor.
 *
 * @param inputFolder
 * @param outputFile
 * @param bindingExtractor
 * @throws IOException
 * @throws JsonIOException
 */
public static void extractBindings(final File inputFolder,
		final File outputFile,
		final AbstractJavaNameBindingsExtractor bindingExtractor)
		throws IOException, JsonIOException {
	final Collection<File> allFiles = FileUtils
			.listFiles(inputFolder, JavaTokenizer.javaCodeFileFilter,
					DirectoryFileFilter.DIRECTORY);
	final List<SerializableResolvedSourceCode> resolvedCode = allFiles
			.parallelStream()
			.map(f -> getResolvedCode(f, bindingExtractor))
			.filter(r -> r != null)
			.map(r -> SerializableResolvedSourceCode
					.fromResolvedSourceCode(r))
			.filter(s -> !s.boundVariables.isEmpty())
			.collect(Collectors.toList());

	final FileWriter writer = new FileWriter(outputFile);
	try {
		final Gson gson = new Gson();
		gson.toJson(resolvedCode, writer);
	} finally {
		writer.close();
	}
}
 

public static SortedSet<Suggestion> getVariableSuggestions(
		final File currentFile, final File directory, final boolean useUNK)
		throws IOException {
	final ITokenizer tokenizer = new JavaTokenizer();

	final AbstractIdentifierRenamings renamer = new BaseIdentifierRenamings(
			tokenizer);

	final Collection<java.io.File> trainingFiles = FileUtils.listFiles(
			directory, tokenizer.getFileFilter(),
			DirectoryFileFilter.DIRECTORY);

	trainingFiles.remove(currentFile);

	renamer.buildRenamingModel(trainingFiles);

	final IScopeExtractor scopeExtractor = new VariableScopeExtractor.VariableScopeSnippetExtractor();

	final SegmentRenamingSuggestion suggestion = new SegmentRenamingSuggestion(
			renamer, scopeExtractor, useUNK);

	return suggestion.rankSuggestions(currentFile);
}
 

/**
 * @param args
 * @throws IOException
 */
public static void main(String[] args) throws IOException {
	if (args.length < 2) {
		System.err.println("Usage <projectFolder> variable|method");
		return;
	}

	final IScopeExtractor scopeExtractor = ScopesTUI
			.getScopeExtractorByName(args[1]);

	final File directory = new File(args[0]);
	final VariableUsageStatistics vus = new VariableUsageStatistics(
			directory, new JavaTokenizer(), scopeExtractor);
	vus.extractStats();
	vus.printStats();
}
 

/**
 * Hacky way to compare snippets.
 *
 * @param snippet
 * @return
 */
private String normalizeCode(final char[] snippet) {
	final List<String> tokens = (new JavaTokenizer())
			.tokenListFromCode(snippet);

	final StringBuffer bf = new StringBuffer();
	for (final String token : tokens) {
		if (token.equals(ITokenizer.SENTENCE_START)
				|| token.equals(ITokenizer.SENTENCE_END)) {
			continue;
		} else {
			bf.append(token);
		}
		bf.append(" ");
	}
	return bf.toString();

}
 

/**
 * Extract the bindings from the input folder to the output file, using the
 * bindingExtractor.
 *
 * @param inputFolder
 * @param outputFile
 * @param bindingExtractor
 * @throws IOException
 * @throws JsonIOException
 */
public static void extractBindings(final File inputFolder,
		final File outputFile,
		final AbstractJavaNameBindingsExtractor bindingExtractor)
		throws IOException, JsonIOException {
	final Collection<File> allFiles = FileUtils
			.listFiles(inputFolder, JavaTokenizer.javaCodeFileFilter,
					DirectoryFileFilter.DIRECTORY);
	final List<SerializableResolvedSourceCode> resolvedCode = allFiles
			.parallelStream()
			.map(f -> getResolvedCode(f, bindingExtractor))
			.filter(r -> r != null)
			.map(r -> SerializableResolvedSourceCode
					.fromResolvedSourceCode(r))
			.filter(s -> !s.boundVariables.isEmpty())
			.collect(Collectors.toList());

	final FileWriter writer = new FileWriter(outputFile);
	try {
		final Gson gson = new Gson();
		gson.toJson(resolvedCode, writer);
	} finally {
		writer.close();
	}
}
 

public static void main(final String[] args)
		throws IllegalArgumentException, SecurityException,
		InstantiationException, IllegalAccessException,
		InvocationTargetException, NoSuchMethodException,
		ClassNotFoundException, IOException {
	if (args.length < 4) {
		System.err
				.println("Usage <TestFile> <TrainDirectory> <renamerClass> variable|method");
		return;
	}

	final ITokenizer tokenizer = new JavaTokenizer();

	final AbstractIdentifierRenamings renamer = (AbstractIdentifierRenamings) Class
			.forName(args[2]).getDeclaredConstructor(ITokenizer.class)
			.newInstance(tokenizer);

	renamer.buildRenamingModel(FileUtils.listFiles(new File(args[1]),
			tokenizer.getFileFilter(), DirectoryFileFilter.DIRECTORY));

	final IScopeExtractor scopeExtractor = ScopesTUI
			.getScopeExtractorByName(args[3]);
	final SegmentRenamingSuggestion suggestion = new SegmentRenamingSuggestion(
			renamer, scopeExtractor, true);

	System.out.println(suggestion.rankSuggestions(new File(args[0])));

}
 

/**
 * @param repositoryDirectory
 * @param walkingStrategy
 * @throws IOException
 */
public ChangingIdentifiersRepositoryWalker(
		final String repositoryDirectory,
		final ICommitWalkingStrategy walkingStrategy) throws IOException {
	super(repositoryDirectory, walkingStrategy);
	editListRetriever = new EditListRetriever(repository,
			JavaTokenizer.javaCodeFileFilter);
}
 

@Test
public void testTokenTypes() {
	ITokenizer tokenizer = new JavaTokenizer();
	assertEquals(
			tokenizer.getTokenFromString("hello"),
			new ITokenizer.FullToken("hello", tokenizer.getIdentifierType()));
	assertEquals(
			tokenizer.getTokenFromString("{"),
			new ITokenizer.FullToken("{", Integer
					.toString(ITerminalSymbols.TokenNameLBRACE)));

}
 

public JavaMethodDeclarationBindingExtractor(
		final boolean includeOverrides, final File inputFolder) {
	super(new JavaTokenizer());
	this.includeOverrides = includeOverrides;
	if (!includeOverrides) {
		pti = buildProjectTypeInformation(inputFolder);
	} else {
		pti = null;
	}
}
 

/**
 * @param args
 */
public static void main(final String[] args) {
	if (args.length != 2) {
		System.err.println("Usage <fullyQualifiedClass> <directory>");
		System.exit(-1);
	}

	final File directory = new File(args[1]);
	final String qualifiedClass = args[0];

	for (final File fi : FileUtils
			.listFiles(directory, JavaTokenizer.javaCodeFileFilter,
					DirectoryFileFilter.DIRECTORY)) {
		try {
			final List<ASTNode> usages = usagePoints(qualifiedClass, fi);
			if (!usages.isEmpty()) {
				System.out.println(fi.getAbsolutePath());
				for (final ASTNode node : usages) {
					System.out
							.println("----------------------------------------------");
					System.out.println(node);
				}
			}
		} catch (final Exception e) {
			System.err.println("Error processing " + fi.getName());
		}

	}

}
 

public JavaMethodDeclarationBindingExtractor(
		final boolean includeOverrides, final File inputFolder) {
	super(new JavaTokenizer());
	this.includeOverrides = includeOverrides;
	if (!includeOverrides) {
		pti = buildProjectTypeInformation(inputFolder);
	} else {
		pti = null;
	}
}
 

public static void main(final String[] args) {
	if (args.length != 1) {
		System.err.println("Usage <projectDir>");
		System.exit(-1);
	}

	final MethodsInClass mic = new MethodsInClass();
	mic.scan(FileUtils
			.listFiles(new File(args[0]), JavaTokenizer.javaCodeFileFilter,
					DirectoryFileFilter.DIRECTORY));
	System.out.println(mic);
}
 

public void collect() {
	final Collection<File> allFiles = FileUtils
			.listFiles(projectDirectory, JavaTokenizer.javaCodeFileFilter,
					DirectoryFileFilter.DIRECTORY);
	methodInformation.scan(allFiles);
	final JavaTypeHierarchyExtractor hierarchyExtractor = new JavaTypeHierarchyExtractor();
	hierarchyExtractor.addFilesToCorpus(allFiles);
	hierarchy = hierarchyExtractor.getHierarchy();
}
 

@Test
public void testTokenTypes() {
	ITokenizer tokenizer = new JavaTokenizer();
	assertEquals(
			tokenizer.getTokenFromString("hello"),
			new ITokenizer.FullToken("hello", tokenizer.getIdentifierType()));
	assertEquals(
			tokenizer.getTokenFromString("{"),
			new ITokenizer.FullToken("{", Integer
					.toString(ITerminalSymbols.TokenNameLBRACE)));

}
 

private boolean snippetMatchesAstTokens(final String snippetCode,
		final ASTNode node) {
	final JavaTokenizer tokenizer = new JavaTokenizer();
	final List<String> snippetTokens = tokenizer
			.tokenListFromCode(snippetCode.toCharArray());
	final List<String> astTokens = tokenizer.tokenListFromCode(node
			.toString().toCharArray());
	return astTokens.equals(snippetTokens);
}
 

/**
 * @param args
 */
public static void main(final String[] args) {
	if (args.length != 2) {
		System.err.println("Usage <fullyQualifiedClass> <directory>");
		System.exit(-1);
	}

	final File directory = new File(args[1]);
	final String qualifiedClass = args[0];

	for (final File fi : FileUtils
			.listFiles(directory, JavaTokenizer.javaCodeFileFilter,
					DirectoryFileFilter.DIRECTORY)) {
		try {
			final List<ASTNode> usages = usagePoints(qualifiedClass, fi);
			if (!usages.isEmpty()) {
				System.out.println(fi.getAbsolutePath());
				for (final ASTNode node : usages) {
					System.out
							.println("----------------------------------------------");
					System.out.println(node);
				}
			}
		} catch (final Exception e) {
			System.err.println("Error processing " + fi.getName());
		}

	}

}
 

public JavaMethodDeclarationBindingExtractor(
		final boolean includeOverrides, final File inputFolder) {
	super(new JavaTokenizer());
	this.includeOverrides = includeOverrides;
	if (!includeOverrides) {
		pti = buildProjectTypeInformation(inputFolder);
	} else {
		pti = null;
	}
}
 

public static void main(final String[] args) {
	if (args.length != 1) {
		System.err.println("Usage <projectDir>");
		System.exit(-1);
	}

	final MethodsInClass mic = new MethodsInClass();
	mic.scan(FileUtils
			.listFiles(new File(args[0]), JavaTokenizer.javaCodeFileFilter,
					DirectoryFileFilter.DIRECTORY));
	System.out.println(mic);
}
 

public void collect() {
	final Collection<File> allFiles = FileUtils
			.listFiles(projectDirectory, JavaTokenizer.javaCodeFileFilter,
					DirectoryFileFilter.DIRECTORY);
	methodInformation.scan(allFiles);
	final JavaTypeHierarchyExtractor hierarchyExtractor = new JavaTypeHierarchyExtractor();
	hierarchyExtractor.addFilesToCorpus(allFiles);
	hierarchy = hierarchyExtractor.getHierarchy();
}
 

@Test
public void testTokenTypes() {
	ITokenizer tokenizer = new JavaTokenizer();
	assertEquals(
			tokenizer.getTokenFromString("hello"),
			new ITokenizer.FullToken("hello", tokenizer.getIdentifierType()));
	assertEquals(
			tokenizer.getTokenFromString("{"),
			new ITokenizer.FullToken("{", Integer
					.toString(ITerminalSymbols.TokenNameLBRACE)));

}
 

private boolean snippetMatchesAstTokens(final String snippetCode,
		final ASTNode node) {
	final JavaTokenizer tokenizer = new JavaTokenizer();
	final List<String> snippetTokens = tokenizer
			.tokenListFromCode(snippetCode.toCharArray());
	final List<String> astTokens = tokenizer.tokenListFromCode(node
			.toString().toCharArray());
	return astTokens.equals(snippetTokens);
}