codemining.languagetools.IScopeExtractor Java Examples

The following examples show how to use codemining.languagetools.IScopeExtractor. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HumanEvaluationOutput.java    From naturalize with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
/**
 * @param args
 * @throws IllegalAccessException
 * @throws InstantiationException
 * @throws ClassNotFoundException
 */
public static void main(String[] args) throws InstantiationException,
		IllegalAccessException, ClassNotFoundException {
	if (args.length < 5) {
		System.err
				.println("Usage: <projectDir> <tokenizerClass> variable|method|class examplesToGenerate <renamerClass> [renamerParams]");
		return;
	}

	final File directory = new File(args[0]);
	final IScopeExtractor extractor = ScopesTUI
			.getScopeExtractorByName(args[2]);
	final long nExamples = Long.parseLong(args[3]);

	final Class<? extends ITokenizer> tokenizerName = (Class<? extends ITokenizer>) Class
			.forName(args[1]);
	final ITokenizer tokenizer = tokenizerName.newInstance();

	final String renamerClass = args[4];

	final HumanEvaluationOutput heo = new HumanEvaluationOutput(directory,
			extractor, renamerClass, args.length == 5 ? null : args[5],
			tokenizer, nExamples);
	heo.getOutput();

}
Example #2
Source File: PerturbationEvaluator.java    From naturalize with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
/**
 * @param args
 */
public static void main(String[] args) {
	if (args.length < 3) {
		System.err
				.println("Usage <directory> <renamerClass> variable|method");
		return;
	}

	final IScopeExtractor scopeExtractor = ScopesTUI
			.getScopeExtractorByName(args[2]);

	final String renamerClass = args[1];

	final File directory = new File(args[0]);

	final PerturbationEvaluator pe = new PerturbationEvaluator(directory,
			new JavaTokenizer(), scopeExtractor, renamerClass);

	pe.performEvaluation();
	pe.er.printStats();

}
Example #3
Source File: SegmentRenamingSuggestion.java    From naturalize with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
public static SortedSet<Suggestion> getVariableSuggestions(
		final File currentFile, final File directory, final boolean useUNK)
		throws IOException {
	final ITokenizer tokenizer = new JavaTokenizer();

	final AbstractIdentifierRenamings renamer = new BaseIdentifierRenamings(
			tokenizer);

	final Collection<java.io.File> trainingFiles = FileUtils.listFiles(
			directory, tokenizer.getFileFilter(),
			DirectoryFileFilter.DIRECTORY);

	trainingFiles.remove(currentFile);

	renamer.buildRenamingModel(trainingFiles);

	final IScopeExtractor scopeExtractor = new VariableScopeExtractor.VariableScopeSnippetExtractor();

	final SegmentRenamingSuggestion suggestion = new SegmentRenamingSuggestion(
			renamer, scopeExtractor, useUNK);

	return suggestion.rankSuggestions(currentFile);
}
Example #4
Source File: VariableUsageStatistics.java    From naturalize with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
/**
 * @param args
 * @throws IOException
 */
public static void main(String[] args) throws IOException {
	if (args.length < 2) {
		System.err.println("Usage <projectFolder> variable|method");
		return;
	}

	final IScopeExtractor scopeExtractor = ScopesTUI
			.getScopeExtractorByName(args[1]);

	final File directory = new File(args[0]);
	final VariableUsageStatistics vus = new VariableUsageStatistics(
			directory, new JavaTokenizer(), scopeExtractor);
	vus.extractStats();
	vus.printStats();
}
Example #5
Source File: ScopesTUI.java    From codemining-core with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
/**
 * @param args
 * @throws IOException
 */
public static void main(String[] args) throws IOException {
	if (args.length < 2) {
		System.err.println("Usage <file> all|variable|method|type");
		return;
	}
	final String name = args[1];
	final IScopeExtractor scopeExtractor = getScopeExtractorByName(name);

	System.out.println(scopeExtractor.getFromFile(new File(args[0])));

}
Example #6
Source File: AllScopeExtractor.java    From tassal with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
@Override
public Multimap<Scope, String> getFromString(final String file,
		final ParseType parseType) {
	final Multimap<Scope, String> scopes = TreeMultimap.create();
	for (final IScopeExtractor extractor : allExtractors) {
		scopes.putAll(extractor.getFromString(file, parseType));
	}
	return scopes;
}
Example #7
Source File: AllScopeExtractor.java    From tassal with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
@Override
public Multimap<Scope, String> getFromNode(ASTNode node) {
	final Multimap<Scope, String> scopes = TreeMultimap.create();
	for (final IScopeExtractor extractor : allExtractors) {
		scopes.putAll(extractor.getFromNode(node));
	}
	return scopes;
}
Example #8
Source File: AllScopeExtractor.java    From tassal with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
@Override
public Multimap<Scope, String> getFromFile(final File file)
		throws IOException {
	final Multimap<Scope, String> scopes = TreeMultimap.create();
	for (final IScopeExtractor extractor : allExtractors) {
		scopes.putAll(extractor.getFromFile(file));
	}
	return scopes;
}
Example #9
Source File: PerturbationEvaluator.java    From naturalize with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
public PerturbationEvaluator(final File directory,
		final ITokenizer tokenizer, final IScopeExtractor scopeExtractor,
		final String renamerClass) {
	allFiles = FileUtils.listFiles(directory, tokenizer.getFileFilter(),
			DirectoryFileFilter.DIRECTORY);
	this.tokenizer = tokenizer;
	this.scopeExtractor = scopeExtractor;
	this.renamerClass = renamerClass;
	varRenamer = new ScopedIdentifierRenaming(scopeExtractor,
			ParseType.COMPILATION_UNIT);
}
Example #10
Source File: ScopesTUI.java    From tassal with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
/**
 * @param args
 * @throws IOException
 */
public static void main(String[] args) throws IOException {
	if (args.length < 2) {
		System.err.println("Usage <file> all|variable|method|type");
		return;
	}
	final String name = args[1];
	final IScopeExtractor scopeExtractor = getScopeExtractorByName(name);

	System.out.println(scopeExtractor.getFromFile(new File(args[0])));

}
Example #11
Source File: VariableUsageStatistics.java    From naturalize with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
/**
 * 
 */
public VariableUsageStatistics(final File directory,
		final ITokenizer tokenizer, final IScopeExtractor scopeExtractor) {
	allFiles = FileUtils.listFiles(directory, tokenizer.getFileFilter(),
			DirectoryFileFilter.DIRECTORY);
	this.scopeExtractor = scopeExtractor;
	statistics = Maps.newTreeMap();
	this.tokenizer = tokenizer;
}
Example #12
Source File: AllScopeExtractor.java    From codemining-core with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
@Override
public Multimap<Scope, String> getFromFile(final File file)
		throws IOException {
	final Multimap<Scope, String> scopes = TreeMultimap.create();
	for (final IScopeExtractor extractor : allExtractors) {
		scopes.putAll(extractor.getFromFile(file));
	}
	return scopes;
}
Example #13
Source File: SegmentRenamingSuggestion.java    From naturalize with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
/**
 * 
 */
public SegmentRenamingSuggestion(final AbstractIdentifierRenamings renamer,
		final IScopeExtractor extractor, final boolean useUNK) {
	this.renamer = renamer;
	scopeExtractor = extractor;
	this.useUNK = useUNK;
}
Example #14
Source File: SegmentRenamingSuggestion.java    From naturalize with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
public static void main(final String[] args)
		throws IllegalArgumentException, SecurityException,
		InstantiationException, IllegalAccessException,
		InvocationTargetException, NoSuchMethodException,
		ClassNotFoundException, IOException {
	if (args.length < 4) {
		System.err
				.println("Usage <TestFile> <TrainDirectory> <renamerClass> variable|method");
		return;
	}

	final ITokenizer tokenizer = new JavaTokenizer();

	final AbstractIdentifierRenamings renamer = (AbstractIdentifierRenamings) Class
			.forName(args[2]).getDeclaredConstructor(ITokenizer.class)
			.newInstance(tokenizer);

	renamer.buildRenamingModel(FileUtils.listFiles(new File(args[1]),
			tokenizer.getFileFilter(), DirectoryFileFilter.DIRECTORY));

	final IScopeExtractor scopeExtractor = ScopesTUI
			.getScopeExtractorByName(args[3]);
	final SegmentRenamingSuggestion suggestion = new SegmentRenamingSuggestion(
			renamer, scopeExtractor, true);

	System.out.println(suggestion.rankSuggestions(new File(args[0])));

}
Example #15
Source File: AllScopeExtractor.java    From codemining-core with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
@Override
public Multimap<Scope, String> getFromNode(ASTNode node) {
	final Multimap<Scope, String> scopes = TreeMultimap.create();
	for (final IScopeExtractor extractor : allExtractors) {
		scopes.putAll(extractor.getFromNode(node));
	}
	return scopes;
}
Example #16
Source File: SnippetScorer.java    From naturalize with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
public static SnippetSuggestions scoreSnippet(final ASTNode node,
		final AbstractIdentifierRenamings renamer,
		final IScopeExtractor scopeExtractor,
		final boolean filterSuggestions, final boolean useUNK)
		throws IOException {
	final SegmentRenamingSuggestion srs = new SegmentRenamingSuggestion(
			renamer, scopeExtractor, useUNK);
	final SortedSet<Suggestion> suggestions = srs.rankSuggestions(node);
	final SortedSet<Suggestion> filteredSuggestions = Sets.newTreeSet();

	double score = 0;
	for (final Suggestion suggestion : suggestions) {
		final SortedSet<Renaming> filteredRenamings;
		if (filterSuggestions) {
			filteredRenamings = applyThresholdToRenamings(
					suggestion.getRenamings(),
					getThresholdFor(suggestion.scope.type));
		} else {
			filteredRenamings = suggestion.getRenamings();
		}
		final double currentScore = getScore(filteredRenamings,
				suggestion.identifierName, useUNK);
		if (Double.compare(currentScore, 0) == 0) {
			continue;
		}
		filteredSuggestions.add(new Suggestion(suggestion
				.getIdentifierName(), suggestion.scope, -currentScore,
				filteredRenamings));
		score += currentScore;
	}
	return new SnippetSuggestions(Sets.newTreeSet(filteredSuggestions),
			score);
}
Example #17
Source File: AllScopeExtractor.java    From codemining-core with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
@Override
public Multimap<Scope, String> getFromString(final String file,
		final ParseType parseType) {
	final Multimap<Scope, String> scopes = TreeMultimap.create();
	for (final IScopeExtractor extractor : allExtractors) {
		scopes.putAll(extractor.getFromString(file, parseType));
	}
	return scopes;
}
Example #18
Source File: LeaveOneOutEvaluator.java    From naturalize with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
public void performEvaluation(final IScopeExtractor scopeExtractor,
		final String renamerClass, final String additionalParams) {
	final ParallelThreadPool threadPool = new ParallelThreadPool();
	int fileNo = 0;
	for (final File fi : allFiles) {
		threadPool.pushTask(new ModelEvaluator(fi, scopeExtractor,
				renamerClass, additionalParams));
		fileNo++;
		if (fileNo % 40 == 0) {
			threadPool.pushTask(new Printer());
		}
	}
	threadPool.waitForTermination();
}
Example #19
Source File: ScopesTUI.java    From api-mining with GNU General Public License v3.0 5 votes vote down vote up 
/**
 * @param args
 * @throws IOException
 */
public static void main(String[] args) throws IOException {
	if (args.length < 2) {
		System.err.println("Usage <file> all|variable|method|type");
		return;
	}
	final String name = args[1];
	final IScopeExtractor scopeExtractor = getScopeExtractorByName(name);

	System.out.println(scopeExtractor.getFromFile(new File(args[0])));

}
Example #20
Source File: AllScopeExtractor.java    From api-mining with GNU General Public License v3.0 5 votes vote down vote up 
@Override
public Multimap<Scope, String> getFromFile(final File file)
		throws IOException {
	final Multimap<Scope, String> scopes = TreeMultimap.create();
	for (final IScopeExtractor extractor : allExtractors) {
		scopes.putAll(extractor.getFromFile(file));
	}
	return scopes;
}
Example #21
Source File: AllScopeExtractor.java    From api-mining with GNU General Public License v3.0 5 votes vote down vote up 
@Override
public Multimap<Scope, String> getFromNode(ASTNode node) {
	final Multimap<Scope, String> scopes = TreeMultimap.create();
	for (final IScopeExtractor extractor : allExtractors) {
		scopes.putAll(extractor.getFromNode(node));
	}
	return scopes;
}
Example #22
Source File: AllScopeExtractor.java    From api-mining with GNU General Public License v3.0 5 votes vote down vote up 
@Override
public Multimap<Scope, String> getFromString(final String file,
		final ParseType parseType) {
	final Multimap<Scope, String> scopes = TreeMultimap.create();
	for (final IScopeExtractor extractor : allExtractors) {
		scopes.putAll(extractor.getFromString(file, parseType));
	}
	return scopes;
}
Example #23
Source File: UnecessaryNamesHumanEvalOutput.java    From naturalize with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
public static void main(final String args[]) {
	if (args.length < 3) {
		System.err
				.println("Usage <directory> class|method|variable|all <renamerClass> [renamerArgs]");
		return;
	}
	final IScopeExtractor extractor = ScopesTUI
			.getScopeExtractorByName(args[1]);
	final File directory = new File(args[0]);
	final String renamerClass = args[2];
	final String renamerConstructorParams;
	if (args.length < 4) {
		renamerConstructorParams = null;
	} else {
		renamerConstructorParams = args[3];
	}

	final SortedMap<SnippetSuggestions, String> methodScores = getSnippetRanking(
			extractor, directory, renamerClass, renamerConstructorParams);

	final List<Entry<SnippetSuggestions, String>> topRenamings = getTop10pct(methodScores);

	int i = 1;
	for (final Entry<SnippetSuggestions, String> entry : topRenamings
			.subList(
					0,
					topRenamings.size() < NUM_EXAMPLES_TO_PRODUCE ? topRenamings
							.size() : NUM_EXAMPLES_TO_PRODUCE)) {
		CodeReviewAssistant.printRenaming(entry.getKey(), entry.getValue(),
				i);
		i++;
	}
}
Example #24
Source File: UnkRenamingsEval.java    From naturalize with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
/**
 * @param args
 */
public static void main(String[] args) {
	if (args.length < 2) {
		System.err.println("Usge <projectDir> all|variable|method|type");
		return;
	}

	final File prj = new File(args[0]);
	final IScopeExtractor extractor = ScopesTUI
			.getScopeExtractorByName(args[1]);

	final UnkRenamingsEval eval = new UnkRenamingsEval(prj, extractor);
	eval.runExperiment();
}
Example #25
Source File: UnkRenamingsEval.java    From naturalize with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
/**
 * 
 */
public UnkRenamingsEval(final File directory,
		final IScopeExtractor extractor) {
	allFiles = FileUtils.listFiles(directory,
			(new JavaTokenizer()).getFileFilter(),
			DirectoryFileFilter.DIRECTORY);
	scopeExtractor = extractor;
}
Example #26
Source File: HumanEvaluationOutput.java    From naturalize with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
/**
 * 
 */
public HumanEvaluationOutput(final File directory,
		final IScopeExtractor extractor, final String renamerClass,
		final String renamerConstructorParam, final ITokenizer tokenizer,
		final long nExamples) {
	allFiles = Lists.newArrayList(FileUtils.listFiles(directory,
			tokenizer.getFileFilter(), DirectoryFileFilter.DIRECTORY));
	Collections.shuffle(allFiles);
	this.tokenizer = tokenizer;
	this.nExamples = nExamples;
	this.renamerClass = renamerClass;
	this.renamerConstructorParam = renamerConstructorParam;
	scopeExtractor = extractor;
}
Example #27
Source File: NamingEvaluator.java    From naturalize with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
public void performEvaluation(final Collection<File> files,
		final IScopeExtractor scopeExtractor) {
	final ParallelThreadPool threadPool = new ParallelThreadPool();
	int fileNo = 0;
	for (final File fi : files) {
		threadPool.pushTask(new RenamingEvaluator(fi, scopeExtractor));
		fileNo++;
		if (fileNo % 50 == 0) {
			threadPool.pushTask(new Printer());
		}
	}
	threadPool.waitForTermination();
	(new Printer()).run();
}
Example #28
Source File: SelectionSuggestionEval.java    From naturalize with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
/**
 * 
 */
public SelectionSuggestionEval(final File directory,
		final ITokenizer codeTokenizer, final IScopeExtractor extractor) {
	tokenizer = codeTokenizer;
	allFiles = FileUtils.listFiles(directory, tokenizer.getFileFilter(),
			DirectoryFileFilter.DIRECTORY);
	scopeExtractor = extractor;
}
Example #29
Source File: StylishEval.java    From naturalize with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
/**
 * 
 */
public StylishEval(final File directory, final ITokenizer codeTokenizer,
		final IScopeExtractor extractor) {
	tokenizer = codeTokenizer;
	allFiles = FileUtils.listFiles(directory, tokenizer.getFileFilter(),
			DirectoryFileFilter.DIRECTORY);
	scopeExtractor = extractor;
}
Example #30
Source File: LeaveOneOutEvaluator.java    From naturalize with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
/**
 * @param args
 * @throws IllegalAccessException
 * @throws InstantiationException
 * @throws ClassNotFoundException
 */
public static void main(String[] args) throws InstantiationException,
		IllegalAccessException, ClassNotFoundException,
		SerializationException {
	if (args.length < 5) {
		System.err
				.println("Usage <folder> <tokenizerClass> <wrapperClass> variable|method <renamingClass> [<renamerConstrParams> ..]");
		return;
	}

	final File directory = new File(args[0]);

	final Class<? extends ITokenizer> tokenizerName = (Class<? extends ITokenizer>) Class
			.forName(args[1]);
	final ITokenizer tokenizer = tokenizerName.newInstance();

	final Class<? extends AbstractNGramLM> smoothedNgramClass = (Class<? extends AbstractNGramLM>) Class
			.forName(args[2]);

	final LeaveOneOutEvaluator eval = new LeaveOneOutEvaluator(directory,
			tokenizer, smoothedNgramClass);

	final IScopeExtractor scopeExtractor = ScopesTUI
			.getScopeExtractorByName(args[3]);

	eval.performEvaluation(scopeExtractor, args[4],
			args.length == 6 ? args[5] : null);
}