Java Code Examples for org.apache.flink.util.FileUtils#copy()

The following examples show how to use org.apache.flink.util.FileUtils#copy() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FileCache.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public Path call() throws IOException {
	// let exceptions propagate. we can retrieve them later from
	// the future and report them upon access to the result
	FileUtils.copy(filePath, cachedPath, this.executable);
	return cachedPath;
}
 
Example 2
Source File: FileCache.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public Path call() throws IOException {
	// let exceptions propagate. we can retrieve them later from
	// the future and report them upon access to the result
	FileUtils.copy(filePath, cachedPath, this.executable);
	return cachedPath;
}
 
Example 3
Source File: PythonEnvUtils.java    From flink with Apache License 2.0 5 votes vote down vote up
private static void addToPythonPath(PythonEnvironment env, List<Path> pythonFiles) throws IOException {
	List<String> pythonPathList = new ArrayList<>();
	Path tmpDirPath = new Path(env.tempDirectory);

	for (Path pythonFile : pythonFiles) {
		String sourceFileName = pythonFile.getName();
		// add random UUID parent directory to avoid name conflict.
		Path targetPath = new Path(
			tmpDirPath,
			String.join(File.separator, UUID.randomUUID().toString(), sourceFileName));
		if (!pythonFile.getFileSystem().isDistributedFS()) {
			// if the path is local file, try to create symbolic link.
			new File(targetPath.getParent().toString()).mkdir();
			createSymbolicLink(
				Paths.get(new File(pythonFile.getPath()).getAbsolutePath()),
				Paths.get(targetPath.toString()));
		} else {
			try {
				FileUtils.copy(pythonFile, targetPath, true);
			} catch (Exception e) {
				LOG.error("Error occurred when copying {} to {}, skipping...", pythonFile, targetPath, e);
				continue;
			}
		}
		if (Files.isRegularFile(Paths.get(targetPath.toString()).toRealPath()) && sourceFileName.endsWith(".py")) {
			// add the parent directory of .py file itself to PYTHONPATH
			pythonPathList.add(targetPath.getParent().toString());
		} else {
			pythonPathList.add(targetPath.toString());
		}
	}

	if (env.pythonPath != null && !env.pythonPath.isEmpty()) {
		pythonPathList.add(env.pythonPath);
	}
	env.pythonPath = String.join(File.pathSeparator, pythonPathList);
}
 
Example 4
Source File: FileCache.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public Path call() throws IOException {
	// let exceptions propagate. we can retrieve them later from
	// the future and report them upon access to the result
	FileUtils.copy(filePath, cachedPath, this.executable);
	if (isZipped) {
		return FileUtils.expandDirectory(cachedPath, cachedPath.getParent());
	}
	return cachedPath;
}
 
Example 5
Source File: BucketStateSerializerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private static BucketState<String> readBucketStateFromTemplate(final String scenarioName, final int version) throws IOException {
	final java.nio.file.Path scenarioPath =  getResourcePath(scenarioName, version);

	// clear the scenario files first
	FileUtils.deleteDirectory(scenarioPath.toFile());

	// prepare the scenario files
	FileUtils.copy(new Path(scenarioPath.toString() + "-template"), new Path(scenarioPath.toString()), false);

	return readBucketState(scenarioName, version);
}
 
Example 6
Source File: PythonPlanBinder.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
private static void copyFile(Path source, Path targetDirectory, String name) throws IOException {
	Path targetFilePath = new Path(targetDirectory, name);
	deleteIfExists(targetFilePath);
	FileUtils.copy(source, targetFilePath, true);
}
 
Example 7
Source File: PythonStreamBinder.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
private static void copyFile(Path source, Path targetDirectory, String name) throws IOException {
	Path targetFilePath = new Path(targetDirectory, name);
	deleteIfExists(targetFilePath);
	FileUtils.copy(source, targetFilePath, true);
}
 
Example 8
Source File: PythonEnvUtils.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Prepares PythonEnvironment to start python process.
 *
 * @param pythonLibFiles The dependent Python files.
 * @return PythonEnvironment the Python environment which will be executed in Python process.
 */
public static PythonEnvironment preparePythonEnvironment(List<Path> pythonLibFiles) throws IOException {
	PythonEnvironment env = new PythonEnvironment();

	// 1. setup temporary local directory for the user files
	String tmpDir = System.getProperty("java.io.tmpdir") +
		File.separator + "pyflink" + File.separator + UUID.randomUUID();

	Path tmpDirPath = new Path(tmpDir);
	FileSystem fs = tmpDirPath.getFileSystem();
	if (fs.exists(tmpDirPath)) {
		fs.delete(tmpDirPath, true);
	}
	fs.mkdirs(tmpDirPath);

	env.workingDirectory = tmpDirPath.toString();

	StringBuilder pythonPathEnv = new StringBuilder();

	pythonPathEnv.append(env.workingDirectory);

	// 2. create symbolLink in the working directory for the pyflink dependency libs.
	List<java.nio.file.Path> pythonLibs = getLibFiles(FLINK_OPT_DIR_PYTHON);
	for (java.nio.file.Path libPath : pythonLibs) {
		java.nio.file.Path symbolicLinkFilePath = FileSystems.getDefault().getPath(env.workingDirectory,
			libPath.getFileName().toString());
		createSymbolicLinkForPyflinkLib(libPath, symbolicLinkFilePath);
		pythonPathEnv.append(File.pathSeparator);
		pythonPathEnv.append(symbolicLinkFilePath.toString());
	}

	// 3. copy relevant python files to tmp dir and set them in PYTHONPATH.
	for (Path pythonFile : pythonLibFiles) {
		String sourceFileName = pythonFile.getName();
		Path targetPath = new Path(tmpDirPath, sourceFileName);
		FileUtils.copy(pythonFile, targetPath, true);
		String targetFileNames = Files.walk(Paths.get(targetPath.toString()))
			.filter(Files::isRegularFile)
			.filter(f -> !f.toString().endsWith(".py"))
			.map(java.nio.file.Path::toString)
			.collect(Collectors.joining(File.pathSeparator));
		pythonPathEnv.append(File.pathSeparator);
		pythonPathEnv.append(targetFileNames);
	}

	// 4. add the parent directory to PYTHONPATH for files suffixed with .py
	String pyFileParents = Files.walk(Paths.get(tmpDirPath.toString()))
		.filter(file -> file.toString().endsWith(".py"))
		.map(java.nio.file.Path::getParent)
		.distinct()
		.map(java.nio.file.Path::toString)
		.collect(Collectors.joining(File.pathSeparator));
	if (!StringUtils.isNullOrWhitespaceOnly(pyFileParents)) {
		pythonPathEnv.append(File.pathSeparator);
		pythonPathEnv.append(pyFileParents);
	}

	env.pythonPath = pythonPathEnv.toString();
	return env;
}
 
Example 9
Source File: ProcessPythonEnvironmentManager.java    From flink with Apache License 2.0 4 votes vote down vote up
private void constructFilesDirectory(Map<String, String> env) throws IOException {
	// link or copy python files to filesDirectory and add them to PYTHONPATH
	List<String> pythonFilePaths = new ArrayList<>();
	for (Map.Entry<String, String> entry : dependencyInfo.getPythonFiles().entrySet()) {
		// The origin file name will be wiped when downloaded from the distributed cache, restore the origin name to
		// make sure the python files could be imported.
		// The path of the restored python file will be as following:
		// ${baseDirectory}/${PYTHON_FILES_DIR}/${distributedCacheFileName}/${originFileName}
		String distributedCacheFileName = new File(entry.getKey()).getName();
		String originFileName = entry.getValue();

		Path target = FileSystems.getDefault().getPath(filesDirectory, distributedCacheFileName, originFileName);
		if (!target.getParent().toFile().mkdirs()) {
			throw new IOException(
				String.format("Could not create the directory: %s !", target.getParent().toString()));
		}
		Path src = FileSystems.getDefault().getPath(entry.getKey());
		try {
			Files.createSymbolicLink(target, src);
		} catch (IOException e) {
			LOG.warn(String.format(
				"Could not create the symbolic link of: %s, the link path is %s, fallback to copy.", src, target),
				e);
			FileUtils.copy(
				new org.apache.flink.core.fs.Path(src.toUri()),
				new org.apache.flink.core.fs.Path(target.toUri()), false);
		}

		File pythonFile = new File(entry.getKey());
		String pythonPath;
		if (pythonFile.isFile() && originFileName.endsWith(".py")) {
			// If the python file is file with suffix .py, add the parent directory to PYTHONPATH.
			pythonPath = String.join(File.separator, filesDirectory, distributedCacheFileName);
		} else {
			pythonPath = String.join(File.separator, filesDirectory, distributedCacheFileName, originFileName);
		}
		pythonFilePaths.add(pythonPath);
	}
	appendToPythonPath(env, pythonFilePaths);
	LOG.info("PYTHONPATH of python worker: {}", env.get("PYTHONPATH"));
}
 
Example 10
Source File: BucketStateSerializerTest.java    From flink with Apache License 2.0 4 votes vote down vote up
private static void moveToTemplateDirectory(java.nio.file.Path scenarioPath) throws IOException {
	FileUtils.copy(new Path(scenarioPath.toString()), new Path(scenarioPath.toString() + "-template"), false);
	FileUtils.deleteDirectory(scenarioPath.toFile());
}