Java Code Examples for org.apache.flink.util.FileUtils#copy()

@Override
public Path call() throws IOException {
	// let exceptions propagate. we can retrieve them later from
	// the future and report them upon access to the result
	FileUtils.copy(filePath, cachedPath, this.executable);
	return cachedPath;
}
 

@Override
public Path call() throws IOException {
	// let exceptions propagate. we can retrieve them later from
	// the future and report them upon access to the result
	FileUtils.copy(filePath, cachedPath, this.executable);
	return cachedPath;
}
 

private static void addToPythonPath(PythonEnvironment env, List<Path> pythonFiles) throws IOException {
	List<String> pythonPathList = new ArrayList<>();
	Path tmpDirPath = new Path(env.tempDirectory);

	for (Path pythonFile : pythonFiles) {
		String sourceFileName = pythonFile.getName();
		// add random UUID parent directory to avoid name conflict.
		Path targetPath = new Path(
			tmpDirPath,
			String.join(File.separator, UUID.randomUUID().toString(), sourceFileName));
		if (!pythonFile.getFileSystem().isDistributedFS()) {
			// if the path is local file, try to create symbolic link.
			new File(targetPath.getParent().toString()).mkdir();
			createSymbolicLink(
				Paths.get(new File(pythonFile.getPath()).getAbsolutePath()),
				Paths.get(targetPath.toString()));
		} else {
			try {
				FileUtils.copy(pythonFile, targetPath, true);
			} catch (Exception e) {
				LOG.error("Error occurred when copying {} to {}, skipping...", pythonFile, targetPath, e);
				continue;
			}
		}
		if (Files.isRegularFile(Paths.get(targetPath.toString()).toRealPath()) && sourceFileName.endsWith(".py")) {
			// add the parent directory of .py file itself to PYTHONPATH
			pythonPathList.add(targetPath.getParent().toString());
		} else {
			pythonPathList.add(targetPath.toString());
		}
	}

	if (env.pythonPath != null && !env.pythonPath.isEmpty()) {
		pythonPathList.add(env.pythonPath);
	}
	env.pythonPath = String.join(File.pathSeparator, pythonPathList);
}
 

@Override
public Path call() throws IOException {
	// let exceptions propagate. we can retrieve them later from
	// the future and report them upon access to the result
	FileUtils.copy(filePath, cachedPath, this.executable);
	if (isZipped) {
		return FileUtils.expandDirectory(cachedPath, cachedPath.getParent());
	}
	return cachedPath;
}
 

private static BucketState<String> readBucketStateFromTemplate(final String scenarioName, final int version) throws IOException {
	final java.nio.file.Path scenarioPath =  getResourcePath(scenarioName, version);

	// clear the scenario files first
	FileUtils.deleteDirectory(scenarioPath.toFile());

	// prepare the scenario files
	FileUtils.copy(new Path(scenarioPath.toString() + "-template"), new Path(scenarioPath.toString()), false);

	return readBucketState(scenarioName, version);
}
 

private static void copyFile(Path source, Path targetDirectory, String name) throws IOException {
	Path targetFilePath = new Path(targetDirectory, name);
	deleteIfExists(targetFilePath);
	FileUtils.copy(source, targetFilePath, true);
}
 

private static void copyFile(Path source, Path targetDirectory, String name) throws IOException {
	Path targetFilePath = new Path(targetDirectory, name);
	deleteIfExists(targetFilePath);
	FileUtils.copy(source, targetFilePath, true);
}
 

/**
 * Prepares PythonEnvironment to start python process.
 *
 * @param pythonLibFiles The dependent Python files.
 * @return PythonEnvironment the Python environment which will be executed in Python process.
 */
public static PythonEnvironment preparePythonEnvironment(List<Path> pythonLibFiles) throws IOException {
	PythonEnvironment env = new PythonEnvironment();

	// 1. setup temporary local directory for the user files
	String tmpDir = System.getProperty("java.io.tmpdir") +
		File.separator + "pyflink" + File.separator + UUID.randomUUID();

	Path tmpDirPath = new Path(tmpDir);
	FileSystem fs = tmpDirPath.getFileSystem();
	if (fs.exists(tmpDirPath)) {
		fs.delete(tmpDirPath, true);
	}
	fs.mkdirs(tmpDirPath);

	env.workingDirectory = tmpDirPath.toString();

	StringBuilder pythonPathEnv = new StringBuilder();

	pythonPathEnv.append(env.workingDirectory);

	// 2. create symbolLink in the working directory for the pyflink dependency libs.
	List<java.nio.file.Path> pythonLibs = getLibFiles(FLINK_OPT_DIR_PYTHON);
	for (java.nio.file.Path libPath : pythonLibs) {
		java.nio.file.Path symbolicLinkFilePath = FileSystems.getDefault().getPath(env.workingDirectory,
			libPath.getFileName().toString());
		createSymbolicLinkForPyflinkLib(libPath, symbolicLinkFilePath);
		pythonPathEnv.append(File.pathSeparator);
		pythonPathEnv.append(symbolicLinkFilePath.toString());
	}

	// 3. copy relevant python files to tmp dir and set them in PYTHONPATH.
	for (Path pythonFile : pythonLibFiles) {
		String sourceFileName = pythonFile.getName();
		Path targetPath = new Path(tmpDirPath, sourceFileName);
		FileUtils.copy(pythonFile, targetPath, true);
		String targetFileNames = Files.walk(Paths.get(targetPath.toString()))
			.filter(Files::isRegularFile)
			.filter(f -> !f.toString().endsWith(".py"))
			.map(java.nio.file.Path::toString)
			.collect(Collectors.joining(File.pathSeparator));
		pythonPathEnv.append(File.pathSeparator);
		pythonPathEnv.append(targetFileNames);
	}

	// 4. add the parent directory to PYTHONPATH for files suffixed with .py
	String pyFileParents = Files.walk(Paths.get(tmpDirPath.toString()))
		.filter(file -> file.toString().endsWith(".py"))
		.map(java.nio.file.Path::getParent)
		.distinct()
		.map(java.nio.file.Path::toString)
		.collect(Collectors.joining(File.pathSeparator));
	if (!StringUtils.isNullOrWhitespaceOnly(pyFileParents)) {
		pythonPathEnv.append(File.pathSeparator);
		pythonPathEnv.append(pyFileParents);
	}

	env.pythonPath = pythonPathEnv.toString();
	return env;
}
 

private void constructFilesDirectory(Map<String, String> env) throws IOException {
	// link or copy python files to filesDirectory and add them to PYTHONPATH
	List<String> pythonFilePaths = new ArrayList<>();
	for (Map.Entry<String, String> entry : dependencyInfo.getPythonFiles().entrySet()) {
		// The origin file name will be wiped when downloaded from the distributed cache, restore the origin name to
		// make sure the python files could be imported.
		// The path of the restored python file will be as following:
		// ${baseDirectory}/${PYTHON_FILES_DIR}/${distributedCacheFileName}/${originFileName}
		String distributedCacheFileName = new File(entry.getKey()).getName();
		String originFileName = entry.getValue();

		Path target = FileSystems.getDefault().getPath(filesDirectory, distributedCacheFileName, originFileName);
		if (!target.getParent().toFile().mkdirs()) {
			throw new IOException(
				String.format("Could not create the directory: %s !", target.getParent().toString()));
		}
		Path src = FileSystems.getDefault().getPath(entry.getKey());
		try {
			Files.createSymbolicLink(target, src);
		} catch (IOException e) {
			LOG.warn(String.format(
				"Could not create the symbolic link of: %s, the link path is %s, fallback to copy.", src, target),
				e);
			FileUtils.copy(
				new org.apache.flink.core.fs.Path(src.toUri()),
				new org.apache.flink.core.fs.Path(target.toUri()), false);
		}

		File pythonFile = new File(entry.getKey());
		String pythonPath;
		if (pythonFile.isFile() && originFileName.endsWith(".py")) {
			// If the python file is file with suffix .py, add the parent directory to PYTHONPATH.
			pythonPath = String.join(File.separator, filesDirectory, distributedCacheFileName);
		} else {
			pythonPath = String.join(File.separator, filesDirectory, distributedCacheFileName, originFileName);
		}
		pythonFilePaths.add(pythonPath);
	}
	appendToPythonPath(env, pythonFilePaths);
	LOG.info("PYTHONPATH of python worker: {}", env.get("PYTHONPATH"));
}
 

private static void moveToTemplateDirectory(java.nio.file.Path scenarioPath) throws IOException {
	FileUtils.copy(new Path(scenarioPath.toString()), new Path(scenarioPath.toString() + "-template"), false);
	FileUtils.deleteDirectory(scenarioPath.toFile());
}