Java Code Examples for org.apache.flink.util.FileUtils#copy()
The following examples show how to use
org.apache.flink.util.FileUtils#copy() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FileCache.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public Path call() throws IOException { // let exceptions propagate. we can retrieve them later from // the future and report them upon access to the result FileUtils.copy(filePath, cachedPath, this.executable); return cachedPath; }
Example 2
Source File: FileCache.java From flink with Apache License 2.0 | 5 votes |
@Override public Path call() throws IOException { // let exceptions propagate. we can retrieve them later from // the future and report them upon access to the result FileUtils.copy(filePath, cachedPath, this.executable); return cachedPath; }
Example 3
Source File: PythonEnvUtils.java From flink with Apache License 2.0 | 5 votes |
private static void addToPythonPath(PythonEnvironment env, List<Path> pythonFiles) throws IOException { List<String> pythonPathList = new ArrayList<>(); Path tmpDirPath = new Path(env.tempDirectory); for (Path pythonFile : pythonFiles) { String sourceFileName = pythonFile.getName(); // add random UUID parent directory to avoid name conflict. Path targetPath = new Path( tmpDirPath, String.join(File.separator, UUID.randomUUID().toString(), sourceFileName)); if (!pythonFile.getFileSystem().isDistributedFS()) { // if the path is local file, try to create symbolic link. new File(targetPath.getParent().toString()).mkdir(); createSymbolicLink( Paths.get(new File(pythonFile.getPath()).getAbsolutePath()), Paths.get(targetPath.toString())); } else { try { FileUtils.copy(pythonFile, targetPath, true); } catch (Exception e) { LOG.error("Error occurred when copying {} to {}, skipping...", pythonFile, targetPath, e); continue; } } if (Files.isRegularFile(Paths.get(targetPath.toString()).toRealPath()) && sourceFileName.endsWith(".py")) { // add the parent directory of .py file itself to PYTHONPATH pythonPathList.add(targetPath.getParent().toString()); } else { pythonPathList.add(targetPath.toString()); } } if (env.pythonPath != null && !env.pythonPath.isEmpty()) { pythonPathList.add(env.pythonPath); } env.pythonPath = String.join(File.pathSeparator, pythonPathList); }
Example 4
Source File: FileCache.java From flink with Apache License 2.0 | 5 votes |
@Override public Path call() throws IOException { // let exceptions propagate. we can retrieve them later from // the future and report them upon access to the result FileUtils.copy(filePath, cachedPath, this.executable); if (isZipped) { return FileUtils.expandDirectory(cachedPath, cachedPath.getParent()); } return cachedPath; }
Example 5
Source File: BucketStateSerializerTest.java From flink with Apache License 2.0 | 5 votes |
private static BucketState<String> readBucketStateFromTemplate(final String scenarioName, final int version) throws IOException { final java.nio.file.Path scenarioPath = getResourcePath(scenarioName, version); // clear the scenario files first FileUtils.deleteDirectory(scenarioPath.toFile()); // prepare the scenario files FileUtils.copy(new Path(scenarioPath.toString() + "-template"), new Path(scenarioPath.toString()), false); return readBucketState(scenarioName, version); }
Example 6
Source File: PythonPlanBinder.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
private static void copyFile(Path source, Path targetDirectory, String name) throws IOException { Path targetFilePath = new Path(targetDirectory, name); deleteIfExists(targetFilePath); FileUtils.copy(source, targetFilePath, true); }
Example 7
Source File: PythonStreamBinder.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
private static void copyFile(Path source, Path targetDirectory, String name) throws IOException { Path targetFilePath = new Path(targetDirectory, name); deleteIfExists(targetFilePath); FileUtils.copy(source, targetFilePath, true); }
Example 8
Source File: PythonEnvUtils.java From flink with Apache License 2.0 | 4 votes |
/** * Prepares PythonEnvironment to start python process. * * @param pythonLibFiles The dependent Python files. * @return PythonEnvironment the Python environment which will be executed in Python process. */ public static PythonEnvironment preparePythonEnvironment(List<Path> pythonLibFiles) throws IOException { PythonEnvironment env = new PythonEnvironment(); // 1. setup temporary local directory for the user files String tmpDir = System.getProperty("java.io.tmpdir") + File.separator + "pyflink" + File.separator + UUID.randomUUID(); Path tmpDirPath = new Path(tmpDir); FileSystem fs = tmpDirPath.getFileSystem(); if (fs.exists(tmpDirPath)) { fs.delete(tmpDirPath, true); } fs.mkdirs(tmpDirPath); env.workingDirectory = tmpDirPath.toString(); StringBuilder pythonPathEnv = new StringBuilder(); pythonPathEnv.append(env.workingDirectory); // 2. create symbolLink in the working directory for the pyflink dependency libs. List<java.nio.file.Path> pythonLibs = getLibFiles(FLINK_OPT_DIR_PYTHON); for (java.nio.file.Path libPath : pythonLibs) { java.nio.file.Path symbolicLinkFilePath = FileSystems.getDefault().getPath(env.workingDirectory, libPath.getFileName().toString()); createSymbolicLinkForPyflinkLib(libPath, symbolicLinkFilePath); pythonPathEnv.append(File.pathSeparator); pythonPathEnv.append(symbolicLinkFilePath.toString()); } // 3. copy relevant python files to tmp dir and set them in PYTHONPATH. for (Path pythonFile : pythonLibFiles) { String sourceFileName = pythonFile.getName(); Path targetPath = new Path(tmpDirPath, sourceFileName); FileUtils.copy(pythonFile, targetPath, true); String targetFileNames = Files.walk(Paths.get(targetPath.toString())) .filter(Files::isRegularFile) .filter(f -> !f.toString().endsWith(".py")) .map(java.nio.file.Path::toString) .collect(Collectors.joining(File.pathSeparator)); pythonPathEnv.append(File.pathSeparator); pythonPathEnv.append(targetFileNames); } // 4. add the parent directory to PYTHONPATH for files suffixed with .py String pyFileParents = Files.walk(Paths.get(tmpDirPath.toString())) .filter(file -> file.toString().endsWith(".py")) .map(java.nio.file.Path::getParent) .distinct() .map(java.nio.file.Path::toString) .collect(Collectors.joining(File.pathSeparator)); if (!StringUtils.isNullOrWhitespaceOnly(pyFileParents)) { pythonPathEnv.append(File.pathSeparator); pythonPathEnv.append(pyFileParents); } env.pythonPath = pythonPathEnv.toString(); return env; }
Example 9
Source File: ProcessPythonEnvironmentManager.java From flink with Apache License 2.0 | 4 votes |
private void constructFilesDirectory(Map<String, String> env) throws IOException { // link or copy python files to filesDirectory and add them to PYTHONPATH List<String> pythonFilePaths = new ArrayList<>(); for (Map.Entry<String, String> entry : dependencyInfo.getPythonFiles().entrySet()) { // The origin file name will be wiped when downloaded from the distributed cache, restore the origin name to // make sure the python files could be imported. // The path of the restored python file will be as following: // ${baseDirectory}/${PYTHON_FILES_DIR}/${distributedCacheFileName}/${originFileName} String distributedCacheFileName = new File(entry.getKey()).getName(); String originFileName = entry.getValue(); Path target = FileSystems.getDefault().getPath(filesDirectory, distributedCacheFileName, originFileName); if (!target.getParent().toFile().mkdirs()) { throw new IOException( String.format("Could not create the directory: %s !", target.getParent().toString())); } Path src = FileSystems.getDefault().getPath(entry.getKey()); try { Files.createSymbolicLink(target, src); } catch (IOException e) { LOG.warn(String.format( "Could not create the symbolic link of: %s, the link path is %s, fallback to copy.", src, target), e); FileUtils.copy( new org.apache.flink.core.fs.Path(src.toUri()), new org.apache.flink.core.fs.Path(target.toUri()), false); } File pythonFile = new File(entry.getKey()); String pythonPath; if (pythonFile.isFile() && originFileName.endsWith(".py")) { // If the python file is file with suffix .py, add the parent directory to PYTHONPATH. pythonPath = String.join(File.separator, filesDirectory, distributedCacheFileName); } else { pythonPath = String.join(File.separator, filesDirectory, distributedCacheFileName, originFileName); } pythonFilePaths.add(pythonPath); } appendToPythonPath(env, pythonFilePaths); LOG.info("PYTHONPATH of python worker: {}", env.get("PYTHONPATH")); }
Example 10
Source File: BucketStateSerializerTest.java From flink with Apache License 2.0 | 4 votes |
private static void moveToTemplateDirectory(java.nio.file.Path scenarioPath) throws IOException { FileUtils.copy(new Path(scenarioPath.toString()), new Path(scenarioPath.toString() + "-template"), false); FileUtils.deleteDirectory(scenarioPath.toFile()); }