Java Code Examples for org.apache.flink.core.fs.Path#getName()

private void createUploadFilePaths(
	FileStatus[] fileStatuses,
	Map<StateHandleID, StreamStateHandle> sstFiles,
	Map<StateHandleID, Path> sstFilePaths,
	Map<StateHandleID, Path> miscFilePaths) {
	for (FileStatus fileStatus : fileStatuses) {
		final Path filePath = fileStatus.getPath();
		final String fileName = filePath.getName();
		final StateHandleID stateHandleID = new StateHandleID(fileName);

		if (fileName.endsWith(SST_FILE_SUFFIX)) {
			final boolean existsAlready = baseSstFiles != null && baseSstFiles.contains(stateHandleID);

			if (existsAlready) {
				// we introduce a placeholder state handle, that is replaced with the
				// original from the shared state registry (created from a previous checkpoint)
				sstFiles.put(stateHandleID, new PlaceholderStreamStateHandle());
			} else {
				sstFilePaths.put(stateHandleID, filePath);
			}
		} else {
			miscFilePaths.put(stateHandleID, filePath);
		}
	}
}
 

private List<String> listNewFiles(FileSystem fileSystem) throws IOException {
	List<String> files = new ArrayList<String>();

	FileStatus[] statuses = fileSystem.listStatus(new Path(path));

	if (statuses == null) {
		LOG.warn("Path does not exist: {}", path);
	} else {
		for (FileStatus status : statuses) {
			Path filePath = status.getPath();
			String fileName = filePath.getName();
			long modificationTime = status.getModificationTime();

			if (!isFiltered(fileName, modificationTime)) {
				files.add(filePath.toString());
				modificationTimes.put(fileName, modificationTime);
			}
		}
	}

	return files;
}
 

/**
 * Make partition spec from path.
 *
 * @param currPath partition file path.
 * @return Sequential partition specs.
 */
public static LinkedHashMap<String, String> extractPartitionSpecFromPath(Path currPath) {
	LinkedHashMap<String, String> fullPartSpec = new LinkedHashMap<>();
	List<String[]> kvs = new ArrayList<>();
	do {
		String component = currPath.getName();
		Matcher m = PARTITION_NAME_PATTERN.matcher(component);
		if (m.matches()) {
			String k = unescapePathName(m.group(1));
			String v = unescapePathName(m.group(2));
			String[] kv = new String[2];
			kv[0] = k;
			kv[1] = v;
			kvs.add(kv);
		}
		currPath = currPath.getParent();
	} while (currPath != null && !currPath.getName().isEmpty());

	// reverse the list since we checked the part from leaf dir to table's base dir
	for (int i = kvs.size(); i > 0; i--) {
		fullPartSpec.put(kvs.get(i - 1)[0], kvs.get(i - 1)[1]);
	}

	return fullPartSpec;
}
 

private List<String> listNewFiles(FileSystem fileSystem) throws IOException {
	List<String> files = new ArrayList<String>();

	FileStatus[] statuses = fileSystem.listStatus(new Path(path));

	if (statuses == null) {
		LOG.warn("Path does not exist: {}", path);
	} else {
		for (FileStatus status : statuses) {
			Path filePath = status.getPath();
			String fileName = filePath.getName();
			long modificationTime = status.getModificationTime();

			if (!isFiltered(fileName, modificationTime)) {
				files.add(filePath.toString());
				modificationTimes.put(fileName, modificationTime);
			}
		}
	}

	return files;
}
 

/**
 * This recreates the new working directory of the recovered RocksDB instance and links/copies the contents from
 * a local state.
 */
private void restoreInstanceDirectoryFromPath(Path source, String instanceRocksDBPath) throws IOException {

	FileSystem fileSystem = source.getFileSystem();

	final FileStatus[] fileStatuses = fileSystem.listStatus(source);

	if (fileStatuses == null) {
		throw new IOException("Cannot list file statues. Directory " + source + " does not exist.");
	}

	for (FileStatus fileStatus : fileStatuses) {
		final Path filePath = fileStatus.getPath();
		final String fileName = filePath.getName();
		File restoreFile = new File(source.getPath(), fileName);
		File targetFile = new File(instanceRocksDBPath, fileName);
		if (fileName.endsWith(SST_FILE_SUFFIX)) {
			// hardlink'ing the immutable sst-files.
			Files.createLink(targetFile.toPath(), restoreFile.toPath());
		} else {
			// true copy for all other files.
			Files.copy(restoreFile.toPath(), targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
		}
	}
}
 

private List<String> listNewFiles(FileSystem fileSystem) throws IOException {
	List<String> files = new ArrayList<String>();

	FileStatus[] statuses = fileSystem.listStatus(new Path(path));

	if (statuses == null) {
		LOG.warn("Path does not exist: {}", path);
	} else {
		for (FileStatus status : statuses) {
			Path filePath = status.getPath();
			String fileName = filePath.getName();
			long modificationTime = status.getModificationTime();

			if (!isFiltered(fileName, modificationTime)) {
				files.add(filePath.toString());
				modificationTimes.put(fileName, modificationTime);
			}
		}
	}

	return files;
}
 

private void createUploadFilePaths(
	FileStatus[] fileStatuses,
	Map<StateHandleID, StreamStateHandle> sstFiles,
	Map<StateHandleID, Path> sstFilePaths,
	Map<StateHandleID, Path> miscFilePaths) {
	for (FileStatus fileStatus : fileStatuses) {
		final Path filePath = fileStatus.getPath();
		final String fileName = filePath.getName();
		final StateHandleID stateHandleID = new StateHandleID(fileName);

		if (fileName.endsWith(SST_FILE_SUFFIX)) {
			final boolean existsAlready = baseSstFiles != null && baseSstFiles.contains(stateHandleID);

			if (existsAlready) {
				// we introduce a placeholder state handle, that is replaced with the
				// original from the shared state registry (created from a previous checkpoint)
				sstFiles.put(stateHandleID, new PlaceholderStreamStateHandle());
			} else {
				sstFilePaths.put(stateHandleID, filePath);
			}
		} else {
			miscFilePaths.put(stateHandleID, filePath);
		}
	}
}
 

/**
 * This recreates the new working directory of the recovered RocksDB instance and links/copies the contents from
 * a local state.
 */
private void restoreInstanceDirectoryFromPath(Path source, String instanceRocksDBPath) throws IOException {

	FileSystem fileSystem = source.getFileSystem();

	final FileStatus[] fileStatuses = fileSystem.listStatus(source);

	if (fileStatuses == null) {
		throw new IOException("Cannot list file statues. Directory " + source + " does not exist.");
	}

	for (FileStatus fileStatus : fileStatuses) {
		final Path filePath = fileStatus.getPath();
		final String fileName = filePath.getName();
		File restoreFile = new File(source.getPath(), fileName);
		File targetFile = new File(instanceRocksDBPath, fileName);
		if (fileName.endsWith(SST_FILE_SUFFIX)) {
			// hardlink'ing the immutable sst-files.
			Files.createLink(targetFile.toPath(), restoreFile.toPath());
		} else {
			// true copy for all other files.
			Files.copy(restoreFile.toPath(), targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
		}
	}
}
 

@Override
public PythonDriverOptions createResult(@Nonnull CommandLine commandLine) throws FlinkParseException {
	String entrypointModule = null;
	final List<Path> pythonLibFiles = new ArrayList<>();

	if (commandLine.hasOption(PY_OPTION.getOpt()) && commandLine.hasOption(PYMODULE_OPTION.getOpt())) {
		throw new FlinkParseException("Cannot use options -py and -pym simultaneously.");
	} else if (commandLine.hasOption(PY_OPTION.getOpt())) {
		Path file = new Path(commandLine.getOptionValue(PY_OPTION.getOpt()));
		String fileName = file.getName();
		if (fileName.endsWith(".py")) {
			entrypointModule = fileName.substring(0, fileName.length() - 3);
			pythonLibFiles.add(file);
		}
	} else if (commandLine.hasOption(PYMODULE_OPTION.getOpt())) {
		entrypointModule = commandLine.getOptionValue(PYMODULE_OPTION.getOpt());
	} else {
		throw new FlinkParseException(
			"The Python entrypoint has not been specified. It can be specified with options -py or -pym");
	}

	if (commandLine.hasOption(PYFILES_OPTION.getOpt())) {
		pythonLibFiles.addAll(
			Arrays.stream(commandLine.getOptionValue(PYFILES_OPTION.getOpt()).split(","))
				.map(Path::new)
				.collect(Collectors.toCollection(ArrayList::new)));
	} else if (commandLine.hasOption(PYMODULE_OPTION.getOpt())) {
		throw new FlinkParseException("Option -pym must be used in conjunction with `--pyFiles`");
	}

	return new PythonDriverOptions(entrypointModule, pythonLibFiles, commandLine.getArgList());
}
 

private static List<String> findTestFiles() throws Exception {
	List<String> files = new ArrayList<>();
	FileSystem fs = FileSystem.getLocalFileSystem();
	FileStatus[] status = fs.listStatus(getBaseTestPythonDir());
	for (FileStatus f : status) {
		Path filePath = f.getPath();
		String fileName = filePath.getName();
		if (fileName.startsWith("test_") && fileName.endsWith(".py")) {
			files.add(filePath.getPath());
		}
	}
	return files;
}
 

/**
 * Moves files from srcDir to destDir.
 */
private void renameFiles(List<Path> srcDirs, Path destDir) throws Exception {
	for (Path srcDir : srcDirs) {
		if (!srcDir.equals(destDir)) {
			FileStatus[] srcFiles = listStatusWithoutHidden(fs, srcDir);
			if (srcFiles != null) {
				for (FileStatus srcFile : srcFiles) {
					Path srcPath = srcFile.getPath();
					Path destPath = new Path(destDir, srcPath.getName());
					fs.rename(srcPath, destPath);
				}
			}
		}
	}
}
 

private static void addToPythonPath(PythonEnvironment env, List<Path> pythonFiles) throws IOException {
	List<String> pythonPathList = new ArrayList<>();
	Path tmpDirPath = new Path(env.tempDirectory);

	for (Path pythonFile : pythonFiles) {
		String sourceFileName = pythonFile.getName();
		// add random UUID parent directory to avoid name conflict.
		Path targetPath = new Path(
			tmpDirPath,
			String.join(File.separator, UUID.randomUUID().toString(), sourceFileName));
		if (!pythonFile.getFileSystem().isDistributedFS()) {
			// if the path is local file, try to create symbolic link.
			new File(targetPath.getParent().toString()).mkdir();
			createSymbolicLink(
				Paths.get(new File(pythonFile.getPath()).getAbsolutePath()),
				Paths.get(targetPath.toString()));
		} else {
			try {
				FileUtils.copy(pythonFile, targetPath, true);
			} catch (Exception e) {
				LOG.error("Error occurred when copying {} to {}, skipping...", pythonFile, targetPath, e);
				continue;
			}
		}
		if (Files.isRegularFile(Paths.get(targetPath.toString()).toRealPath()) && sourceFileName.endsWith(".py")) {
			// add the parent directory of .py file itself to PYTHONPATH
			pythonPathList.add(targetPath.getParent().toString());
		} else {
			pythonPathList.add(targetPath.toString());
		}
	}

	if (env.pythonPath != null && !env.pythonPath.isEmpty()) {
		pythonPathList.add(env.pythonPath);
	}
	env.pythonPath = String.join(File.pathSeparator, pythonPathList);
}
 

/**
 * Prepares PythonEnvironment to start python process.
 *
 * @param pythonLibFiles The dependent Python files.
 * @return PythonEnvironment the Python environment which will be executed in Python process.
 */
public static PythonEnvironment preparePythonEnvironment(List<Path> pythonLibFiles) throws IOException {
	PythonEnvironment env = new PythonEnvironment();

	// 1. setup temporary local directory for the user files
	String tmpDir = System.getProperty("java.io.tmpdir") +
		File.separator + "pyflink" + File.separator + UUID.randomUUID();

	Path tmpDirPath = new Path(tmpDir);
	FileSystem fs = tmpDirPath.getFileSystem();
	if (fs.exists(tmpDirPath)) {
		fs.delete(tmpDirPath, true);
	}
	fs.mkdirs(tmpDirPath);

	env.workingDirectory = tmpDirPath.toString();

	StringBuilder pythonPathEnv = new StringBuilder();

	pythonPathEnv.append(env.workingDirectory);

	// 2. create symbolLink in the working directory for the pyflink dependency libs.
	List<java.nio.file.Path> pythonLibs = getLibFiles(FLINK_OPT_DIR_PYTHON);
	for (java.nio.file.Path libPath : pythonLibs) {
		java.nio.file.Path symbolicLinkFilePath = FileSystems.getDefault().getPath(env.workingDirectory,
			libPath.getFileName().toString());
		createSymbolicLinkForPyflinkLib(libPath, symbolicLinkFilePath);
		pythonPathEnv.append(File.pathSeparator);
		pythonPathEnv.append(symbolicLinkFilePath.toString());
	}

	// 3. copy relevant python files to tmp dir and set them in PYTHONPATH.
	for (Path pythonFile : pythonLibFiles) {
		String sourceFileName = pythonFile.getName();
		Path targetPath = new Path(tmpDirPath, sourceFileName);
		FileUtils.copy(pythonFile, targetPath, true);
		String targetFileNames = Files.walk(Paths.get(targetPath.toString()))
			.filter(Files::isRegularFile)
			.filter(f -> !f.toString().endsWith(".py"))
			.map(java.nio.file.Path::toString)
			.collect(Collectors.joining(File.pathSeparator));
		pythonPathEnv.append(File.pathSeparator);
		pythonPathEnv.append(targetFileNames);
	}

	// 4. add the parent directory to PYTHONPATH for files suffixed with .py
	String pyFileParents = Files.walk(Paths.get(tmpDirPath.toString()))
		.filter(file -> file.toString().endsWith(".py"))
		.map(java.nio.file.Path::getParent)
		.distinct()
		.map(java.nio.file.Path::toString)
		.collect(Collectors.joining(File.pathSeparator));
	if (!StringUtils.isNullOrWhitespaceOnly(pyFileParents)) {
		pythonPathEnv.append(File.pathSeparator);
		pythonPathEnv.append(pyFileParents);
	}

	env.pythonPath = pythonPathEnv.toString();
	return env;
}
 

private void assertParent(Path parent, Path child) {
	Path path = new Path(parent, child.getName());
	assertEquals(path, child);
}
 

private void assertParent(Path parent, Path child) {
	Path path = new Path(parent, child.getName());
	assertEquals(path, child);
}
 

private String getKey() {
	Path parent = path.getParent();
	return parent.getName().startsWith("task-") ?
			parent.getParent().getName() :
			parent.getParent().getParent().getName() + Path.SEPARATOR + parent.getName();
}
 

private void assertParent(Path parent, Path child) {
	Path path = new Path(parent, child.getName());
	assertEquals(path, child);
}