Java Code Examples for org.apache.flink.core.fs.Path#getName()

The following examples show how to use org.apache.flink.core.fs.Path#getName() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
/**
 * This recreates the new working directory of the recovered RocksDB instance and links/copies the contents from
 * a local state.
 */
private void restoreInstanceDirectoryFromPath(Path source, String instanceRocksDBPath) throws IOException {

	FileSystem fileSystem = source.getFileSystem();

	final FileStatus[] fileStatuses = fileSystem.listStatus(source);

	if (fileStatuses == null) {
		throw new IOException("Cannot list file statues. Directory " + source + " does not exist.");
	}

	for (FileStatus fileStatus : fileStatuses) {
		final Path filePath = fileStatus.getPath();
		final String fileName = filePath.getName();
		File restoreFile = new File(source.getPath(), fileName);
		File targetFile = new File(instanceRocksDBPath, fileName);
		if (fileName.endsWith(SST_FILE_SUFFIX)) {
			// hardlink'ing the immutable sst-files.
			Files.createLink(targetFile.toPath(), restoreFile.toPath());
		} else {
			// true copy for all other files.
			Files.copy(restoreFile.toPath(), targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
		}
	}
}
 
Example 2
private void createUploadFilePaths(
	FileStatus[] fileStatuses,
	Map<StateHandleID, StreamStateHandle> sstFiles,
	Map<StateHandleID, Path> sstFilePaths,
	Map<StateHandleID, Path> miscFilePaths) {
	for (FileStatus fileStatus : fileStatuses) {
		final Path filePath = fileStatus.getPath();
		final String fileName = filePath.getName();
		final StateHandleID stateHandleID = new StateHandleID(fileName);

		if (fileName.endsWith(SST_FILE_SUFFIX)) {
			final boolean existsAlready = baseSstFiles != null && baseSstFiles.contains(stateHandleID);

			if (existsAlready) {
				// we introduce a placeholder state handle, that is replaced with the
				// original from the shared state registry (created from a previous checkpoint)
				sstFiles.put(stateHandleID, new PlaceholderStreamStateHandle());
			} else {
				sstFilePaths.put(stateHandleID, filePath);
			}
		} else {
			miscFilePaths.put(stateHandleID, filePath);
		}
	}
}
 
Example 3
private List<String> listNewFiles(FileSystem fileSystem) throws IOException {
	List<String> files = new ArrayList<String>();

	FileStatus[] statuses = fileSystem.listStatus(new Path(path));

	if (statuses == null) {
		LOG.warn("Path does not exist: {}", path);
	} else {
		for (FileStatus status : statuses) {
			Path filePath = status.getPath();
			String fileName = filePath.getName();
			long modificationTime = status.getModificationTime();

			if (!isFiltered(fileName, modificationTime)) {
				files.add(filePath.toString());
				modificationTimes.put(fileName, modificationTime);
			}
		}
	}

	return files;
}
 
Example 4
/**
 * This recreates the new working directory of the recovered RocksDB instance and links/copies the contents from
 * a local state.
 */
private void restoreInstanceDirectoryFromPath(Path source, String instanceRocksDBPath) throws IOException {

	FileSystem fileSystem = source.getFileSystem();

	final FileStatus[] fileStatuses = fileSystem.listStatus(source);

	if (fileStatuses == null) {
		throw new IOException("Cannot list file statues. Directory " + source + " does not exist.");
	}

	for (FileStatus fileStatus : fileStatuses) {
		final Path filePath = fileStatus.getPath();
		final String fileName = filePath.getName();
		File restoreFile = new File(source.getPath(), fileName);
		File targetFile = new File(instanceRocksDBPath, fileName);
		if (fileName.endsWith(SST_FILE_SUFFIX)) {
			// hardlink'ing the immutable sst-files.
			Files.createLink(targetFile.toPath(), restoreFile.toPath());
		} else {
			// true copy for all other files.
			Files.copy(restoreFile.toPath(), targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
		}
	}
}
 
Example 5
private void createUploadFilePaths(
	FileStatus[] fileStatuses,
	Map<StateHandleID, StreamStateHandle> sstFiles,
	Map<StateHandleID, Path> sstFilePaths,
	Map<StateHandleID, Path> miscFilePaths) {
	for (FileStatus fileStatus : fileStatuses) {
		final Path filePath = fileStatus.getPath();
		final String fileName = filePath.getName();
		final StateHandleID stateHandleID = new StateHandleID(fileName);

		if (fileName.endsWith(SST_FILE_SUFFIX)) {
			final boolean existsAlready = baseSstFiles != null && baseSstFiles.contains(stateHandleID);

			if (existsAlready) {
				// we introduce a placeholder state handle, that is replaced with the
				// original from the shared state registry (created from a previous checkpoint)
				sstFiles.put(stateHandleID, new PlaceholderStreamStateHandle());
			} else {
				sstFilePaths.put(stateHandleID, filePath);
			}
		} else {
			miscFilePaths.put(stateHandleID, filePath);
		}
	}
}
 
Example 6
Source Project: flink   File: FileMonitoringFunction.java    License: Apache License 2.0 6 votes vote down vote up
private List<String> listNewFiles(FileSystem fileSystem) throws IOException {
	List<String> files = new ArrayList<String>();

	FileStatus[] statuses = fileSystem.listStatus(new Path(path));

	if (statuses == null) {
		LOG.warn("Path does not exist: {}", path);
	} else {
		for (FileStatus status : statuses) {
			Path filePath = status.getPath();
			String fileName = filePath.getName();
			long modificationTime = status.getModificationTime();

			if (!isFiltered(fileName, modificationTime)) {
				files.add(filePath.toString());
				modificationTimes.put(fileName, modificationTime);
			}
		}
	}

	return files;
}
 
Example 7
Source Project: flink   File: PartitionPathUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Make partition spec from path.
 *
 * @param currPath partition file path.
 * @return Sequential partition specs.
 */
public static LinkedHashMap<String, String> extractPartitionSpecFromPath(Path currPath) {
	LinkedHashMap<String, String> fullPartSpec = new LinkedHashMap<>();
	List<String[]> kvs = new ArrayList<>();
	do {
		String component = currPath.getName();
		Matcher m = PARTITION_NAME_PATTERN.matcher(component);
		if (m.matches()) {
			String k = unescapePathName(m.group(1));
			String v = unescapePathName(m.group(2));
			String[] kv = new String[2];
			kv[0] = k;
			kv[1] = v;
			kvs.add(kv);
		}
		currPath = currPath.getParent();
	} while (currPath != null && !currPath.getName().isEmpty());

	// reverse the list since we checked the part from leaf dir to table's base dir
	for (int i = kvs.size(); i > 0; i--) {
		fullPartSpec.put(kvs.get(i - 1)[0], kvs.get(i - 1)[1]);
	}

	return fullPartSpec;
}
 
Example 8
Source Project: flink   File: FileMonitoringFunction.java    License: Apache License 2.0 6 votes vote down vote up
private List<String> listNewFiles(FileSystem fileSystem) throws IOException {
	List<String> files = new ArrayList<String>();

	FileStatus[] statuses = fileSystem.listStatus(new Path(path));

	if (statuses == null) {
		LOG.warn("Path does not exist: {}", path);
	} else {
		for (FileStatus status : statuses) {
			Path filePath = status.getPath();
			String fileName = filePath.getName();
			long modificationTime = status.getModificationTime();

			if (!isFiltered(fileName, modificationTime)) {
				files.add(filePath.toString());
				modificationTimes.put(fileName, modificationTime);
			}
		}
	}

	return files;
}
 
Example 9
private static List<String> findTestFiles() throws Exception {
	List<String> files = new ArrayList<>();
	FileSystem fs = FileSystem.getLocalFileSystem();
	FileStatus[] status = fs.listStatus(getBaseTestPythonDir());
	for (FileStatus f : status) {
		Path filePath = f.getPath();
		String fileName = filePath.getName();
		if (fileName.startsWith("test_") && fileName.endsWith(".py")) {
			files.add(filePath.getPath());
		}
	}
	return files;
}
 
Example 10
@Override
public PythonDriverOptions createResult(@Nonnull CommandLine commandLine) throws FlinkParseException {
	String entrypointModule = null;
	final List<Path> pythonLibFiles = new ArrayList<>();

	if (commandLine.hasOption(PY_OPTION.getOpt()) && commandLine.hasOption(PYMODULE_OPTION.getOpt())) {
		throw new FlinkParseException("Cannot use options -py and -pym simultaneously.");
	} else if (commandLine.hasOption(PY_OPTION.getOpt())) {
		Path file = new Path(commandLine.getOptionValue(PY_OPTION.getOpt()));
		String fileName = file.getName();
		if (fileName.endsWith(".py")) {
			entrypointModule = fileName.substring(0, fileName.length() - 3);
			pythonLibFiles.add(file);
		}
	} else if (commandLine.hasOption(PYMODULE_OPTION.getOpt())) {
		entrypointModule = commandLine.getOptionValue(PYMODULE_OPTION.getOpt());
	} else {
		throw new FlinkParseException(
			"The Python entrypoint has not been specified. It can be specified with options -py or -pym");
	}

	if (commandLine.hasOption(PYFILES_OPTION.getOpt())) {
		pythonLibFiles.addAll(
			Arrays.stream(commandLine.getOptionValue(PYFILES_OPTION.getOpt()).split(","))
				.map(Path::new)
				.collect(Collectors.toCollection(ArrayList::new)));
	} else if (commandLine.hasOption(PYMODULE_OPTION.getOpt())) {
		throw new FlinkParseException("Option -pym must be used in conjunction with `--pyFiles`");
	}

	return new PythonDriverOptions(entrypointModule, pythonLibFiles, commandLine.getArgList());
}
 
Example 11
Source Project: flink   File: PartitionLoader.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Moves files from srcDir to destDir.
 */
private void renameFiles(List<Path> srcDirs, Path destDir) throws Exception {
	for (Path srcDir : srcDirs) {
		if (!srcDir.equals(destDir)) {
			FileStatus[] srcFiles = listStatusWithoutHidden(fs, srcDir);
			if (srcFiles != null) {
				for (FileStatus srcFile : srcFiles) {
					Path srcPath = srcFile.getPath();
					Path destPath = new Path(destDir, srcPath.getName());
					fs.rename(srcPath, destPath);
				}
			}
		}
	}
}
 
Example 12
Source Project: flink   File: PythonEnvUtils.java    License: Apache License 2.0 5 votes vote down vote up
private static void addToPythonPath(PythonEnvironment env, List<Path> pythonFiles) throws IOException {
	List<String> pythonPathList = new ArrayList<>();
	Path tmpDirPath = new Path(env.tempDirectory);

	for (Path pythonFile : pythonFiles) {
		String sourceFileName = pythonFile.getName();
		// add random UUID parent directory to avoid name conflict.
		Path targetPath = new Path(
			tmpDirPath,
			String.join(File.separator, UUID.randomUUID().toString(), sourceFileName));
		if (!pythonFile.getFileSystem().isDistributedFS()) {
			// if the path is local file, try to create symbolic link.
			new File(targetPath.getParent().toString()).mkdir();
			createSymbolicLink(
				Paths.get(new File(pythonFile.getPath()).getAbsolutePath()),
				Paths.get(targetPath.toString()));
		} else {
			try {
				FileUtils.copy(pythonFile, targetPath, true);
			} catch (Exception e) {
				LOG.error("Error occurred when copying {} to {}, skipping...", pythonFile, targetPath, e);
				continue;
			}
		}
		if (Files.isRegularFile(Paths.get(targetPath.toString()).toRealPath()) && sourceFileName.endsWith(".py")) {
			// add the parent directory of .py file itself to PYTHONPATH
			pythonPathList.add(targetPath.getParent().toString());
		} else {
			pythonPathList.add(targetPath.toString());
		}
	}

	if (env.pythonPath != null && !env.pythonPath.isEmpty()) {
		pythonPathList.add(env.pythonPath);
	}
	env.pythonPath = String.join(File.pathSeparator, pythonPathList);
}
 
Example 13
private void assertParent(Path parent, Path child) {
	Path path = new Path(parent, child.getName());
	assertEquals(path, child);
}
 
Example 14
Source Project: flink   File: PythonEnvUtils.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Prepares PythonEnvironment to start python process.
 *
 * @param pythonLibFiles The dependent Python files.
 * @return PythonEnvironment the Python environment which will be executed in Python process.
 */
public static PythonEnvironment preparePythonEnvironment(List<Path> pythonLibFiles) throws IOException {
	PythonEnvironment env = new PythonEnvironment();

	// 1. setup temporary local directory for the user files
	String tmpDir = System.getProperty("java.io.tmpdir") +
		File.separator + "pyflink" + File.separator + UUID.randomUUID();

	Path tmpDirPath = new Path(tmpDir);
	FileSystem fs = tmpDirPath.getFileSystem();
	if (fs.exists(tmpDirPath)) {
		fs.delete(tmpDirPath, true);
	}
	fs.mkdirs(tmpDirPath);

	env.workingDirectory = tmpDirPath.toString();

	StringBuilder pythonPathEnv = new StringBuilder();

	pythonPathEnv.append(env.workingDirectory);

	// 2. create symbolLink in the working directory for the pyflink dependency libs.
	List<java.nio.file.Path> pythonLibs = getLibFiles(FLINK_OPT_DIR_PYTHON);
	for (java.nio.file.Path libPath : pythonLibs) {
		java.nio.file.Path symbolicLinkFilePath = FileSystems.getDefault().getPath(env.workingDirectory,
			libPath.getFileName().toString());
		createSymbolicLinkForPyflinkLib(libPath, symbolicLinkFilePath);
		pythonPathEnv.append(File.pathSeparator);
		pythonPathEnv.append(symbolicLinkFilePath.toString());
	}

	// 3. copy relevant python files to tmp dir and set them in PYTHONPATH.
	for (Path pythonFile : pythonLibFiles) {
		String sourceFileName = pythonFile.getName();
		Path targetPath = new Path(tmpDirPath, sourceFileName);
		FileUtils.copy(pythonFile, targetPath, true);
		String targetFileNames = Files.walk(Paths.get(targetPath.toString()))
			.filter(Files::isRegularFile)
			.filter(f -> !f.toString().endsWith(".py"))
			.map(java.nio.file.Path::toString)
			.collect(Collectors.joining(File.pathSeparator));
		pythonPathEnv.append(File.pathSeparator);
		pythonPathEnv.append(targetFileNames);
	}

	// 4. add the parent directory to PYTHONPATH for files suffixed with .py
	String pyFileParents = Files.walk(Paths.get(tmpDirPath.toString()))
		.filter(file -> file.toString().endsWith(".py"))
		.map(java.nio.file.Path::getParent)
		.distinct()
		.map(java.nio.file.Path::toString)
		.collect(Collectors.joining(File.pathSeparator));
	if (!StringUtils.isNullOrWhitespaceOnly(pyFileParents)) {
		pythonPathEnv.append(File.pathSeparator);
		pythonPathEnv.append(pyFileParents);
	}

	env.pythonPath = pythonPathEnv.toString();
	return env;
}
 
Example 15
Source Project: flink   File: FsCheckpointStorageTest.java    License: Apache License 2.0 4 votes vote down vote up
private void assertParent(Path parent, Path child) {
	Path path = new Path(parent, child.getName());
	assertEquals(path, child);
}
 
Example 16
Source Project: flink   File: PartitionWriterTest.java    License: Apache License 2.0 4 votes vote down vote up
private String getKey() {
	Path parent = path.getParent();
	return parent.getName().startsWith("task-") ?
			parent.getParent().getName() :
			parent.getParent().getParent().getName() + Path.SEPARATOR + parent.getName();
}
 
Example 17
Source Project: flink   File: FsCheckpointStorageTest.java    License: Apache License 2.0 4 votes vote down vote up
private void assertParent(Path parent, Path child) {
	Path path = new Path(parent, child.getName());
	assertEquals(path, child);
}