Java Code Examples for org.apache.flink.core.fs.FileSystem#create()

private static void internalCopyFile(Path sourcePath, Path targetPath, boolean executable, FileSystem sFS, FileSystem tFS) throws IOException {
	try (FSDataOutputStream lfsOutput = tFS.create(targetPath, FileSystem.WriteMode.NO_OVERWRITE); FSDataInputStream fsInput = sFS.open(sourcePath)) {
		IOUtils.copyBytes(fsInput, lfsOutput);
		//noinspection ResultOfMethodCallIgnored
		new File(targetPath.toString()).setExecutable(executable);
	}
}
 

private static Path writeFile(FileSystem dfs, Path rootDir, String fileName) throws IOException {
	Path file = new Path(rootDir, fileName);
	try (
		DataOutputStream outStream = new DataOutputStream(dfs.create(file,
			FileSystem.WriteMode.OVERWRITE))) {
		outStream.writeUTF(testFileContent);
	}
	return file;
}
 

public static Path expandDirectory(Path file, Path targetDirectory) throws IOException {
	FileSystem sourceFs = file.getFileSystem();
	FileSystem targetFs = targetDirectory.getFileSystem();
	Path rootDir = null;
	try (ZipInputStream zis = new ZipInputStream(sourceFs.open(file))) {
		ZipEntry entry;
		while ((entry = zis.getNextEntry()) != null) {
			Path relativePath = new Path(entry.getName());
			if (rootDir == null) {
				// the first entry contains the name of the original directory that was zipped
				rootDir = relativePath;
			}

			Path newFile = new Path(targetDirectory, relativePath);
			if (entry.isDirectory()) {
				targetFs.mkdirs(newFile);
			} else {
				try (FSDataOutputStream fileStream = targetFs.create(newFile, FileSystem.WriteMode.NO_OVERWRITE)) {
					// do not close the streams here as it prevents access to further zip entries
					IOUtils.copyBytes(zis, fileStream, false);
				}
			}
			zis.closeEntry();
		}
	}
	return new Path(targetDirectory, rootDir);
}
 

public static Path compressDirectory(Path directory, Path target) throws IOException {
	FileSystem sourceFs = directory.getFileSystem();
	FileSystem targetFs = target.getFileSystem();

	try (ZipOutputStream out = new ZipOutputStream(targetFs.create(target, FileSystem.WriteMode.NO_OVERWRITE))) {
		addToZip(directory, sourceFs, directory.getParent(), out);
	}
	return target;
}
 

@Test
public void testPointerPathResolution() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path metadataFile = new Path(Path.fromLocalFile(tmp.newFolder()), AbstractFsCheckpointStorage.METADATA_FILE_NAME);

	final String basePointer = metadataFile.getParent().toString();

	final String pointer1 = metadataFile.toString();
	final String pointer2 = metadataFile.getParent().toString();
	final String pointer3 = metadataFile.getParent().toString() + '/';

	// create the storage for some random checkpoint directory
	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());

	final byte[] data = new byte[23686];
	new Random().nextBytes(data);
	try (FSDataOutputStream out = fs.create(metadataFile, WriteMode.NO_OVERWRITE)) {
		out.write(data);
	}

	CompletedCheckpointStorageLocation completed1 = storage.resolveCheckpoint(pointer1);
	CompletedCheckpointStorageLocation completed2 = storage.resolveCheckpoint(pointer2);
	CompletedCheckpointStorageLocation completed3 = storage.resolveCheckpoint(pointer3);

	assertEquals(basePointer, completed1.getExternalPointer());
	assertEquals(basePointer, completed2.getExternalPointer());
	assertEquals(basePointer, completed3.getExternalPointer());

	StreamStateHandle handle1 = completed1.getMetadataHandle();
	StreamStateHandle handle2 = completed2.getMetadataHandle();
	StreamStateHandle handle3 = completed3.getMetadataHandle();

	assertNotNull(handle1);
	assertNotNull(handle2);
	assertNotNull(handle3);

	validateContents(handle1, data);
	validateContents(handle2, data);
	validateContents(handle3, data);
}
 

/**
 * Copies the file from a single state handle to the given path.
 */
private void downloadDataForStateHandle(
	Path restoreFilePath,
	StreamStateHandle remoteFileHandle,
	CloseableRegistry closeableRegistry) throws IOException {

	FSDataInputStream inputStream = null;
	FSDataOutputStream outputStream = null;

	try {
		FileSystem restoreFileSystem = restoreFilePath.getFileSystem();
		inputStream = remoteFileHandle.openInputStream();
		closeableRegistry.registerCloseable(inputStream);

		outputStream = restoreFileSystem.create(restoreFilePath, FileSystem.WriteMode.OVERWRITE);
		closeableRegistry.registerCloseable(outputStream);

		byte[] buffer = new byte[8 * 1024];
		while (true) {
			int numBytes = inputStream.read(buffer);
			if (numBytes == -1) {
				break;
			}

			outputStream.write(buffer, 0, numBytes);
		}
	} finally {
		if (closeableRegistry.unregisterCloseable(inputStream)) {
			inputStream.close();
		}

		if (closeableRegistry.unregisterCloseable(outputStream)) {
			outputStream.close();
		}
	}
}
 

public FsCheckpointMetadataOutputStream(
		FileSystem fileSystem,
		Path metadataFilePath,
		Path exclusiveCheckpointDir) throws IOException {

	this.fileSystem = checkNotNull(fileSystem);
	this.metadataFilePath = checkNotNull(metadataFilePath);
	this.exclusiveCheckpointDir = checkNotNull(exclusiveCheckpointDir);

	this.out = fileSystem.create(metadataFilePath, WriteMode.NO_OVERWRITE);
}
 

/**
 * Copies the file from a single state handle to the given path.
 */
private void downloadDataForStateHandle(
	Path restoreFilePath,
	StreamStateHandle remoteFileHandle,
	CloseableRegistry closeableRegistry) throws IOException {

	FSDataInputStream inputStream = null;
	FSDataOutputStream outputStream = null;

	try {
		FileSystem restoreFileSystem = restoreFilePath.getFileSystem();
		inputStream = remoteFileHandle.openInputStream();
		closeableRegistry.registerCloseable(inputStream);

		outputStream = restoreFileSystem.create(restoreFilePath, FileSystem.WriteMode.OVERWRITE);
		closeableRegistry.registerCloseable(outputStream);

		byte[] buffer = new byte[8 * 1024];
		while (true) {
			int numBytes = inputStream.read(buffer);
			if (numBytes == -1) {
				break;
			}

			outputStream.write(buffer, 0, numBytes);
		}
	} finally {
		if (closeableRegistry.unregisterCloseable(inputStream)) {
			inputStream.close();
		}

		if (closeableRegistry.unregisterCloseable(outputStream)) {
			outputStream.close();
		}
	}
}
 

/**
 * Test that {@link FileUtils#deletePathIfEmpty(FileSystem, Path)} deletes the path if it is
 * empty. A path can only be empty if it is a directory which does not contain any
 * files/directories.
 */
@Test
public void testDeletePathIfEmpty() throws IOException {
	final Path basePath = new Path(hdfsURI);
	final Path directory = new Path(basePath, UUID.randomUUID().toString());
	final Path directoryFile = new Path(directory, UUID.randomUUID().toString());
	final Path singleFile = new Path(basePath, UUID.randomUUID().toString());

	FileSystem fs = basePath.getFileSystem();

	fs.mkdirs(directory);

	byte[] data = "HDFSTest#testDeletePathIfEmpty".getBytes(ConfigConstants.DEFAULT_CHARSET);

	for (Path file: Arrays.asList(singleFile, directoryFile)) {
		org.apache.flink.core.fs.FSDataOutputStream outputStream = fs.create(file, FileSystem.WriteMode.OVERWRITE);
		outputStream.write(data);
		outputStream.close();
	}

	// verify that the files have been created
	assertTrue(fs.exists(singleFile));
	assertTrue(fs.exists(directoryFile));

	// delete the single file
	assertFalse(FileUtils.deletePathIfEmpty(fs, singleFile));
	assertTrue(fs.exists(singleFile));

	// try to delete the non-empty directory
	assertFalse(FileUtils.deletePathIfEmpty(fs, directory));
	assertTrue(fs.exists(directory));

	// delete the file contained in the directory
	assertTrue(fs.delete(directoryFile, false));

	// now the deletion should work
	assertTrue(FileUtils.deletePathIfEmpty(fs, directory));
	assertFalse(fs.exists(directory));
}
 

private static void internalCopyFile(Path sourcePath, Path targetPath, boolean executable, FileSystem sFS, FileSystem tFS) throws IOException {
	try (FSDataOutputStream lfsOutput = tFS.create(targetPath, FileSystem.WriteMode.NO_OVERWRITE); FSDataInputStream fsInput = sFS.open(sourcePath)) {
		IOUtils.copyBytes(fsInput, lfsOutput);
		//noinspection ResultOfMethodCallIgnored
		new File(targetPath.toString()).setExecutable(executable);
	}
}
 

private static void unzipPythonLibrary(Path targetDir) throws IOException {
	FileSystem targetFs = targetDir.getFileSystem();
	ClassLoader classLoader = PythonPlanBinder.class.getClassLoader();
	try (ZipInputStream zis = new ZipInputStream(classLoader.getResourceAsStream("python-source.zip"))) {
		ZipEntry entry = zis.getNextEntry();
		while (entry != null) {
			String fileName = entry.getName();
			Path newFile = new Path(targetDir, fileName);
			if (entry.isDirectory()) {
				targetFs.mkdirs(newFile);
			} else {
				try {
					LOG.debug("Unzipping to {}.", newFile);
					FSDataOutputStream fsDataOutputStream = targetFs.create(newFile, FileSystem.WriteMode.NO_OVERWRITE);
					IOUtils.copyBytes(zis, fsDataOutputStream, false);
				} catch (Exception e) {
					zis.closeEntry();
					throw new IOException("Failed to unzip flink python library.", e);
				}
			}

			zis.closeEntry();
			entry = zis.getNextEntry();
		}
		zis.closeEntry();
	}
}
 

public static Path expandDirectory(Path file, Path targetDirectory) throws IOException {
	FileSystem sourceFs = file.getFileSystem();
	FileSystem targetFs = targetDirectory.getFileSystem();
	Path rootDir = null;
	try (ZipInputStream zis = new ZipInputStream(sourceFs.open(file))) {
		ZipEntry entry;
		while ((entry = zis.getNextEntry()) != null) {
			Path relativePath = new Path(entry.getName());
			if (rootDir == null) {
				// the first entry contains the name of the original directory that was zipped
				rootDir = relativePath;
			}

			Path newFile = new Path(targetDirectory, relativePath);
			if (entry.isDirectory()) {
				targetFs.mkdirs(newFile);
			} else {
				try (FSDataOutputStream fileStream = targetFs.create(newFile, FileSystem.WriteMode.NO_OVERWRITE)) {
					// do not close the streams here as it prevents access to further zip entries
					IOUtils.copyBytes(zis, fileStream, false);
				}
			}
			zis.closeEntry();
		}
	}
	return new Path(targetDirectory, rootDir);
}
 

public FsCheckpointMetadataOutputStream(
		FileSystem fileSystem,
		Path metadataFilePath,
		Path exclusiveCheckpointDir) throws IOException {

	this.fileSystem = checkNotNull(fileSystem);
	this.metadataFilePath = checkNotNull(metadataFilePath);
	this.exclusiveCheckpointDir = checkNotNull(exclusiveCheckpointDir);

	this.out = fileSystem.create(metadataFilePath, WriteMode.NO_OVERWRITE);
}
 

@Test
public void testDirectoryListing() throws Exception {
	final Configuration conf = createConfiguration();

	FileSystem.initialize(conf);

	final Path directory = new Path("swift://" + CONTAINER + '.' + SERVICENAME + '/' + TEST_DATA_DIR + "/testdir/");
	final FileSystem fs = directory.getFileSystem();

	// directory must not yet exist
	assertFalse(fs.exists(directory));

	try {
		// create directory
		assertTrue(fs.mkdirs(directory));

		// seems the file system does not assume existence of empty directories
		assertTrue(fs.exists(directory));

		// directory empty
		assertEquals(0, fs.listStatus(directory).length);

		// create some files
		final int numFiles = 3;
		for (int i = 0; i < numFiles; i++) {
			Path file = new Path(directory, "/file-" + i);
			try (FSDataOutputStream out = fs.create(file, FileSystem.WriteMode.NO_OVERWRITE);
				OutputStreamWriter writer = new OutputStreamWriter(out, StandardCharsets.UTF_8)) {
				writer.write("hello-" + i + "\n");
			}
		}

		FileStatus[] files = fs.listStatus(directory);
		assertNotNull(files);
		assertEquals(3, files.length);

		for (FileStatus status : files) {
			assertFalse(status.isDir());
		}

		// now that there are files, the directory must exist
		assertTrue(fs.exists(directory));
	}
	finally {
		// clean up
		fs.delete(directory, true);
	}

	// now directory must be gone
	assertFalse(fs.exists(directory));
}
 

@Override
public void open(int taskNumber, int numTasks) throws IOException {
	if (taskNumber < 0 || numTasks < 1) {
		throw new IllegalArgumentException("TaskNumber: " + taskNumber + ", numTasks: " + numTasks);
	}
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Opening stream for output (" + (taskNumber+1) + "/" + numTasks + "). WriteMode=" + writeMode +
				", OutputDirectoryMode=" + outputDirectoryMode);
	}
	
	Path p = this.outputFilePath;
	if (p == null) {
		throw new IOException("The file path is null.");
	}
	
	final FileSystem fs = p.getFileSystem();

	// if this is a local file system, we need to initialize the local output directory here
	if (!fs.isDistributedFS()) {
		
		if (numTasks == 1 && outputDirectoryMode == OutputDirectoryMode.PARONLY) {
			// output should go to a single file
			
			// prepare local output path. checks for write mode and removes existing files in case of OVERWRITE mode
			if(!fs.initOutPathLocalFS(p, writeMode, false)) {
				// output preparation failed! Cancel task.
				throw new IOException("Output path '" + p.toString() + "' could not be initialized. Canceling task...");
			}
		}
		else {
			// numTasks > 1 || outDirMode == OutputDirectoryMode.ALWAYS
			
			if(!fs.initOutPathLocalFS(p, writeMode, true)) {
				// output preparation failed! Cancel task.
				throw new IOException("Output directory '" + p.toString() + "' could not be created. Canceling task...");
			}
		}
	}



	// Suffix the path with the parallel instance index, if needed
	this.actualFilePath = (numTasks > 1 || outputDirectoryMode == OutputDirectoryMode.ALWAYS) ? p.suffix("/" + getDirectoryFileName(taskNumber)) : p;

	// create output file
	this.stream = fs.create(this.actualFilePath, writeMode);
	
	// at this point, the file creation must have succeeded, or an exception has been thrown
	this.fileCreated = true;
}
 

public FileBasedStateOutputStream(FileSystem fileSystem, Path path) throws IOException {
	this.fileSystem = checkNotNull(fileSystem);
	this.path = checkNotNull(path);

	this.out = fileSystem.create(path, WriteMode.NO_OVERWRITE);
}
 

public FileBasedStateOutputStream(FileSystem fileSystem, Path path) throws IOException {
	this.fileSystem = checkNotNull(fileSystem);
	this.path = checkNotNull(path);

	this.out = fileSystem.create(path, WriteMode.NO_OVERWRITE);
}
 

@Override
public void open(int taskNumber, int numTasks) throws IOException {
	if (taskNumber < 0 || numTasks < 1) {
		throw new IllegalArgumentException("TaskNumber: " + taskNumber + ", numTasks: " + numTasks);
	}
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Opening stream for output (" + (taskNumber+1) + "/" + numTasks + "). WriteMode=" + writeMode +
				", OutputDirectoryMode=" + outputDirectoryMode);
	}
	
	Path p = this.outputFilePath;
	if (p == null) {
		throw new IOException("The file path is null.");
	}
	
	final FileSystem fs = p.getFileSystem();

	// if this is a local file system, we need to initialize the local output directory here
	if (!fs.isDistributedFS()) {
		
		if (numTasks == 1 && outputDirectoryMode == OutputDirectoryMode.PARONLY) {
			// output should go to a single file
			
			// prepare local output path. checks for write mode and removes existing files in case of OVERWRITE mode
			if(!fs.initOutPathLocalFS(p, writeMode, false)) {
				// output preparation failed! Cancel task.
				throw new IOException("Output path '" + p.toString() + "' could not be initialized. Canceling task...");
			}
		}
		else {
			// numTasks > 1 || outDirMode == OutputDirectoryMode.ALWAYS
			
			if(!fs.initOutPathLocalFS(p, writeMode, true)) {
				// output preparation failed! Cancel task.
				throw new IOException("Output directory '" + p.toString() + "' could not be created. Canceling task...");
			}
		}
	}



	// Suffix the path with the parallel instance index, if needed
	this.actualFilePath = (numTasks > 1 || outputDirectoryMode == OutputDirectoryMode.ALWAYS) ? p.suffix("/" + getDirectoryFileName(taskNumber)) : p;

	// create output file
	this.stream = fs.create(this.actualFilePath, writeMode);
	
	// at this point, the file creation must have succeeded, or an exception has been thrown
	this.fileCreated = true;
}
 

/**
 * Generates a random content file.
 *
 * @param outputFile the path of the output file
 * @param length the size of content to generate
 *
 * @return MD5 of the output file
 *
 * @throws IOException
 * @throws NoSuchAlgorithmException
 */
private static String generateTestFile(String outputFile, int length) throws IOException, NoSuchAlgorithmException {
	Path outputFilePath = new Path(outputFile);

	final FileSystem fileSystem = outputFilePath.getFileSystem();
	try (final FSDataOutputStream fsDataOutputStream = fileSystem.create(outputFilePath, FileSystem.WriteMode.OVERWRITE)) {
		return writeRandomContent(fsDataOutputStream, length);
	}
}
 

/**
 * Generates a random content file.
 *
 * @param outputFile the path of the output file
 * @param length the size of content to generate
 *
 * @return MD5 of the output file
 *
 * @throws IOException
 * @throws NoSuchAlgorithmException
 */
private static String generateTestFile(String outputFile, int length) throws IOException, NoSuchAlgorithmException {
	Path outputFilePath = new Path(outputFile);

	final FileSystem fileSystem = outputFilePath.getFileSystem();
	try (final FSDataOutputStream fsDataOutputStream = fileSystem.create(outputFilePath, FileSystem.WriteMode.OVERWRITE)) {
		return writeRandomContent(fsDataOutputStream, length);
	}
}