Java Code Examples for org.apache.flink.core.fs.FileSystem#getLocalFileSystem()

@Test
public void testRenameToNonEmptyTargetDir() throws IOException {
	final FileSystem fs = FileSystem.getLocalFileSystem();

	// a source folder with a file
	final File srcFolder = temporaryFolder.newFolder();
	final File srcFile = new File(srcFolder, "someFile.txt");
	assertTrue(srcFile.createNewFile());

	// a non-empty destination folder
	final File dstFolder = temporaryFolder.newFolder();
	final File dstFile  = new File(dstFolder, "target");
	assertTrue(dstFile.createNewFile());

	// this cannot succeed because the destination folder is not empty
	assertFalse(fs.rename(new Path(srcFolder.toURI()), new Path(dstFolder.toURI())));

	// retry after deleting the occupying target file
	assertTrue(dstFile.delete());
	assertTrue(fs.rename(new Path(srcFolder.toURI()), new Path(dstFolder.toURI())));
	assertTrue(new File(dstFolder, srcFile.getName()).exists());
}
 

@Test
public void testCompression() throws Exception {
	// given
	final Path outputPath = new Path(File.createTempFile("avro-output-file", "avro").getAbsolutePath());
	final AvroOutputFormat<User> outputFormat = new AvroOutputFormat<>(outputPath, User.class);
	outputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);

	final Path compressedOutputPath = new Path(File.createTempFile("avro-output-file", "compressed.avro").getAbsolutePath());
	final AvroOutputFormat<User> compressedOutputFormat = new AvroOutputFormat<>(compressedOutputPath, User.class);
	compressedOutputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
	compressedOutputFormat.setCodec(AvroOutputFormat.Codec.SNAPPY);

	// when
	output(outputFormat);
	output(compressedOutputFormat);

	// then
	assertTrue(fileSize(outputPath) > fileSize(compressedOutputPath));

	// cleanup
	FileSystem fs = FileSystem.getLocalFileSystem();
	fs.delete(outputPath, false);
	fs.delete(compressedOutputPath, false);
}
 

@Test
public void testGenericRecord() throws IOException {
	final Path outputPath = new Path(File.createTempFile("avro-output-file", "generic.avro").getAbsolutePath());
	final AvroOutputFormat<GenericRecord> outputFormat = new AvroOutputFormat<>(outputPath, GenericRecord.class);
	Schema schema = new Schema.Parser().parse("{\"type\":\"record\", \"name\":\"user\", \"fields\": [{\"name\":\"user_name\", \"type\":\"string\"}, {\"name\":\"favorite_number\", \"type\":\"int\"}, {\"name\":\"favorite_color\", \"type\":\"string\"}]}");
	outputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
	outputFormat.setSchema(schema);
	output(outputFormat, schema);

	GenericDatumReader<GenericRecord> reader = new GenericDatumReader<>(schema);
	DataFileReader<GenericRecord> dataFileReader = new DataFileReader<>(new File(outputPath.getPath()), reader);

	while (dataFileReader.hasNext()) {
		GenericRecord record = dataFileReader.next();
		assertEquals(record.get("user_name").toString(), "testUser");
		assertEquals(record.get("favorite_number"), 1);
		assertEquals(record.get("favorite_color").toString(), "blue");
	}

	//cleanup
	FileSystem fs = FileSystem.getLocalFileSystem();
	fs.delete(outputPath, false);
}
 

/**
 * Validates that even empty streams create a file and a file state handle.
 */
@Test
public void testEmptyState() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path folder = baseFolder();
	final String fileName = "myFileName";
	final Path filePath = new Path(folder, fileName);

	final FileStateHandle handle;
	try (FSDataOutputStream stream = createTestStream(fs, folder, fileName)) {
		handle = closeAndGetResult(stream);
	}

	// must have created a handle
	assertNotNull(handle);
	assertEquals(filePath, handle.getFilePath());

	// the pointer path should exist as a directory
	assertTrue(fs.exists(handle.getFilePath()));
	assertFalse(fs.getFileStatus(filePath).isDir());

	// the contents should be empty
	try (FSDataInputStream in = handle.openInputStream()) {
		assertEquals(-1, in.read());
	}
}
 

@Test
public void testDeletePathIfEmpty() throws IOException {
	final FileSystem localFs = FileSystem.getLocalFileSystem();

	final File dir = tmp.newFolder();
	assertTrue(dir.exists());

	final Path dirPath = new Path(dir.toURI());

	// deleting an empty directory should work
	assertTrue(FileUtils.deletePathIfEmpty(localFs, dirPath));

	// deleting a non existing directory should work
	assertTrue(FileUtils.deletePathIfEmpty(localFs, dirPath));

	// create a non-empty dir
	final File nonEmptyDir = tmp.newFolder();
	final Path nonEmptyDirPath = new Path(nonEmptyDir.toURI());
	new FileOutputStream(new File(nonEmptyDir, "filename")).close();
	assertFalse(FileUtils.deletePathIfEmpty(localFs, nonEmptyDirPath));
}
 

/**
 * Test that {@link FileUtils#deletePathIfEmpty(FileSystem, Path)} deletes the path if it is
 * empty. A path can only be empty if it is a directory which does not contain any
 * files/directories.
 */
@Test
public void testDeletePathIfEmpty() throws IOException {
	File file = temporaryFolder.newFile();
	File directory = temporaryFolder.newFolder();
	File directoryFile = new File(directory, UUID.randomUUID().toString());

	assertTrue(directoryFile.createNewFile());

	Path filePath = new Path(file.toURI());
	Path directoryPath = new Path(directory.toURI());
	Path directoryFilePath = new Path(directoryFile.toURI());

	FileSystem fs = FileSystem.getLocalFileSystem();

	// verify that the files have been created
	assertTrue(fs.exists(filePath));
	assertTrue(fs.exists(directoryFilePath));

	// delete the single file
	assertFalse(FileUtils.deletePathIfEmpty(fs, filePath));
	assertTrue(fs.exists(filePath));

	// try to delete the non-empty directory
	assertFalse(FileUtils.deletePathIfEmpty(fs, directoryPath));
	assertTrue(fs.exists(directoryPath));

	// delete the file contained in the directory
	assertTrue(fs.delete(directoryFilePath, false));

	// now the deletion should work
	assertTrue(FileUtils.deletePathIfEmpty(fs, directoryPath));
	assertFalse(fs.exists(directoryPath));
}
 

@Test
public void testPointerPathResolution() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path metadataFile = new Path(Path.fromLocalFile(tmp.newFolder()), AbstractFsCheckpointStorage.METADATA_FILE_NAME);

	final String basePointer = metadataFile.getParent().toString();

	final String pointer1 = metadataFile.toString();
	final String pointer2 = metadataFile.getParent().toString();
	final String pointer3 = metadataFile.getParent().toString() + '/';

	// create the storage for some random checkpoint directory
	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());

	final byte[] data = new byte[23686];
	new Random().nextBytes(data);
	try (FSDataOutputStream out = fs.create(metadataFile, WriteMode.NO_OVERWRITE)) {
		out.write(data);
	}

	CompletedCheckpointStorageLocation completed1 = storage.resolveCheckpoint(pointer1);
	CompletedCheckpointStorageLocation completed2 = storage.resolveCheckpoint(pointer2);
	CompletedCheckpointStorageLocation completed3 = storage.resolveCheckpoint(pointer3);

	assertEquals(basePointer, completed1.getExternalPointer());
	assertEquals(basePointer, completed2.getExternalPointer());
	assertEquals(basePointer, completed3.getExternalPointer());

	StreamStateHandle handle1 = completed1.getMetadataHandle();
	StreamStateHandle handle2 = completed2.getMetadataHandle();
	StreamStateHandle handle3 = completed3.getMetadataHandle();

	assertNotNull(handle1);
	assertNotNull(handle2);
	assertNotNull(handle3);

	validateContents(handle1, data);
	validateContents(handle2, data);
	validateContents(handle3, data);
}
 

@Test
public void testRenameNonExistingFile() throws IOException {
	final FileSystem fs = FileSystem.getLocalFileSystem();

	final File srcFile = new File(temporaryFolder.newFolder(), "someFile.txt");
	final File destFile  = new File(temporaryFolder.newFolder(), "target");

	final Path srcFilePath = new Path(srcFile.toURI());
	final Path destFilePath = new Path(destFile.toURI());

	// this cannot succeed because the source file does not exist
	assertFalse(fs.rename(srcFilePath, destFilePath));
}
 

/**
 * Test that {@link FileUtils#deletePathIfEmpty(FileSystem, Path)} deletes the path if it is
 * empty. A path can only be empty if it is a directory which does not contain any
 * files/directories.
 */
@Test
public void testDeletePathIfEmpty() throws IOException {
	File file = temporaryFolder.newFile();
	File directory = temporaryFolder.newFolder();
	File directoryFile = new File(directory, UUID.randomUUID().toString());

	assertTrue(directoryFile.createNewFile());

	Path filePath = new Path(file.toURI());
	Path directoryPath = new Path(directory.toURI());
	Path directoryFilePath = new Path(directoryFile.toURI());

	FileSystem fs = FileSystem.getLocalFileSystem();

	// verify that the files have been created
	assertTrue(fs.exists(filePath));
	assertTrue(fs.exists(directoryFilePath));

	// delete the single file
	assertFalse(FileUtils.deletePathIfEmpty(fs, filePath));
	assertTrue(fs.exists(filePath));

	// try to delete the non-empty directory
	assertFalse(FileUtils.deletePathIfEmpty(fs, directoryPath));
	assertTrue(fs.exists(directoryPath));

	// delete the file contained in the directory
	assertTrue(fs.delete(directoryFilePath, false));

	// now the deletion should work
	assertTrue(FileUtils.deletePathIfEmpty(fs, directoryPath));
	assertFalse(fs.exists(directoryPath));
}
 

@Test
public void testRenameFileWithNoAccess() throws IOException {
	final FileSystem fs = FileSystem.getLocalFileSystem();

	final File srcFile = temporaryFolder.newFile("someFile.txt");
	final File destFile = new File(temporaryFolder.newFolder(), "target");

	// we need to make the file non-modifiable so that the rename fails
	Assume.assumeTrue(srcFile.getParentFile().setWritable(false, false));
	Assume.assumeTrue(srcFile.setWritable(false, false));

	try {
		final Path srcFilePath = new Path(srcFile.toURI());
		final Path destFilePath = new Path(destFile.toURI());

		// this cannot succeed because the source folder has no permission to remove the file
		assertFalse(fs.rename(srcFilePath, destFilePath));
	}
	finally {
		// make sure we give permission back to ensure proper cleanup

		//noinspection ResultOfMethodCallIgnored
		srcFile.getParentFile().setWritable(true, false);
		//noinspection ResultOfMethodCallIgnored
		srcFile.setWritable(true, false);
	}
}
 

@Test
public void testPointerPathResolution() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path metadataFile = new Path(Path.fromLocalFile(tmp.newFolder()), AbstractFsCheckpointStorage.METADATA_FILE_NAME);

	final String basePointer = metadataFile.getParent().toString();

	final String pointer1 = metadataFile.toString();
	final String pointer2 = metadataFile.getParent().toString();
	final String pointer3 = metadataFile.getParent().toString() + '/';

	// create the storage for some random checkpoint directory
	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());

	final byte[] data = new byte[23686];
	new Random().nextBytes(data);
	try (FSDataOutputStream out = fs.create(metadataFile, WriteMode.NO_OVERWRITE)) {
		out.write(data);
	}

	CompletedCheckpointStorageLocation completed1 = storage.resolveCheckpoint(pointer1);
	CompletedCheckpointStorageLocation completed2 = storage.resolveCheckpoint(pointer2);
	CompletedCheckpointStorageLocation completed3 = storage.resolveCheckpoint(pointer3);

	assertEquals(basePointer, completed1.getExternalPointer());
	assertEquals(basePointer, completed2.getExternalPointer());
	assertEquals(basePointer, completed3.getExternalPointer());

	StreamStateHandle handle1 = completed1.getMetadataHandle();
	StreamStateHandle handle2 = completed2.getMetadataHandle();
	StreamStateHandle handle3 = completed3.getMetadataHandle();

	assertNotNull(handle1);
	assertNotNull(handle2);
	assertNotNull(handle3);

	validateContents(handle1, data);
	validateContents(handle2, data);
	validateContents(handle3, data);
}
 

@Test
public void testEmptyState() throws Exception {
	FsCheckpointStreamFactory.CheckpointStateOutputStream stream =
			new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
					Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 1024, 512);

	StreamStateHandle handle = stream.closeAndGetHandle();
	assertTrue(handle == null);
}
 

@Test
public void testGetPos() throws Exception {
	FsCheckpointStreamFactory.CheckpointStateOutputStream stream =
			new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
					Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 31, 17);

	for (int i = 0; i < 64; ++i) {
		Assert.assertEquals(i, stream.getPos());
		stream.write(0x42);
	}

	stream.closeAndGetHandle();

	// ----------------------------------------------------

	stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
			Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 31, 17);

	byte[] data = "testme!".getBytes(ConfigConstants.DEFAULT_CHARSET);

	for (int i = 0; i < 7; ++i) {
		Assert.assertEquals(i * (1 + data.length), stream.getPos());
		stream.write(0x42);
		stream.write(data);
	}

	stream.closeAndGetHandle();
}
 

@Test
public void testKind() {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	assertEquals(FileSystemKind.FILE_SYSTEM, fs.getKind());
}
 

private void runTest(int numBytes, int bufferSize, int threshold, boolean expectFile) throws Exception {
	FsCheckpointStreamFactory.CheckpointStateOutputStream stream =
		new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
				Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), bufferSize, threshold);

	Random rnd = new Random();
	byte[] original = new byte[numBytes];
	byte[] bytes = new byte[original.length];

	rnd.nextBytes(original);
	System.arraycopy(original, 0, bytes, 0, original.length);

	// the test writes a mixture of writing individual bytes and byte arrays
	int pos = 0;
	while (pos < bytes.length) {
		boolean single = rnd.nextBoolean();
		if (single) {
			stream.write(bytes[pos++]);
		}
		else {
			int num = rnd.nextBoolean() ?
				(bytes.length - pos) : rnd.nextInt(bytes.length - pos);
			stream.write(bytes, pos, num);
			pos += num;
		}
	}

	StreamStateHandle handle = stream.closeAndGetHandle();
	if (expectFile) {
		assertTrue(handle instanceof FileStateHandle);
	} else {
		assertTrue(handle instanceof ByteStreamStateHandle);
	}

	// make sure the writing process did not alter the original byte array
	assertArrayEquals(original, bytes);

	try (InputStream inStream = handle.openInputStream()) {
		byte[] validation = new byte[bytes.length];

		DataInputStream dataInputStream = new DataInputStream(inStream);
		dataInputStream.readFully(validation);

		assertArrayEquals(bytes, validation);
	}

	handle.discardState();
}
 

@Override
public FileSystem initializeFileSystem() {
	return FileSystem.getLocalFileSystem();
}
 

@Test
public void testKind() {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	assertEquals(FileSystemKind.FILE_SYSTEM, fs.getKind());
}
 

private void runTest(int numBytes, int bufferSize, int threshold, boolean expectFile) throws Exception {
	FsCheckpointStreamFactory.CheckpointStateOutputStream stream =
		new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
				Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), bufferSize, threshold);

	Random rnd = new Random();
	byte[] original = new byte[numBytes];
	byte[] bytes = new byte[original.length];

	rnd.nextBytes(original);
	System.arraycopy(original, 0, bytes, 0, original.length);

	// the test writes a mixture of writing individual bytes and byte arrays
	int pos = 0;
	while (pos < bytes.length) {
		boolean single = rnd.nextBoolean();
		if (single) {
			stream.write(bytes[pos++]);
		}
		else {
			int num = rnd.nextInt(Math.min(10, bytes.length - pos));
			stream.write(bytes, pos, num);
			pos += num;
		}
	}

	StreamStateHandle handle = stream.closeAndGetHandle();
	if (expectFile) {
		assertTrue(handle instanceof FileStateHandle);
	} else {
		assertTrue(handle instanceof ByteStreamStateHandle);
	}

	// make sure the writing process did not alter the original byte array
	assertArrayEquals(original, bytes);

	try (InputStream inStream = handle.openInputStream()) {
		byte[] validation = new byte[bytes.length];

		DataInputStream dataInputStream = new DataInputStream(inStream);
		dataInputStream.readFully(validation);

		assertArrayEquals(bytes, validation);
	}

	handle.discardState();
}
 

@Test
public void testMixedBelowAndAboveThreshold() throws Exception {
	final byte[] state1 = new byte[1274673];
	final byte[] state2 = new byte[1];
	final byte[] state3 = new byte[0];
	final byte[] state4 = new byte[177];

	final Random rnd = new Random();
	rnd.nextBytes(state1);
	rnd.nextBytes(state2);
	rnd.nextBytes(state3);
	rnd.nextBytes(state4);

	final File directory = tempDir.newFolder();
	final Path basePath = Path.fromLocalFile(directory);

	final Supplier<CheckpointStateOutputStream> factory = () ->
			new FsCheckpointStateOutputStream(basePath, FileSystem.getLocalFileSystem(), 1024, 15);

	CheckpointStateOutputStream stream1 = factory.get();
	CheckpointStateOutputStream stream2 = factory.get();
	CheckpointStateOutputStream stream3 = factory.get();

	stream1.write(state1);
	stream2.write(state2);
	stream3.write(state3);

	FileStateHandle handle1 = (FileStateHandle) stream1.closeAndGetHandle();
	ByteStreamStateHandle handle2 = (ByteStreamStateHandle) stream2.closeAndGetHandle();
	ByteStreamStateHandle handle3 = (ByteStreamStateHandle) stream3.closeAndGetHandle();

	// use with try-with-resources
	StreamStateHandle handle4;
	try (CheckpointStreamFactory.CheckpointStateOutputStream stream4 = factory.get()) {
		stream4.write(state4);
		handle4 = stream4.closeAndGetHandle();
	}

	// close before accessing handle
	CheckpointStreamFactory.CheckpointStateOutputStream stream5 = factory.get();
	stream5.write(state4);
	stream5.close();
	try {
		stream5.closeAndGetHandle();
		fail();
	} catch (IOException e) {
		// uh-huh
	}

	validateBytesInStream(handle1.openInputStream(), state1);
	handle1.discardState();
	assertFalse(isDirectoryEmpty(directory));
	ensureLocalFileDeleted(handle1.getFilePath());

	validateBytesInStream(handle2.openInputStream(), state2);
	handle2.discardState();
	assertFalse(isDirectoryEmpty(directory));

	// nothing was written to the stream, so it will return nothing
	assertNull(handle3);
	assertFalse(isDirectoryEmpty(directory));

	validateBytesInStream(handle4.openInputStream(), state4);
	handle4.discardState();
	assertTrue(isDirectoryEmpty(directory));
}
 

/**
 * Validates that multiple checkpoints from different jobs with the same checkpoint ID do not
 * interfere with each other.
 */
@Test
public void testPersistMultipleMetadataOnlyCheckpoints() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path checkpointDir = new Path(tmp.newFolder().toURI());

	final long checkpointId = 177;

	final CheckpointStorage storage1 = createCheckpointStorage(checkpointDir);
	storage1.initializeBaseLocations();
	final CheckpointStorage storage2 = createCheckpointStorage(checkpointDir);
	storage2.initializeBaseLocations();

	final CheckpointStorageLocation loc1 = storage1.initializeLocationForCheckpoint(checkpointId);
	final CheckpointStorageLocation loc2 = storage2.initializeLocationForCheckpoint(checkpointId);

	final byte[] data1 = {77, 66, 55, 99, 88};
	final byte[] data2 = {1, 3, 2, 5, 4};

	final CompletedCheckpointStorageLocation completedLocation1;
	try (CheckpointMetadataOutputStream out = loc1.createMetadataOutputStream()) {
		out.write(data1);
		completedLocation1 = out.closeAndFinalizeCheckpoint();
	}
	final String result1 = completedLocation1.getExternalPointer();

	final CompletedCheckpointStorageLocation completedLocation2;
	try (CheckpointMetadataOutputStream out = loc2.createMetadataOutputStream()) {
		out.write(data2);
		completedLocation2 = out.closeAndFinalizeCheckpoint();
	}
	final String result2 = completedLocation2.getExternalPointer();

	// check that this went to a file, but in a nested directory structure

	// one directory per storage
	FileStatus[] files = fs.listStatus(checkpointDir);
	assertEquals(2, files.length);

	// in each per-storage directory, one for the checkpoint
	FileStatus[] job1Files = fs.listStatus(files[0].getPath());
	FileStatus[] job2Files = fs.listStatus(files[1].getPath());
	assertTrue(job1Files.length >= 1);
	assertTrue(job2Files.length >= 1);

	assertTrue(fs.exists(new Path(result1, AbstractFsCheckpointStorage.METADATA_FILE_NAME)));
	assertTrue(fs.exists(new Path(result2, AbstractFsCheckpointStorage.METADATA_FILE_NAME)));

	// check that both storages can resolve each others contents
	validateContents(storage1.resolveCheckpoint(result1).getMetadataHandle(), data1);
	validateContents(storage1.resolveCheckpoint(result2).getMetadataHandle(), data2);
	validateContents(storage2.resolveCheckpoint(result1).getMetadataHandle(), data1);
	validateContents(storage2.resolveCheckpoint(result2).getMetadataHandle(), data2);
}