org.apache.flink.core.fs.FileSystem.WriteMode Java Examples

private void createStream() throws IOException {
	Exception latestException = null;
	for (int attempt = 0; attempt < 10; attempt++) {
		try {
			OutputStreamAndPath streamAndPath = EntropyInjector.createEntropyAware(
					fs, createStatePath(), WriteMode.NO_OVERWRITE);
			this.outStream = streamAndPath.stream();
			this.statePath = streamAndPath.path();
			return;
		}
		catch (Exception e) {
			latestException = e;
		}
	}

	throw new IOException("Could not open output stream for state backend", latestException);
}
 

/**
 * Writes a DataStream to the file specified by the path parameter. The
 * writing is performed periodically every millis milliseconds.
 *
 * <p>For every field of an element of the DataStream the result of {@link Object#toString()}
 * is written. This method can only be used on data streams of tuples.
 *
 * @param path
 *            the path pointing to the location the text file is written to
 * @param writeMode
 *            Controls the behavior for existing files. Options are
 *            NO_OVERWRITE and OVERWRITE.
 * @param rowDelimiter
 *            the delimiter for two rows
 * @param fieldDelimiter
 *            the delimiter for two fields
 *
 * @return the closed DataStream
 *
 * @deprecated Please use the {@link org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink} explicitly using the
 * {@link #addSink(SinkFunction)} method.
 */
@SuppressWarnings("unchecked")
@Deprecated
@PublicEvolving
public <X extends Tuple> DataStreamSink<T> writeAsCsv(
		String path,
		WriteMode writeMode,
		String rowDelimiter,
		String fieldDelimiter) {
	Preconditions.checkArgument(
		getType().isTupleType(),
		"The writeAsCsv() method can only be used on data streams of tuples.");

	CsvOutputFormat<X> of = new CsvOutputFormat<>(
		new Path(path),
		rowDelimiter,
		fieldDelimiter);

	if (writeMode != null) {
		of.setWriteMode(writeMode);
	}

	return writeUsingOutputFormat((OutputFormat<T>) of);
}
 

/**
 * Writes a DataStream to the file specified by the path parameter. The
 * writing is performed periodically every millis milliseconds.
 *
 * <p>For every field of an element of the DataStream the result of {@link Object#toString()}
 * is written. This method can only be used on data streams of tuples.
 *
 * @param path
 *            the path pointing to the location the text file is written to
 * @param writeMode
 *            Controls the behavior for existing files. Options are
 *            NO_OVERWRITE and OVERWRITE.
 * @param rowDelimiter
 *            the delimiter for two rows
 * @param fieldDelimiter
 *            the delimiter for two fields
 *
 * @return the closed DataStream
 */
@SuppressWarnings("unchecked")
@PublicEvolving
public <X extends Tuple> DataStreamSink<T> writeAsCsv(
		String path,
		WriteMode writeMode,
		String rowDelimiter,
		String fieldDelimiter) {
	Preconditions.checkArgument(
		getType().isTupleType(),
		"The writeAsCsv() method can only be used on data streams of tuples.");

	CsvOutputFormat<X> of = new CsvOutputFormat<>(
		new Path(path),
		rowDelimiter,
		fieldDelimiter);

	if (writeMode != null) {
		of.setWriteMode(writeMode);
	}

	return writeUsingOutputFormat((OutputFormat<T>) of);
}
 

private void createStream() throws IOException {
	Exception latestException = null;
	for (int attempt = 0; attempt < 10; attempt++) {
		try {
			OutputStreamAndPath streamAndPath = EntropyInjector.createEntropyAware(
					fs, createStatePath(), WriteMode.NO_OVERWRITE);
			this.outStream = streamAndPath.stream();
			this.statePath = streamAndPath.path();
			return;
		}
		catch (Exception e) {
			latestException = e;
		}
	}

	throw new IOException("Could not open output stream for state backend", latestException);
}
 

private void createStream() throws IOException {
	Exception latestException = null;
	for (int attempt = 0; attempt < 10; attempt++) {
		try {
			OutputStreamAndPath streamAndPath = EntropyInjector.createEntropyAware(
					fs, createStatePath(), WriteMode.NO_OVERWRITE);
			this.outStream = streamAndPath.stream();
			this.statePath = streamAndPath.path();
			return;
		}
		catch (Exception e) {
			latestException = e;
		}
	}

	throw new IOException("Could not open output stream for state backend", latestException);
}
 

@Override
public void go() throws Exception {

	try (FSDataOutputStream stream = fs.create(path, WriteMode.OVERWRITE)) {
		assertTrue(fs.getNumberOfOpenOutputStreams() <= maxConcurrentOutputStreams);
		assertTrue(fs.getTotalNumberOfOpenStreams() <= maxConcurrentStreamsTotal);

		final Random rnd = new Random();
		final byte[] data = new byte[rnd.nextInt(10000) + 1];
		rnd.nextBytes(data);
		stream.write(data);

		waitTillWokenUp();

		// try to write one more thing, which might/should fail with an I/O exception
		stream.write(rnd.nextInt());
	}
}
 

@Override
public void go() throws Exception {

	try (FSDataOutputStream stream = fs.create(path, WriteMode.OVERWRITE)) {
		assertTrue(fs.getNumberOfOpenOutputStreams() <= maxConcurrentOutputStreams);
		assertTrue(fs.getTotalNumberOfOpenStreams() <= maxConcurrentStreamsTotal);

		final Random rnd = new Random();
		final byte[] data = new byte[rnd.nextInt(10000) + 1];
		rnd.nextBytes(data);
		stream.write(data);

		waitTillWokenUp();

		// try to write one more thing, which might/should fail with an I/O exception
		stream.write(rnd.nextInt());
	}
}
 

@Override
public void go() throws Exception {

	try (FSDataOutputStream stream = fs.create(path, WriteMode.OVERWRITE)) {
		assertTrue(fs.getNumberOfOpenOutputStreams() <= maxConcurrentOutputStreams);
		assertTrue(fs.getTotalNumberOfOpenStreams() <= maxConcurrentStreamsTotal);

		final Random rnd = new Random();
		final byte[] data = new byte[rnd.nextInt(10000) + 1];
		rnd.nextBytes(data);
		stream.write(data);

		waitTillWokenUp();

		// try to write one more thing, which might/should fail with an I/O exception
		stream.write(rnd.nextInt());
	}
}
 

@Test
public void testClassLoaderFixingFs() throws Exception {
	final String entropyKey = "__ekey__";
	final String entropyValue = "abc";

	final File folder = TMP_FOLDER.newFolder();

	final Path path = new Path(Path.fromLocalFile(folder), entropyKey + "/path/");
	final Path pathWithEntropy = new Path(Path.fromLocalFile(folder), entropyValue + "/path/");

	PluginFileSystemFactory pluginFsFactory = PluginFileSystemFactory.of(
			new TestFileSystemFactory(entropyKey, entropyValue));
	FileSystem testFs = pluginFsFactory.create(URI.create("test"));

	FileSystemSafetyNet.initializeSafetyNetForThread();
	FileSystem fs = FileSystemSafetyNet.wrapWithSafetyNetWhenActivated(testFs);
	try  {
		OutputStreamAndPath streamAndPath = EntropyInjector.createEntropyAware(
				fs, path, WriteMode.NO_OVERWRITE);

		assertEquals(pathWithEntropy, streamAndPath.path());
	}
	finally {
		FileSystemSafetyNet.closeSafetyNetAndGuardedResourcesForThread();
	}
}
 

/**
 * Writes a DataStream to the file specified by the path parameter. The
 * writing is performed periodically every millis milliseconds.
 *
 * <p>For every field of an element of the DataStream the result of {@link Object#toString()}
 * is written. This method can only be used on data streams of tuples.
 *
 * @param path
 *            the path pointing to the location the text file is written to
 * @param writeMode
 *            Controls the behavior for existing files. Options are
 *            NO_OVERWRITE and OVERWRITE.
 * @param rowDelimiter
 *            the delimiter for two rows
 * @param fieldDelimiter
 *            the delimiter for two fields
 *
 * @return the closed DataStream
 */
@SuppressWarnings("unchecked")
@PublicEvolving
public <X extends Tuple> DataStreamSink<T> writeAsCsv(
		String path,
		WriteMode writeMode,
		String rowDelimiter,
		String fieldDelimiter) {
	Preconditions.checkArgument(
		getType().isTupleType(),
		"The writeAsCsv() method can only be used on data streams of tuples.");

	CsvOutputFormat<X> of = new CsvOutputFormat<>(
		new Path(path),
		rowDelimiter,
		fieldDelimiter);

	if (writeMode != null) {
		of.setWriteMode(writeMode);
	}

	return writeUsingOutputFormat((OutputFormat<T>) of);
}
 

public FsCheckpointMetadataOutputStream(
		FileSystem fileSystem,
		Path metadataFilePath,
		Path exclusiveCheckpointDir) throws IOException {

	this.fileSystem = checkNotNull(fileSystem);
	this.metadataFilePath = checkNotNull(metadataFilePath);
	this.exclusiveCheckpointDir = checkNotNull(exclusiveCheckpointDir);

	this.out = fileSystem.create(metadataFilePath, WriteMode.NO_OVERWRITE);
}
 

@Test
public void testOpenTimeoutOutputStreams() throws Exception {
	final long openTimeout = 50L;
	final int maxConcurrentOpen = 2;

	final LimitedConnectionsFileSystem limitedFs = new LimitedConnectionsFileSystem(
			LocalFileSystem.getSharedInstance(),
			maxConcurrentOpen, // limited total
			openTimeout,       // small opening timeout
			0L);               // infinite inactivity timeout

	// create the threads that block all streams
	final BlockingWriterThread[] threads = new BlockingWriterThread[maxConcurrentOpen];
	for (int i = 0; i < maxConcurrentOpen; i++) {
		Path path = new Path(tempFolder.newFile().toURI());
		threads[i] = new BlockingWriterThread(limitedFs, path, Integer.MAX_VALUE, maxConcurrentOpen);
		threads[i].start();
	}

	// wait until all are open
	while (limitedFs.getTotalNumberOfOpenStreams() < maxConcurrentOpen) {
		Thread.sleep(1);
	}

	// try to open another thread
	try {
		limitedFs.create(new Path(tempFolder.newFile().toURI()), WriteMode.OVERWRITE);
		fail("this should have timed out");
	}
	catch (IOException e) {
		// expected
	}

	// clean shutdown
	for (BlockingWriterThread t : threads) {
		t.wakeup();
		t.sync();
	}
}
 

@SuppressWarnings("unchecked")
private <X extends Tuple> DataSink<T> internalWriteAsCsv(Path filePath, String rowDelimiter, String fieldDelimiter, WriteMode wm) {
	Preconditions.checkArgument(getType().isTupleType(), "The writeAsCsv() method can only be used on data sets of tuples.");
	CsvOutputFormat<X> of = new CsvOutputFormat<>(filePath, rowDelimiter, fieldDelimiter);
	if (wm != null) {
		of.setWriteMode(wm);
	}
	return output((OutputFormat<T>) of);
}
 

@Test
public void testCreateEntropyAwarePlainFs() throws Exception {
	File folder = TMP_FOLDER.newFolder();
	Path path = new Path(Path.fromLocalFile(folder), "_entropy_/file");

	OutputStreamAndPath out = EntropyInjector.createEntropyAware(
			LocalFileSystem.getSharedInstance(), path, WriteMode.NO_OVERWRITE);

	out.stream().close();

	assertEquals(path, out.path());
	assertTrue(new File (new File(folder, "_entropy_"), "file").exists());
}
 

/**
 * Initialization of the distributed file system if it is used.
 *
 * @param parallelism The task parallelism.
 */
@Override
public void initializeGlobal(int parallelism) throws IOException {
	final Path path = getOutputFilePath();
	final FileSystem fs = path.getFileSystem();
	
	// only distributed file systems can be initialized at start-up time.
	if (fs.isDistributedFS()) {
		
		final WriteMode writeMode = getWriteMode();
		final OutputDirectoryMode outDirMode = getOutputDirectoryMode();

		if (parallelism == 1 && outDirMode == OutputDirectoryMode.PARONLY) {
			// output is not written in parallel and should be written to a single file.
			// prepare distributed output path
			if(!fs.initOutPathDistFS(path, writeMode, false)) {
				// output preparation failed! Cancel task.
				throw new IOException("Output path could not be initialized.");
			}

		} else {
			// output should be written to a directory

			// only distributed file systems can be initialized at start-up time.
			if(!fs.initOutPathDistFS(path, writeMode, true)) {
				throw new IOException("Output directory could not be created.");
			}
		}
	}
}
 

public void registerFileOutputTask(
	FileOutputFormat<Record> outputFormat,
	String outPath,
	Configuration formatParams) {

	outputFormat.setOutputFilePath(new Path(outPath));
	outputFormat.setWriteMode(WriteMode.OVERWRITE);

	OperatorID operatorID = new OperatorID();
	new InputOutputFormatContainer(Thread.currentThread().getContextClassLoader())
		.addOutputFormat(operatorID, outputFormat)
		.addParameters(operatorID, formatParams)
		.write(new TaskConfig(this.mockEnv.getTaskConfiguration()));
}
 

@Override
public void go() throws Exception {
	fs.initOutPathLocalFS(path.getParent(), WriteMode.OVERWRITE, true);
	try (FSDataOutputStream out = fs.create(path, WriteMode.OVERWRITE)) {
		out.write(11);
	}
}
 

public void setWriteMode(WriteMode mode) {
	if (mode == null) {
		throw new NullPointerException();
	}
	
	this.writeMode = mode;
}
 

@Override
public void go() throws Exception {
	fs.initOutPathLocalFS(path.getParent(), WriteMode.OVERWRITE, true);
	try (FSDataOutputStream out = fs.create(path, WriteMode.OVERWRITE)) {
		out.write(11);
	}
}
 

/**
 * Handles entropy injection across regular and entropy-aware file systems.
 *
 * <p>If the given file system is entropy-aware (a implements {@link EntropyInjectingFileSystem}),
 * then this method replaces the entropy marker in the path with random characters.
 * The entropy marker is defined by {@link EntropyInjectingFileSystem#getEntropyInjectionKey()}.
 *
 * <p>If the given file system does not implement {@code EntropyInjectingFileSystem},
 * then this method delegates to {@link FileSystem#create(Path, WriteMode)} and
 * returns the same path in the resulting {@code OutputStreamAndPath}.
 */
public static OutputStreamAndPath createEntropyAware(
		FileSystem fs,
		Path path,
		WriteMode writeMode) throws IOException {

	// check and possibly inject entropy into the path
	final EntropyInjectingFileSystem efs = getEntropyFs(fs);
	final Path processedPath = efs == null ? path : resolveEntropy(path, efs, true);

	// create the stream on the original file system to let the safety net
	// take its effect
	final FSDataOutputStream out = fs.create(processedPath, writeMode);
	return new OutputStreamAndPath(out, processedPath);
}
 

@Override
public void go() throws Exception {
	fs.initOutPathLocalFS(path.getParent(), WriteMode.OVERWRITE, true);
	try (FSDataOutputStream out = fs.create(path, WriteMode.OVERWRITE)) {
		out.write(11);
	}
}
 

@Test
public void testOpenTimeoutOutputStreams() throws Exception {
	final long openTimeout = 50L;
	final int maxConcurrentOpen = 2;

	final LimitedConnectionsFileSystem limitedFs = new LimitedConnectionsFileSystem(
			LocalFileSystem.getSharedInstance(),
			maxConcurrentOpen, // limited total
			openTimeout,       // small opening timeout
			0L);               // infinite inactivity timeout

	// create the threads that block all streams
	final BlockingWriterThread[] threads = new BlockingWriterThread[maxConcurrentOpen];
	for (int i = 0; i < maxConcurrentOpen; i++) {
		Path path = new Path(tempFolder.newFile().toURI());
		threads[i] = new BlockingWriterThread(limitedFs, path, Integer.MAX_VALUE, maxConcurrentOpen);
		threads[i].start();
	}

	// wait until all are open
	while (limitedFs.getTotalNumberOfOpenStreams() < maxConcurrentOpen) {
		Thread.sleep(1);
	}

	// try to open another thread
	try {
		limitedFs.create(new Path(tempFolder.newFile().toURI()), WriteMode.OVERWRITE);
		fail("this should have timed out");
	}
	catch (IOException e) {
		// expected
	}

	// clean shutdown
	for (BlockingWriterThread t : threads) {
		t.wakeup();
		t.sync();
	}
}
 

@Override
public void go() throws Exception {

	try (FSDataOutputStream stream = fs.create(path, WriteMode.OVERWRITE)) {
		assertTrue(fs.getNumberOfOpenOutputStreams() <= maxConcurrentOutputStreams);
		assertTrue(fs.getTotalNumberOfOpenStreams() <= maxConcurrentStreamsTotal);

		final Random rnd = new Random();
		final byte[] data = new byte[rnd.nextInt(10000) + 1];
		rnd.nextBytes(data);
		stream.write(data);
	}
}
 

@Test
public void testDelegateOutStreamMethods() throws IOException {

	// mock the output stream
	final FSDataOutputStream mockOut = mock(FSDataOutputStream.class);
	final long outPos = 46651L;
	when(mockOut.getPos()).thenReturn(outPos);

	final FileSystem fs = mock(FileSystem.class);
	when(fs.create(any(Path.class), any(WriteMode.class))).thenReturn(mockOut);

	final LimitedConnectionsFileSystem lfs = new LimitedConnectionsFileSystem(fs, 100);
	final FSDataOutputStream out = lfs.create(mock(Path.class), WriteMode.OVERWRITE);

	// validate the output stream

	out.write(77);
	verify(mockOut).write(77);

	{
		byte[] bytes = new byte[1786];
		out.write(bytes, 100, 111);
		verify(mockOut).write(bytes, 100, 111);
	}

	assertEquals(outPos, out.getPos());

	out.flush();
	verify(mockOut).flush();

	out.sync();
	verify(mockOut).sync();

	out.close();
	verify(mockOut).close();
}
 

@Test
public void testWithSafetyNet() throws Exception {
	final String entropyKey = "__ekey__";
	final String entropyValue = "abc";

	final File folder = TMP_FOLDER.newFolder();

	final Path path = new Path(Path.fromLocalFile(folder), entropyKey + "/path/");
	final Path pathWithEntropy = new Path(Path.fromLocalFile(folder), entropyValue + "/path/");

	TestEntropyInjectingFs efs = new TestEntropyInjectingFs(entropyKey, entropyValue);

	FSDataOutputStream out;

	FileSystemSafetyNet.initializeSafetyNetForThread();
	FileSystem fs = FileSystemSafetyNet.wrapWithSafetyNetWhenActivated(efs);
	try  {
		OutputStreamAndPath streamAndPath = EntropyInjector.createEntropyAware(
				fs, path, WriteMode.NO_OVERWRITE);

		out = streamAndPath.stream();

		assertEquals(pathWithEntropy, streamAndPath.path());
	}
	finally {
		FileSystemSafetyNet.closeSafetyNetAndGuardedResourcesForThread();
	}

	// check that the safety net closed the stream
	try {
		out.write(42);
		out.flush();
		fail("stream should be already close and hence fail with an exception");
	} catch (IOException ignored) {}
}
 

@Test
public void testCreateEntropyAwareEntropyFs() throws Exception {
	File folder = TMP_FOLDER.newFolder();
	Path path = new Path(Path.fromLocalFile(folder), "_entropy_/file");
	Path pathWithEntropy = new Path(Path.fromLocalFile(folder), "test-entropy/file");

	FileSystem fs = new TestEntropyInjectingFs("_entropy_", "test-entropy");

	OutputStreamAndPath out = EntropyInjector.createEntropyAware(fs, path, WriteMode.NO_OVERWRITE);

	out.stream().close();

	assertEquals(new Path(Path.fromLocalFile(folder), "test-entropy/file"), out.path());
	assertTrue(new File (new File(folder, "test-entropy"), "file").exists());
}
 

@Test
public void testWithSafetyNet() throws Exception {
	final String entropyKey = "__ekey__";
	final String entropyValue = "abc";

	final File folder = TMP_FOLDER.newFolder();

	final Path path = new Path(Path.fromLocalFile(folder), entropyKey + "/path/");
	final Path pathWithEntropy = new Path(Path.fromLocalFile(folder), entropyValue + "/path/");

	TestEntropyInjectingFs efs = new TestEntropyInjectingFs(entropyKey, entropyValue);

	FSDataOutputStream out;

	FileSystemSafetyNet.initializeSafetyNetForThread();
	FileSystem fs = FileSystemSafetyNet.wrapWithSafetyNetWhenActivated(efs);
	try  {
		OutputStreamAndPath streamAndPath = EntropyInjector.createEntropyAware(
				fs, path, WriteMode.NO_OVERWRITE);

		out = streamAndPath.stream();

		assertEquals(pathWithEntropy, streamAndPath.path());
	}
	finally {
		FileSystemSafetyNet.closeSafetyNetAndGuardedResourcesForThread();
	}

	// check that the safety net closed the stream
	try {
		out.write(42);
		out.flush();
		fail("stream should be already close and hence fail with an exception");
	} catch (IOException ignored) {}
}
 

private void createFile(Path path, boolean overwrite) throws IOException {
	final WriteMode writeMode = overwrite ? WriteMode.OVERWRITE : WriteMode.NO_OVERWRITE;

	try (FSDataOutputStream out = fileSystem.create(path, writeMode)) {
		out.write(42);
	}
}
 

@Test
public void testPointerPathResolution() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path metadataFile = new Path(Path.fromLocalFile(tmp.newFolder()), AbstractFsCheckpointStorage.METADATA_FILE_NAME);

	final String basePointer = metadataFile.getParent().toString();

	final String pointer1 = metadataFile.toString();
	final String pointer2 = metadataFile.getParent().toString();
	final String pointer3 = metadataFile.getParent().toString() + '/';

	// create the storage for some random checkpoint directory
	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());

	final byte[] data = new byte[23686];
	new Random().nextBytes(data);
	try (FSDataOutputStream out = fs.create(metadataFile, WriteMode.NO_OVERWRITE)) {
		out.write(data);
	}

	CompletedCheckpointStorageLocation completed1 = storage.resolveCheckpoint(pointer1);
	CompletedCheckpointStorageLocation completed2 = storage.resolveCheckpoint(pointer2);
	CompletedCheckpointStorageLocation completed3 = storage.resolveCheckpoint(pointer3);

	assertEquals(basePointer, completed1.getExternalPointer());
	assertEquals(basePointer, completed2.getExternalPointer());
	assertEquals(basePointer, completed3.getExternalPointer());

	StreamStateHandle handle1 = completed1.getMetadataHandle();
	StreamStateHandle handle2 = completed2.getMetadataHandle();
	StreamStateHandle handle3 = completed3.getMetadataHandle();

	assertNotNull(handle1);
	assertNotNull(handle2);
	assertNotNull(handle3);

	validateContents(handle1, data);
	validateContents(handle2, data);
	validateContents(handle3, data);
}
 

public void registerFileOutputTask(
	FileOutputFormat<Record> outputFormat,
	String outPath,
	Configuration formatParams) {

	outputFormat.setOutputFilePath(new Path(outPath));
	outputFormat.setWriteMode(WriteMode.OVERWRITE);

	OperatorID operatorID = new OperatorID();
	new InputOutputFormatContainer(Thread.currentThread().getContextClassLoader())
		.addOutputFormat(operatorID, outputFormat)
		.addParameters(operatorID, formatParams)
		.write(new TaskConfig(this.mockEnv.getTaskConfiguration()));
}