Java Code Examples for org.apache.flink.core.fs.FileSystem.WriteMode

The following examples show how to use org.apache.flink.core.fs.FileSystem.WriteMode. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Override
public void go() throws Exception {

	try (FSDataOutputStream stream = fs.create(path, WriteMode.OVERWRITE)) {
		assertTrue(fs.getNumberOfOpenOutputStreams() <= maxConcurrentOutputStreams);
		assertTrue(fs.getTotalNumberOfOpenStreams() <= maxConcurrentStreamsTotal);

		final Random rnd = new Random();
		final byte[] data = new byte[rnd.nextInt(10000) + 1];
		rnd.nextBytes(data);
		stream.write(data);

		waitTillWokenUp();

		// try to write one more thing, which might/should fail with an I/O exception
		stream.write(rnd.nextInt());
	}
}
 
Example 2
Source Project: flink   Source File: EntropyInjectorTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testClassLoaderFixingFs() throws Exception {
	final String entropyKey = "__ekey__";
	final String entropyValue = "abc";

	final File folder = TMP_FOLDER.newFolder();

	final Path path = new Path(Path.fromLocalFile(folder), entropyKey + "/path/");
	final Path pathWithEntropy = new Path(Path.fromLocalFile(folder), entropyValue + "/path/");

	PluginFileSystemFactory pluginFsFactory = PluginFileSystemFactory.of(
			new TestFileSystemFactory(entropyKey, entropyValue));
	FileSystem testFs = pluginFsFactory.create(URI.create("test"));

	FileSystemSafetyNet.initializeSafetyNetForThread();
	FileSystem fs = FileSystemSafetyNet.wrapWithSafetyNetWhenActivated(testFs);
	try  {
		OutputStreamAndPath streamAndPath = EntropyInjector.createEntropyAware(
				fs, path, WriteMode.NO_OVERWRITE);

		assertEquals(pathWithEntropy, streamAndPath.path());
	}
	finally {
		FileSystemSafetyNet.closeSafetyNetAndGuardedResourcesForThread();
	}
}
 
Example 3
Source Project: flink   Source File: LimitedConnectionsFileSystemTest.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void go() throws Exception {

	try (FSDataOutputStream stream = fs.create(path, WriteMode.OVERWRITE)) {
		assertTrue(fs.getNumberOfOpenOutputStreams() <= maxConcurrentOutputStreams);
		assertTrue(fs.getTotalNumberOfOpenStreams() <= maxConcurrentStreamsTotal);

		final Random rnd = new Random();
		final byte[] data = new byte[rnd.nextInt(10000) + 1];
		rnd.nextBytes(data);
		stream.write(data);

		waitTillWokenUp();

		// try to write one more thing, which might/should fail with an I/O exception
		stream.write(rnd.nextInt());
	}
}
 
Example 4
Source Project: Flink-CEPplus   Source File: FsCheckpointStreamFactory.java    License: Apache License 2.0 6 votes vote down vote up
private void createStream() throws IOException {
	Exception latestException = null;
	for (int attempt = 0; attempt < 10; attempt++) {
		try {
			OutputStreamAndPath streamAndPath = EntropyInjector.createEntropyAware(
					fs, createStatePath(), WriteMode.NO_OVERWRITE);
			this.outStream = streamAndPath.stream();
			this.statePath = streamAndPath.path();
			return;
		}
		catch (Exception e) {
			latestException = e;
		}
	}

	throw new IOException("Could not open output stream for state backend", latestException);
}
 
Example 5
Source Project: flink   Source File: DataStream.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Writes a DataStream to the file specified by the path parameter. The
 * writing is performed periodically every millis milliseconds.
 *
 * <p>For every field of an element of the DataStream the result of {@link Object#toString()}
 * is written. This method can only be used on data streams of tuples.
 *
 * @param path
 *            the path pointing to the location the text file is written to
 * @param writeMode
 *            Controls the behavior for existing files. Options are
 *            NO_OVERWRITE and OVERWRITE.
 * @param rowDelimiter
 *            the delimiter for two rows
 * @param fieldDelimiter
 *            the delimiter for two fields
 *
 * @return the closed DataStream
 *
 * @deprecated Please use the {@link org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink} explicitly using the
 * {@link #addSink(SinkFunction)} method.
 */
@SuppressWarnings("unchecked")
@Deprecated
@PublicEvolving
public <X extends Tuple> DataStreamSink<T> writeAsCsv(
		String path,
		WriteMode writeMode,
		String rowDelimiter,
		String fieldDelimiter) {
	Preconditions.checkArgument(
		getType().isTupleType(),
		"The writeAsCsv() method can only be used on data streams of tuples.");

	CsvOutputFormat<X> of = new CsvOutputFormat<>(
		new Path(path),
		rowDelimiter,
		fieldDelimiter);

	if (writeMode != null) {
		of.setWriteMode(writeMode);
	}

	return writeUsingOutputFormat((OutputFormat<T>) of);
}
 
Example 6
Source Project: Flink-CEPplus   Source File: DataStream.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Writes a DataStream to the file specified by the path parameter. The
 * writing is performed periodically every millis milliseconds.
 *
 * <p>For every field of an element of the DataStream the result of {@link Object#toString()}
 * is written. This method can only be used on data streams of tuples.
 *
 * @param path
 *            the path pointing to the location the text file is written to
 * @param writeMode
 *            Controls the behavior for existing files. Options are
 *            NO_OVERWRITE and OVERWRITE.
 * @param rowDelimiter
 *            the delimiter for two rows
 * @param fieldDelimiter
 *            the delimiter for two fields
 *
 * @return the closed DataStream
 */
@SuppressWarnings("unchecked")
@PublicEvolving
public <X extends Tuple> DataStreamSink<T> writeAsCsv(
		String path,
		WriteMode writeMode,
		String rowDelimiter,
		String fieldDelimiter) {
	Preconditions.checkArgument(
		getType().isTupleType(),
		"The writeAsCsv() method can only be used on data streams of tuples.");

	CsvOutputFormat<X> of = new CsvOutputFormat<>(
		new Path(path),
		rowDelimiter,
		fieldDelimiter);

	if (writeMode != null) {
		of.setWriteMode(writeMode);
	}

	return writeUsingOutputFormat((OutputFormat<T>) of);
}
 
Example 7
Source Project: flink   Source File: FsCheckpointStreamFactory.java    License: Apache License 2.0 6 votes vote down vote up
private void createStream() throws IOException {
	Exception latestException = null;
	for (int attempt = 0; attempt < 10; attempt++) {
		try {
			OutputStreamAndPath streamAndPath = EntropyInjector.createEntropyAware(
					fs, createStatePath(), WriteMode.NO_OVERWRITE);
			this.outStream = streamAndPath.stream();
			this.statePath = streamAndPath.path();
			return;
		}
		catch (Exception e) {
			latestException = e;
		}
	}

	throw new IOException("Could not open output stream for state backend", latestException);
}
 
Example 8
Source Project: flink   Source File: LimitedConnectionsFileSystemTest.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void go() throws Exception {

	try (FSDataOutputStream stream = fs.create(path, WriteMode.OVERWRITE)) {
		assertTrue(fs.getNumberOfOpenOutputStreams() <= maxConcurrentOutputStreams);
		assertTrue(fs.getTotalNumberOfOpenStreams() <= maxConcurrentStreamsTotal);

		final Random rnd = new Random();
		final byte[] data = new byte[rnd.nextInt(10000) + 1];
		rnd.nextBytes(data);
		stream.write(data);

		waitTillWokenUp();

		// try to write one more thing, which might/should fail with an I/O exception
		stream.write(rnd.nextInt());
	}
}
 
Example 9
Source Project: flink   Source File: FsCheckpointStreamFactory.java    License: Apache License 2.0 6 votes vote down vote up
private void createStream() throws IOException {
	Exception latestException = null;
	for (int attempt = 0; attempt < 10; attempt++) {
		try {
			OutputStreamAndPath streamAndPath = EntropyInjector.createEntropyAware(
					fs, createStatePath(), WriteMode.NO_OVERWRITE);
			this.outStream = streamAndPath.stream();
			this.statePath = streamAndPath.path();
			return;
		}
		catch (Exception e) {
			latestException = e;
		}
	}

	throw new IOException("Could not open output stream for state backend", latestException);
}
 
Example 10
Source Project: flink   Source File: DataStream.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Writes a DataStream to the file specified by the path parameter. The
 * writing is performed periodically every millis milliseconds.
 *
 * <p>For every field of an element of the DataStream the result of {@link Object#toString()}
 * is written. This method can only be used on data streams of tuples.
 *
 * @param path
 *            the path pointing to the location the text file is written to
 * @param writeMode
 *            Controls the behavior for existing files. Options are
 *            NO_OVERWRITE and OVERWRITE.
 * @param rowDelimiter
 *            the delimiter for two rows
 * @param fieldDelimiter
 *            the delimiter for two fields
 *
 * @return the closed DataStream
 */
@SuppressWarnings("unchecked")
@PublicEvolving
public <X extends Tuple> DataStreamSink<T> writeAsCsv(
		String path,
		WriteMode writeMode,
		String rowDelimiter,
		String fieldDelimiter) {
	Preconditions.checkArgument(
		getType().isTupleType(),
		"The writeAsCsv() method can only be used on data streams of tuples.");

	CsvOutputFormat<X> of = new CsvOutputFormat<>(
		new Path(path),
		rowDelimiter,
		fieldDelimiter);

	if (writeMode != null) {
		of.setWriteMode(writeMode);
	}

	return writeUsingOutputFormat((OutputFormat<T>) of);
}
 
Example 11
Source Project: Flink-CEPplus   Source File: DataSet.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private <X extends Tuple> DataSink<T> internalWriteAsCsv(Path filePath, String rowDelimiter, String fieldDelimiter, WriteMode wm) {
	Preconditions.checkArgument(getType().isTupleType(), "The writeAsCsv() method can only be used on data sets of tuples.");
	CsvOutputFormat<X> of = new CsvOutputFormat<>(filePath, rowDelimiter, fieldDelimiter);
	if (wm != null) {
		of.setWriteMode(wm);
	}
	return output((OutputFormat<T>) of);
}
 
Example 12
Source Project: Flink-CEPplus   Source File: FileOutputFormat.java    License: Apache License 2.0 5 votes vote down vote up
public void setWriteMode(WriteMode mode) {
	if (mode == null) {
		throw new NullPointerException();
	}
	
	this.writeMode = mode;
}
 
Example 13
Source Project: Flink-CEPplus   Source File: FileOutputFormat.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Initialization of the distributed file system if it is used.
 *
 * @param parallelism The task parallelism.
 */
@Override
public void initializeGlobal(int parallelism) throws IOException {
	final Path path = getOutputFilePath();
	final FileSystem fs = path.getFileSystem();
	
	// only distributed file systems can be initialized at start-up time.
	if (fs.isDistributedFS()) {
		
		final WriteMode writeMode = getWriteMode();
		final OutputDirectoryMode outDirMode = getOutputDirectoryMode();

		if (parallelism == 1 && outDirMode == OutputDirectoryMode.PARONLY) {
			// output is not written in parallel and should be written to a single file.
			// prepare distributed output path
			if(!fs.initOutPathDistFS(path, writeMode, false)) {
				// output preparation failed! Cancel task.
				throw new IOException("Output path could not be initialized.");
			}

		} else {
			// output should be written to a directory

			// only distributed file systems can be initialized at start-up time.
			if(!fs.initOutPathDistFS(path, writeMode, true)) {
				throw new IOException("Output directory could not be created.");
			}
		}
	}
}
 
Example 14
Source Project: Flink-CEPplus   Source File: EntropyInjector.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Handles entropy injection across regular and entropy-aware file systems.
 *
 * <p>If the given file system is entropy-aware (a implements {@link EntropyInjectingFileSystem}),
 * then this method replaces the entropy marker in the path with random characters.
 * The entropy marker is defined by {@link EntropyInjectingFileSystem#getEntropyInjectionKey()}.
 *
 * <p>If the given file system does not implement {@code EntropyInjectingFileSystem},
 * then this method delegates to {@link FileSystem#create(Path, WriteMode)} and
 * returns the same path in the resulting {@code OutputStreamAndPath}.
 */
public static OutputStreamAndPath createEntropyAware(
		FileSystem fs,
		Path path,
		WriteMode writeMode) throws IOException {

	// check and possibly inject entropy into the path
	final EntropyInjectingFileSystem efs = getEntropyFs(fs);
	final Path processedPath = efs == null ? path : resolveEntropy(path, efs, true);

	// create the stream on the original file system to let the safety net
	// take its effect
	final FSDataOutputStream out = fs.create(processedPath, writeMode);
	return new OutputStreamAndPath(out, processedPath);
}
 
Example 15
Source Project: Flink-CEPplus   Source File: InitOutputPathTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void go() throws Exception {
	fs.initOutPathLocalFS(path.getParent(), WriteMode.OVERWRITE, true);
	try (FSDataOutputStream out = fs.create(path, WriteMode.OVERWRITE)) {
		out.write(11);
	}
}
 
Example 16
@Test
public void testOpenTimeoutOutputStreams() throws Exception {
	final long openTimeout = 50L;
	final int maxConcurrentOpen = 2;

	final LimitedConnectionsFileSystem limitedFs = new LimitedConnectionsFileSystem(
			LocalFileSystem.getSharedInstance(),
			maxConcurrentOpen, // limited total
			openTimeout,       // small opening timeout
			0L);               // infinite inactivity timeout

	// create the threads that block all streams
	final BlockingWriterThread[] threads = new BlockingWriterThread[maxConcurrentOpen];
	for (int i = 0; i < maxConcurrentOpen; i++) {
		Path path = new Path(tempFolder.newFile().toURI());
		threads[i] = new BlockingWriterThread(limitedFs, path, Integer.MAX_VALUE, maxConcurrentOpen);
		threads[i].start();
	}

	// wait until all are open
	while (limitedFs.getTotalNumberOfOpenStreams() < maxConcurrentOpen) {
		Thread.sleep(1);
	}

	// try to open another thread
	try {
		limitedFs.create(new Path(tempFolder.newFile().toURI()), WriteMode.OVERWRITE);
		fail("this should have timed out");
	}
	catch (IOException e) {
		// expected
	}

	// clean shutdown
	for (BlockingWriterThread t : threads) {
		t.wakeup();
		t.sync();
	}
}
 
Example 17
@Override
public void go() throws Exception {

	try (FSDataOutputStream stream = fs.create(path, WriteMode.OVERWRITE)) {
		assertTrue(fs.getNumberOfOpenOutputStreams() <= maxConcurrentOutputStreams);
		assertTrue(fs.getTotalNumberOfOpenStreams() <= maxConcurrentStreamsTotal);

		final Random rnd = new Random();
		final byte[] data = new byte[rnd.nextInt(10000) + 1];
		rnd.nextBytes(data);
		stream.write(data);
	}
}
 
Example 18
@Test
public void testDelegateOutStreamMethods() throws IOException {

	// mock the output stream
	final FSDataOutputStream mockOut = mock(FSDataOutputStream.class);
	final long outPos = 46651L;
	when(mockOut.getPos()).thenReturn(outPos);

	final FileSystem fs = mock(FileSystem.class);
	when(fs.create(any(Path.class), any(WriteMode.class))).thenReturn(mockOut);

	final LimitedConnectionsFileSystem lfs = new LimitedConnectionsFileSystem(fs, 100);
	final FSDataOutputStream out = lfs.create(mock(Path.class), WriteMode.OVERWRITE);

	// validate the output stream

	out.write(77);
	verify(mockOut).write(77);

	{
		byte[] bytes = new byte[1786];
		out.write(bytes, 100, 111);
		verify(mockOut).write(bytes, 100, 111);
	}

	assertEquals(outPos, out.getPos());

	out.flush();
	verify(mockOut).flush();

	out.sync();
	verify(mockOut).sync();

	out.close();
	verify(mockOut).close();
}
 
Example 19
Source Project: Flink-CEPplus   Source File: EntropyInjectorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCreateEntropyAwarePlainFs() throws Exception {
	File folder = TMP_FOLDER.newFolder();
	Path path = new Path(Path.fromLocalFile(folder), "_entropy_/file");

	OutputStreamAndPath out = EntropyInjector.createEntropyAware(
			LocalFileSystem.getSharedInstance(), path, WriteMode.NO_OVERWRITE);

	out.stream().close();

	assertEquals(path, out.path());
	assertTrue(new File (new File(folder, "_entropy_"), "file").exists());
}
 
Example 20
Source Project: Flink-CEPplus   Source File: EntropyInjectorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCreateEntropyAwareEntropyFs() throws Exception {
	File folder = TMP_FOLDER.newFolder();
	Path path = new Path(Path.fromLocalFile(folder), "_entropy_/file");
	Path pathWithEntropy = new Path(Path.fromLocalFile(folder), "test-entropy/file");

	FileSystem fs = new TestEntropyInjectingFs("_entropy_", "test-entropy");

	OutputStreamAndPath out = EntropyInjector.createEntropyAware(fs, path, WriteMode.NO_OVERWRITE);

	out.stream().close();

	assertEquals(new Path(Path.fromLocalFile(folder), "test-entropy/file"), out.path());
	assertTrue(new File (new File(folder, "test-entropy"), "file").exists());
}
 
Example 21
Source Project: Flink-CEPplus   Source File: EntropyInjectorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testWithSafetyNet() throws Exception {
	final String entropyKey = "__ekey__";
	final String entropyValue = "abc";

	final File folder = TMP_FOLDER.newFolder();

	final Path path = new Path(Path.fromLocalFile(folder), entropyKey + "/path/");
	final Path pathWithEntropy = new Path(Path.fromLocalFile(folder), entropyValue + "/path/");

	TestEntropyInjectingFs efs = new TestEntropyInjectingFs(entropyKey, entropyValue);

	FSDataOutputStream out;

	FileSystemSafetyNet.initializeSafetyNetForThread();
	FileSystem fs = FileSystemSafetyNet.wrapWithSafetyNetWhenActivated(efs);
	try  {
		OutputStreamAndPath streamAndPath = EntropyInjector.createEntropyAware(
				fs, path, WriteMode.NO_OVERWRITE);

		out = streamAndPath.stream();

		assertEquals(pathWithEntropy, streamAndPath.path());
	}
	finally {
		FileSystemSafetyNet.closeSafetyNetAndGuardedResourcesForThread();
	}

	// check that the safety net closed the stream
	try {
		out.write(42);
		out.flush();
		fail("stream should be already close and hence fail with an exception");
	} catch (IOException ignored) {}
}
 
Example 22
Source Project: Flink-CEPplus   Source File: HadoopSwiftFileSystemITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSimpleFileWriteAndRead() throws Exception {
	final Configuration conf = createConfiguration();

	final String testLine = "Hello Upload!";

	FileSystem.initialize(conf);

	final Path path = new Path("swift://" + CONTAINER + '.' + SERVICENAME + '/' + TEST_DATA_DIR + "/test.txt");
	final FileSystem fs = path.getFileSystem();

	try {
		try (FSDataOutputStream out = fs.create(path, WriteMode.OVERWRITE);
			OutputStreamWriter writer = new OutputStreamWriter(out, StandardCharsets.UTF_8)) {
			writer.write(testLine);
		}

		try (FSDataInputStream in = fs.open(path);
			InputStreamReader ir = new InputStreamReader(in, StandardCharsets.UTF_8);
			BufferedReader reader = new BufferedReader(ir)) {
			String line = reader.readLine();
			assertEquals(testLine, line);
		}
	}
	finally {
		fs.delete(path, false);
	}
}
 
Example 23
public FsCheckpointMetadataOutputStream(
		FileSystem fileSystem,
		Path metadataFilePath,
		Path exclusiveCheckpointDir) throws IOException {

	this.fileSystem = checkNotNull(fileSystem);
	this.metadataFilePath = checkNotNull(metadataFilePath);
	this.exclusiveCheckpointDir = checkNotNull(exclusiveCheckpointDir);

	this.out = fileSystem.create(metadataFilePath, WriteMode.NO_OVERWRITE);
}
 
Example 24
@Test
public void testDelegateOutStreamMethods() throws IOException {

	// mock the output stream
	final FSDataOutputStream mockOut = mock(FSDataOutputStream.class);
	final long outPos = 46651L;
	when(mockOut.getPos()).thenReturn(outPos);

	final FileSystem fs = mock(FileSystem.class);
	when(fs.create(any(Path.class), any(WriteMode.class))).thenReturn(mockOut);

	final LimitedConnectionsFileSystem lfs = new LimitedConnectionsFileSystem(fs, 100);
	final FSDataOutputStream out = lfs.create(mock(Path.class), WriteMode.OVERWRITE);

	// validate the output stream

	out.write(77);
	verify(mockOut).write(77);

	{
		byte[] bytes = new byte[1786];
		out.write(bytes, 100, 111);
		verify(mockOut).write(bytes, 100, 111);
	}

	assertEquals(outPos, out.getPos());

	out.flush();
	verify(mockOut).flush();

	out.sync();
	verify(mockOut).sync();

	out.close();
	verify(mockOut).close();
}
 
Example 25
@Test
public void testPointerPathResolution() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path metadataFile = new Path(Path.fromLocalFile(tmp.newFolder()), AbstractFsCheckpointStorage.METADATA_FILE_NAME);

	final String basePointer = metadataFile.getParent().toString();

	final String pointer1 = metadataFile.toString();
	final String pointer2 = metadataFile.getParent().toString();
	final String pointer3 = metadataFile.getParent().toString() + '/';

	// create the storage for some random checkpoint directory
	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());

	final byte[] data = new byte[23686];
	new Random().nextBytes(data);
	try (FSDataOutputStream out = fs.create(metadataFile, WriteMode.NO_OVERWRITE)) {
		out.write(data);
	}

	CompletedCheckpointStorageLocation completed1 = storage.resolveCheckpoint(pointer1);
	CompletedCheckpointStorageLocation completed2 = storage.resolveCheckpoint(pointer2);
	CompletedCheckpointStorageLocation completed3 = storage.resolveCheckpoint(pointer3);

	assertEquals(basePointer, completed1.getExternalPointer());
	assertEquals(basePointer, completed2.getExternalPointer());
	assertEquals(basePointer, completed3.getExternalPointer());

	StreamStateHandle handle1 = completed1.getMetadataHandle();
	StreamStateHandle handle2 = completed2.getMetadataHandle();
	StreamStateHandle handle3 = completed3.getMetadataHandle();

	assertNotNull(handle1);
	assertNotNull(handle2);
	assertNotNull(handle3);

	validateContents(handle1, data);
	validateContents(handle2, data);
	validateContents(handle3, data);
}
 
Example 26
Source Project: flink   Source File: TaskTestBase.java    License: Apache License 2.0 5 votes vote down vote up
public void registerFileOutputTask(
	FileOutputFormat<Record> outputFormat,
	String outPath,
	Configuration formatParams) {

	outputFormat.setOutputFilePath(new Path(outPath));
	outputFormat.setWriteMode(WriteMode.OVERWRITE);

	OperatorID operatorID = new OperatorID();
	new InputOutputFormatContainer(Thread.currentThread().getContextClassLoader())
		.addOutputFormat(operatorID, outputFormat)
		.addParameters(operatorID, formatParams)
		.write(new TaskConfig(this.mockEnv.getTaskConfiguration()));
}
 
Example 27
Source Project: flink   Source File: FsCheckpointMetadataOutputStream.java    License: Apache License 2.0 5 votes vote down vote up
public FsCheckpointMetadataOutputStream(
		FileSystem fileSystem,
		Path metadataFilePath,
		Path exclusiveCheckpointDir) throws IOException {

	this.fileSystem = checkNotNull(fileSystem);
	this.metadataFilePath = checkNotNull(metadataFilePath);
	this.exclusiveCheckpointDir = checkNotNull(exclusiveCheckpointDir);

	this.out = fileSystem.create(metadataFilePath, WriteMode.NO_OVERWRITE);
}
 
Example 28
Source Project: flink   Source File: DataSet.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private <X extends Tuple> DataSink<T> internalWriteAsCsv(Path filePath, String rowDelimiter, String fieldDelimiter, WriteMode wm) {
	Preconditions.checkArgument(getType().isTupleType(), "The writeAsCsv() method can only be used on data sets of tuples.");
	CsvOutputFormat<X> of = new CsvOutputFormat<>(filePath, rowDelimiter, fieldDelimiter);
	if (wm != null) {
		of.setWriteMode(wm);
	}
	return output((OutputFormat<T>) of);
}
 
Example 29
Source Project: flink   Source File: LimitedConnectionsFileSystemTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testOpenTimeoutOutputStreams() throws Exception {
	final long openTimeout = 50L;
	final int maxConcurrentOpen = 2;

	final LimitedConnectionsFileSystem limitedFs = new LimitedConnectionsFileSystem(
			LocalFileSystem.getSharedInstance(),
			maxConcurrentOpen, // limited total
			openTimeout,       // small opening timeout
			0L);               // infinite inactivity timeout

	// create the threads that block all streams
	final BlockingWriterThread[] threads = new BlockingWriterThread[maxConcurrentOpen];
	for (int i = 0; i < maxConcurrentOpen; i++) {
		Path path = new Path(tempFolder.newFile().toURI());
		threads[i] = new BlockingWriterThread(limitedFs, path, Integer.MAX_VALUE, maxConcurrentOpen);
		threads[i].start();
	}

	// wait until all are open
	while (limitedFs.getTotalNumberOfOpenStreams() < maxConcurrentOpen) {
		Thread.sleep(1);
	}

	// try to open another thread
	try {
		limitedFs.create(new Path(tempFolder.newFile().toURI()), WriteMode.OVERWRITE);
		fail("this should have timed out");
	}
	catch (IOException e) {
		// expected
	}

	// clean shutdown
	for (BlockingWriterThread t : threads) {
		t.wakeup();
		t.sync();
	}
}
 
Example 30
Source Project: flink   Source File: FileOutputFormat.java    License: Apache License 2.0 5 votes vote down vote up
public void setWriteMode(WriteMode mode) {
	if (mode == null) {
		throw new NullPointerException();
	}
	
	this.writeMode = mode;
}