org.apache.flink.core.fs.FSDataOutputStream Java Examples

The following examples show how to use org.apache.flink.core.fs.FSDataOutputStream. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: LocalFileSystem.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public FSDataOutputStream create(final Path filePath, final WriteMode overwrite) throws IOException {
	checkNotNull(filePath, "filePath");

	if (exists(filePath) && overwrite == WriteMode.NO_OVERWRITE) {
		throw new FileAlreadyExistsException("File already exists: " + filePath);
	}

	final Path parent = filePath.getParent();
	if (parent != null && !mkdirs(parent)) {
		throw new IOException("Mkdirs failed to create " + parent);
	}

	final File file = pathToFile(filePath);
	return new LocalDataOutputStream(file);
}
 
Example #2
Source File: CheckpointStateOutputStreamTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Validates that even empty streams create a file and a file state handle.
 */
@Test
public void testEmptyState() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path folder = baseFolder();
	final String fileName = "myFileName";
	final Path filePath = new Path(folder, fileName);

	final FileStateHandle handle;
	try (FSDataOutputStream stream = createTestStream(fs, folder, fileName)) {
		handle = closeAndGetResult(stream);
	}

	// must have created a handle
	assertNotNull(handle);
	assertEquals(filePath, handle.getFilePath());

	// the pointer path should exist as a directory
	assertTrue(fs.exists(handle.getFilePath()));
	assertFalse(fs.getFileStatus(filePath).isDir());

	// the contents should be empty
	try (FSDataInputStream in = handle.openInputStream()) {
		assertEquals(-1, in.read());
	}
}
 
Example #3
Source File: LocalFileSystem.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public FSDataOutputStream create(final Path filePath, final WriteMode overwrite) throws IOException {
	checkNotNull(filePath, "filePath");

	if (exists(filePath) && overwrite == WriteMode.NO_OVERWRITE) {
		throw new FileAlreadyExistsException("File already exists: " + filePath);
	}

	final Path parent = filePath.getParent();
	if (parent != null && !mkdirs(parent)) {
		throw new IOException("Mkdirs failed to create " + parent);
	}

	final File file = pathToFile(filePath);
	return new LocalDataOutputStream(file);
}
 
Example #4
Source File: FileSystemTableSink.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public BulkWriter<RowData> create(FSDataOutputStream out) throws IOException {
	BulkWriter<RowData> writer = factory.create(out);
	return new BulkWriter<RowData>() {

		@Override
		public void addElement(RowData element) throws IOException {
			writer.addElement(computer.projectColumnsToWrite(element));
		}

		@Override
		public void flush() throws IOException {
			writer.flush();
		}

		@Override
		public void finish() throws IOException {
			writer.finish();
		}
	};
}
 
Example #5
Source File: BlockingCheckpointOutputStream.java    From flink with Apache License 2.0 6 votes vote down vote up
public BlockingCheckpointOutputStream(
	@Nullable FSDataOutputStream delegate,
	@Nullable OneShotLatch waitForBlocking,
	@Nullable OneShotLatch triggerUnblock,
	long blockAtPosition) {

	this.delegate = delegate;
	this.triggerUnblock = triggerUnblock;
	this.waitForBlocking = waitForBlocking;
	this.blockAtPosition = blockAtPosition;
	if (delegate != null) {
		try {
			this.position = delegate.getPos();
		} catch (IOException e) {
			throw new RuntimeException(e);
		}
	} else {
		this.position = 0;
	}
	this.closed = new AtomicBoolean(false);
}
 
Example #6
Source File: FileSystemStateStorageHelper.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public RetrievableStateHandle<T> store(T state) throws Exception {
	Exception latestException = null;

	for (int attempt = 0; attempt < 10; attempt++) {
		Path filePath = getNewFilePath();

		try (FSDataOutputStream outStream = fs.create(filePath, FileSystem.WriteMode.NO_OVERWRITE)) {
			InstantiationUtil.serializeObject(outStream, state);
			return new RetrievableStreamStateHandle<T>(filePath, outStream.getPos());
		}
		catch (Exception e) {
			latestException = e;
		}
	}

	throw new Exception("Could not open output stream for state backend", latestException);
}
 
Example #7
Source File: CheckpointStateOutputStreamTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the underlying stream file is deleted upon calling close.
 */
@Test
public void testCleanupWhenClosingStream() throws IOException {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path folder = new Path(tmp.newFolder().toURI());
	final String fileName = "nonCreativeTestFileName";
	final Path path = new Path(folder, fileName);

	// write some test data and close the stream
	try (FSDataOutputStream stream = createTestStream(fs, folder, fileName)) {
		Random rnd = new Random();
		for (int i = 0; i < rnd.nextInt(1000); i++) {
			stream.write(rnd.nextInt(100));
		}
		assertTrue(fs.exists(path));
	}

	assertFalse(fs.exists(path));
}
 
Example #8
Source File: CheckpointStateOutputStreamTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the underlying stream file is deleted if the closeAndGetHandle method fails.
 */
@Test
public void testCleanupWhenFailingCloseAndGetHandle() throws IOException {
	final Path folder = new Path(tmp.newFolder().toURI());
	final String fileName = "test_name";
	final Path filePath = new Path(folder, fileName);

	final FileSystem fs = spy(new TestFs((path) -> new FailingCloseStream(new File(path.getPath()))));

	FSDataOutputStream stream = createTestStream(fs, folder, fileName);
	stream.write(new byte[] {1, 2, 3, 4, 5});

	try {
		closeAndGetResult(stream);
		fail("Expected IOException");
	}
	catch (IOException ignored) {
		// expected exception
	}

	verify(fs).delete(filePath, false);
}
 
Example #9
Source File: FsCheckpointStateOutputStreamTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the underlying stream file is deleted upon calling close.
 */
@Test
public void testCleanupWhenClosingStream() throws IOException {

	final FileSystem fs = mock(FileSystem.class);
	final FSDataOutputStream outputStream = mock(FSDataOutputStream.class);

	final ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);

	when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream);

	CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
		Path.fromLocalFile(tempDir.newFolder()),
		fs,
		4,
		0);

	// this should create the underlying file stream
	stream.write(new byte[] {1, 2, 3, 4, 5});

	verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class));

	stream.close();

	verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean());
}
 
Example #10
Source File: CheckpointStateOutputStreamTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the underlying stream file is deleted if the closeAndGetHandle method fails.
 */
@Test
public void testCleanupWhenFailingCloseAndGetHandle() throws IOException {
	final Path folder = new Path(tmp.newFolder().toURI());
	final String fileName = "test_name";
	final Path filePath = new Path(folder, fileName);

	final FileSystem fs = spy(new TestFs((path) -> new FailingCloseStream(new File(path.getPath()))));

	FSDataOutputStream stream = createTestStream(fs, folder, fileName);
	stream.write(new byte[] {1, 2, 3, 4, 5});

	try {
		closeAndGetResult(stream);
		fail("Expected IOException");
	}
	catch (IOException ignored) {
		// expected exception
	}

	verify(fs).delete(filePath, false);
}
 
Example #11
Source File: BlockingCheckpointOutputStream.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public BlockingCheckpointOutputStream(
	@Nullable FSDataOutputStream delegate,
	@Nullable OneShotLatch waitForBlocking,
	@Nullable OneShotLatch triggerUnblock,
	long blockAtPosition) {

	this.delegate = delegate;
	this.triggerUnblock = triggerUnblock;
	this.waitForBlocking = waitForBlocking;
	this.blockAtPosition = blockAtPosition;
	if (delegate != null) {
		try {
			this.position = delegate.getPos();
		} catch (IOException e) {
			throw new RuntimeException(e);
		}
	} else {
		this.position = 0;
	}
	this.closed = new AtomicBoolean(false);
}
 
Example #12
Source File: CheckpointStateOutputStreamTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the underlying stream file is deleted if the closeAndGetHandle method fails.
 */
@Test
public void testCleanupWhenFailingCloseAndGetHandle() throws IOException {
	final Path folder = new Path(tmp.newFolder().toURI());
	final String fileName = "test_name";
	final Path filePath = new Path(folder, fileName);

	final FileSystem fs = spy(new TestFs((path) -> new FailingCloseStream(new File(path.getPath()))));

	FSDataOutputStream stream = createTestStream(fs, folder, fileName);
	stream.write(new byte[] {1, 2, 3, 4, 5});

	try {
		closeAndGetResult(stream);
		fail("Expected IOException");
	}
	catch (IOException ignored) {
		// expected exception
	}

	verify(fs).delete(filePath, false);
}
 
Example #13
Source File: CheckpointStateOutputStreamTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the underlying stream file is deleted upon calling close.
 */
@Test
public void testCleanupWhenClosingStream() throws IOException {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path folder = new Path(tmp.newFolder().toURI());
	final String fileName = "nonCreativeTestFileName";
	final Path path = new Path(folder, fileName);

	// write some test data and close the stream
	try (FSDataOutputStream stream = createTestStream(fs, folder, fileName)) {
		Random rnd = new Random();
		for (int i = 0; i < rnd.nextInt(1000); i++) {
			stream.write(rnd.nextInt(100));
		}
		assertTrue(fs.exists(path));
	}

	assertFalse(fs.exists(path));
}
 
Example #14
Source File: LocalFileSystem.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public FSDataOutputStream create(final Path filePath, final WriteMode overwrite) throws IOException {
	checkNotNull(filePath, "filePath");

	if (exists(filePath) && overwrite == WriteMode.NO_OVERWRITE) {
		throw new FileAlreadyExistsException("File already exists: " + filePath);
	}

	final Path parent = filePath.getParent();
	if (parent != null && !mkdirs(parent)) {
		throw new IOException("Mkdirs failed to create " + parent);
	}

	final File file = pathToFile(filePath);
	return new LocalDataOutputStream(file);
}
 
Example #15
Source File: CheckpointStateOutputStreamTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Validates that even empty streams create a file and a file state handle.
 */
@Test
public void testEmptyState() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path folder = baseFolder();
	final String fileName = "myFileName";
	final Path filePath = new Path(folder, fileName);

	final FileStateHandle handle;
	try (FSDataOutputStream stream = createTestStream(fs, folder, fileName)) {
		handle = closeAndGetResult(stream);
	}

	// must have created a handle
	assertNotNull(handle);
	assertEquals(filePath, handle.getFilePath());

	// the pointer path should exist as a directory
	assertTrue(fs.exists(handle.getFilePath()));
	assertFalse(fs.getFileStatus(filePath).isDir());

	// the contents should be empty
	try (FSDataInputStream in = handle.openInputStream()) {
		assertEquals(-1, in.read());
	}
}
 
Example #16
Source File: AvroFileSystemFormatFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public BulkWriter<RowData> create(FSDataOutputStream out) throws IOException {
	BulkWriter<GenericRecord> writer = factory.create(out);
	AvroRowDataSerializationSchema.SerializationRuntimeConverter converter =
			AvroRowDataSerializationSchema.createRowConverter(rowType);
	Schema schema = AvroSchemaConverter.convertToSchema(rowType);
	return new BulkWriter<RowData>() {

		@Override
		public void addElement(RowData element) throws IOException {
			GenericRecord record = (GenericRecord) converter.convert(schema, element);
			writer.addElement(record);
		}

		@Override
		public void flush() throws IOException {
			writer.flush();
		}

		@Override
		public void finish() throws IOException {
			writer.finish();
		}
	};
}
 
Example #17
Source File: FileSystemStateStorageHelper.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public RetrievableStateHandle<T> store(T state) throws Exception {
	Exception latestException = null;

	for (int attempt = 0; attempt < 10; attempt++) {
		Path filePath = getNewFilePath();

		try (FSDataOutputStream outStream = fs.create(filePath, FileSystem.WriteMode.NO_OVERWRITE)) {
			InstantiationUtil.serializeObject(outStream, state);
			return new RetrievableStreamStateHandle<T>(filePath, outStream.getPos());
		}
		catch (Exception e) {
			latestException = e;
		}
	}

	throw new Exception("Could not open output stream for state backend", latestException);
}
 
Example #18
Source File: PartitionableListState.java    From flink with Apache License 2.0 5 votes vote down vote up
public long[] write(FSDataOutputStream out) throws IOException {

		long[] partitionOffsets = new long[internalList.size()];

		DataOutputView dov = new DataOutputViewStreamWrapper(out);

		for (int i = 0; i < internalList.size(); ++i) {
			S element = internalList.get(i);
			partitionOffsets[i] = out.getPos();
			getStateMetaInfo().getPartitionStateSerializer().serialize(element, dov);
		}

		return partitionOffsets;
	}
 
Example #19
Source File: HadoopSwiftFileSystemITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSimpleFileWriteAndRead() throws Exception {
	final Configuration conf = createConfiguration();

	final String testLine = "Hello Upload!";

	FileSystem.initialize(conf);

	final Path path = new Path("swift://" + CONTAINER + '.' + SERVICENAME + '/' + TEST_DATA_DIR + "/test.txt");
	final FileSystem fs = path.getFileSystem();

	try {
		try (FSDataOutputStream out = fs.create(path, WriteMode.OVERWRITE);
			OutputStreamWriter writer = new OutputStreamWriter(out, StandardCharsets.UTF_8)) {
			writer.write(testLine);
		}

		try (FSDataInputStream in = fs.open(path);
			InputStreamReader ir = new InputStreamReader(in, StandardCharsets.UTF_8);
			BufferedReader reader = new BufferedReader(ir)) {
			String line = reader.readLine();
			assertEquals(testLine, line);
		}
	}
	finally {
		fs.delete(path, false);
	}
}
 
Example #20
Source File: HeapBroadcastState.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public long write(FSDataOutputStream out) throws IOException {
	long partitionOffset = out.getPos();

	DataOutputView dov = new DataOutputViewStreamWrapper(out);
	dov.writeInt(backingMap.size());
	for (Map.Entry<K, V> entry: backingMap.entrySet()) {
		getStateMetaInfo().getKeySerializer().serialize(entry.getKey(), dov);
		getStateMetaInfo().getValueSerializer().serialize(entry.getValue(), dov);
	}

	return partitionOffset;
}
 
Example #21
Source File: CheckpointStateOutputStreamTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * This test validates that a close operation can happen even while a 'closeAndGetHandle()'
 * call is in progress.
 * <p>
 * <p>That behavior is essential for fast cancellation (concurrent cleanup).
 */
@Test
public void testCloseDoesNotLock() throws Exception {
	final Path folder = new Path(tmp.newFolder().toURI());
	final String fileName = "this-is-ignored-anyways.file";

	final FileSystem fileSystem = spy(new TestFs((path) -> new BlockerStream()));

	final FSDataOutputStream checkpointStream =
		createTestStream(fileSystem, folder, fileName);

	final OneShotLatch sync = new OneShotLatch();

	final CheckedThread thread = new CheckedThread() {

		@Override
		public void go() throws Exception {
			sync.trigger();
			// that call should now block, because it accesses the position
			closeAndGetResult(checkpointStream);
		}
	};
	thread.start();

	sync.await();
	checkpointStream.close();

	// the thread may or may not fail, that depends on the thread race
	// it is not important for this test, important is that the thread does not freeze/lock up
	try {
		thread.sync();
	} catch (IOException ignored) {}
}
 
Example #22
Source File: FsNegativeRunningJobsRegistry.java    From flink with Apache License 2.0 5 votes vote down vote up
private void createFile(Path path, boolean overwrite) throws IOException {
	final WriteMode writeMode = overwrite ? WriteMode.OVERWRITE : WriteMode.NO_OVERWRITE;

	try (FSDataOutputStream out = fileSystem.create(path, writeMode)) {
		out.write(42);
	}
}
 
Example #23
Source File: FileOutputFormat.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void close() throws IOException {
	final FSDataOutputStream s = this.stream;
	if (s != null) {
		this.stream = null;
		s.close();
	}
}
 
Example #24
Source File: FileUtils.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private static void internalCopyFile(Path sourcePath, Path targetPath, boolean executable, FileSystem sFS, FileSystem tFS) throws IOException {
	try (FSDataOutputStream lfsOutput = tFS.create(targetPath, FileSystem.WriteMode.NO_OVERWRITE); FSDataInputStream fsInput = sFS.open(sourcePath)) {
		IOUtils.copyBytes(fsInput, lfsOutput);
		//noinspection ResultOfMethodCallIgnored
		new File(targetPath.toString()).setExecutable(executable);
	}
}
 
Example #25
Source File: FsCheckpointStateOutputStreamTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the underlying stream file is deleted if the closeAndGetHandle method fails.
 */
@Test
public void testCleanupWhenFailingCloseAndGetHandle() throws IOException {
	final FileSystem fs = mock(FileSystem.class);
	final FSDataOutputStream outputStream = mock(FSDataOutputStream.class);

	final ArgumentCaptor<Path>  pathCaptor = ArgumentCaptor.forClass(Path.class);

	when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream);
	doThrow(new IOException("Test IOException.")).when(outputStream).close();

	CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
		Path.fromLocalFile(tempDir.newFolder()),
		fs,
		4,
		0,
		relativePaths);

	// this should create the underlying file stream
	stream.write(new byte[] {1, 2, 3, 4, 5});

	verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class));

	try {
		stream.closeAndGetHandle();
		fail("Expected IOException");
	} catch (IOException ioE) {
		// expected exception
	}

	verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean());
}
 
Example #26
Source File: CheckpointStateOutputStreamTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * This test validates that a close operation can happen even while a 'closeAndGetHandle()'
 * call is in progress.
 * <p>
 * <p>That behavior is essential for fast cancellation (concurrent cleanup).
 */
@Test
public void testCloseDoesNotLock() throws Exception {
	final Path folder = new Path(tmp.newFolder().toURI());
	final String fileName = "this-is-ignored-anyways.file";

	final FileSystem fileSystem = spy(new TestFs((path) -> new BlockerStream()));

	final FSDataOutputStream checkpointStream =
		createTestStream(fileSystem, folder, fileName);

	final OneShotLatch sync = new OneShotLatch();

	final CheckedThread thread = new CheckedThread() {

		@Override
		public void go() throws Exception {
			sync.trigger();
			// that call should now block, because it accesses the position
			closeAndGetResult(checkpointStream);
		}
	};
	thread.start();

	sync.await();
	checkpointStream.close();

	// the thread may or may not fail, that depends on the thread race
	// it is not important for this test, important is that the thread does not freeze/lock up
	try {
		thread.sync();
	} catch (IOException ignored) {}
}
 
Example #27
Source File: HeapBroadcastState.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public long write(FSDataOutputStream out) throws IOException {
	long partitionOffset = out.getPos();

	DataOutputView dov = new DataOutputViewStreamWrapper(out);
	dov.writeInt(backingMap.size());
	for (Map.Entry<K, V> entry: backingMap.entrySet()) {
		getStateMetaInfo().getKeySerializer().serialize(entry.getKey(), dov);
		getStateMetaInfo().getValueSerializer().serialize(entry.getValue(), dov);
	}

	return partitionOffset;
}
 
Example #28
Source File: CheckpointStateOutputStreamTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Closes the stream successfully and returns a FileStateHandle to the result.
 */
private FileStateHandle closeAndGetResult(FSDataOutputStream stream) throws IOException {
	switch (stateOutputStreamType) {
		case FileBasedState:
			return ((FileBasedStateOutputStream) stream).closeAndGetHandle();
		case FsCheckpointMetaData:
			return ((FsCheckpointMetadataOutputStream) stream).closeAndFinalizeCheckpoint().getMetadataHandle();
		default:
			throw new IllegalStateException("Unsupported checkpoint stream output type.");
	}
}
 
Example #29
Source File: PhysicalWriterImpl.java    From flink with Apache License 2.0 5 votes vote down vote up
void spillToDiskAndClear(FSDataOutputStream raw) throws IOException {
	if (!isSuppressed) {
		for (ByteBuffer buffer: output) {
			raw.write(buffer.array(), buffer.arrayOffset() + buffer.position(),
				buffer.remaining());
		}
		output.clear();
	}
	isSuppressed = false;
}
 
Example #30
Source File: FileUtils.java    From flink with Apache License 2.0 5 votes vote down vote up
public static Path expandDirectory(Path file, Path targetDirectory) throws IOException {
	FileSystem sourceFs = file.getFileSystem();
	FileSystem targetFs = targetDirectory.getFileSystem();
	Path rootDir = null;
	try (ZipInputStream zis = new ZipInputStream(sourceFs.open(file))) {
		ZipEntry entry;
		while ((entry = zis.getNextEntry()) != null) {
			Path relativePath = new Path(entry.getName());
			if (rootDir == null) {
				// the first entry contains the name of the original directory that was zipped
				rootDir = relativePath;
			}

			Path newFile = new Path(targetDirectory, relativePath);
			if (entry.isDirectory()) {
				targetFs.mkdirs(newFile);
			} else {
				try (FSDataOutputStream fileStream = targetFs.create(newFile, FileSystem.WriteMode.NO_OVERWRITE)) {
					// do not close the streams here as it prevents access to further zip entries
					IOUtils.copyBytes(zis, fileStream, false);
				}
			}
			zis.closeEntry();
		}
	}
	return new Path(targetDirectory, rootDir);
}