Java Code Examples for org.apache.flink.fs.s3.common.utils.RefCountedBufferingFileStream

The following examples show how to use org.apache.flink.fs.s3.common.utils.RefCountedBufferingFileStream. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
public static S3RecoverableFsDataOutputStream newStream(
		final RecoverableMultiPartUpload upload,
		final FunctionWithException<File, RefCountedFile, IOException> tmpFileCreator,
		final long userDefinedMinPartSize) throws IOException {

	checkArgument(userDefinedMinPartSize >= S3_MULTIPART_MIN_PART_SIZE);

	final RefCountedBufferingFileStream fileStream = boundedBufferingFileStream(tmpFileCreator, Optional.empty());

	return new S3RecoverableFsDataOutputStream(
			upload,
			tmpFileCreator,
			fileStream,
			userDefinedMinPartSize,
			0L);
}
 
Example 2
public static S3RecoverableFsDataOutputStream recoverStream(
		final RecoverableMultiPartUpload upload,
		final FunctionWithException<File, RefCountedFile, IOException> tmpFileCreator,
		final long userDefinedMinPartSize,
		final long bytesBeforeCurrentPart) throws IOException {

	checkArgument(userDefinedMinPartSize >= S3_MULTIPART_MIN_PART_SIZE);

	final RefCountedBufferingFileStream fileStream = boundedBufferingFileStream(
			tmpFileCreator,
			upload.getIncompletePart());

	return new S3RecoverableFsDataOutputStream(
			upload,
			tmpFileCreator,
			fileStream,
			userDefinedMinPartSize,
			bytesBeforeCurrentPart);
}
 
Example 3
@Before
public void beforeTest() throws IOException {
	fileProvider = new TestFileProvider(TEMP_FOLDER);

	multipartUploadUnderTest = new TestMultipartUpload(fileProvider);

	RefCountedBufferingFileStream fileStream = RefCountedBufferingFileStream.openNew(fileProvider);

	streamUnderTest = new S3RecoverableFsDataOutputStream(
			multipartUploadUnderTest,
			fileProvider,
			fileStream,
			USER_DEFINED_MIN_PART_SIZE,
			0L
	);
}
 
Example 4
Source Project: flink   Source File: S3RecoverableFsDataOutputStream.java    License: Apache License 2.0 6 votes vote down vote up
public static S3RecoverableFsDataOutputStream newStream(
		final RecoverableMultiPartUpload upload,
		final FunctionWithException<File, RefCountedFile, IOException> tmpFileCreator,
		final long userDefinedMinPartSize) throws IOException {

	checkArgument(userDefinedMinPartSize >= S3_MULTIPART_MIN_PART_SIZE);

	final RefCountedBufferingFileStream fileStream = boundedBufferingFileStream(tmpFileCreator, Optional.empty());

	return new S3RecoverableFsDataOutputStream(
			upload,
			tmpFileCreator,
			fileStream,
			userDefinedMinPartSize,
			0L);
}
 
Example 5
Source Project: flink   Source File: S3RecoverableFsDataOutputStream.java    License: Apache License 2.0 6 votes vote down vote up
public static S3RecoverableFsDataOutputStream recoverStream(
		final RecoverableMultiPartUpload upload,
		final FunctionWithException<File, RefCountedFile, IOException> tmpFileCreator,
		final long userDefinedMinPartSize,
		final long bytesBeforeCurrentPart) throws IOException {

	checkArgument(userDefinedMinPartSize >= S3_MULTIPART_MIN_PART_SIZE);

	final RefCountedBufferingFileStream fileStream = boundedBufferingFileStream(
			tmpFileCreator,
			upload.getIncompletePart());

	return new S3RecoverableFsDataOutputStream(
			upload,
			tmpFileCreator,
			fileStream,
			userDefinedMinPartSize,
			bytesBeforeCurrentPart);
}
 
Example 6
Source Project: flink   Source File: S3RecoverableFsDataOutputStreamTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void beforeTest() throws IOException {
	fileProvider = new TestFileProvider(TEMP_FOLDER);

	multipartUploadUnderTest = new TestMultipartUpload(fileProvider);

	RefCountedBufferingFileStream fileStream = RefCountedBufferingFileStream.openNew(fileProvider);

	streamUnderTest = new S3RecoverableFsDataOutputStream(
			multipartUploadUnderTest,
			fileProvider,
			fileStream,
			USER_DEFINED_MIN_PART_SIZE,
			0L
	);
}
 
Example 7
Source Project: flink   Source File: S3RecoverableFsDataOutputStream.java    License: Apache License 2.0 6 votes vote down vote up
public static S3RecoverableFsDataOutputStream newStream(
		final RecoverableMultiPartUpload upload,
		final FunctionWithException<File, RefCountedFileWithStream, IOException> tmpFileCreator,
		final long userDefinedMinPartSize) throws IOException {

	checkArgument(userDefinedMinPartSize >= S3_MULTIPART_MIN_PART_SIZE);

	final RefCountedBufferingFileStream fileStream = boundedBufferingFileStream(tmpFileCreator, Optional.empty());

	return new S3RecoverableFsDataOutputStream(
			upload,
			tmpFileCreator,
			fileStream,
			userDefinedMinPartSize,
			0L);
}
 
Example 8
Source Project: flink   Source File: S3RecoverableFsDataOutputStream.java    License: Apache License 2.0 6 votes vote down vote up
public static S3RecoverableFsDataOutputStream recoverStream(
		final RecoverableMultiPartUpload upload,
		final FunctionWithException<File, RefCountedFileWithStream, IOException> tmpFileCreator,
		final long userDefinedMinPartSize,
		final long bytesBeforeCurrentPart) throws IOException {

	checkArgument(userDefinedMinPartSize >= S3_MULTIPART_MIN_PART_SIZE);

	final RefCountedBufferingFileStream fileStream = boundedBufferingFileStream(
			tmpFileCreator,
			upload.getIncompletePart());

	return new S3RecoverableFsDataOutputStream(
			upload,
			tmpFileCreator,
			fileStream,
			userDefinedMinPartSize,
			bytesBeforeCurrentPart);
}
 
Example 9
Source Project: flink   Source File: S3RecoverableFsDataOutputStreamTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void beforeTest() throws IOException {
	fileProvider = new TestFileProvider(TEMP_FOLDER);

	multipartUploadUnderTest = new TestMultipartUpload(fileProvider);

	RefCountedBufferingFileStream fileStream = RefCountedBufferingFileStream.openNew(fileProvider);

	streamUnderTest = new S3RecoverableFsDataOutputStream(
			multipartUploadUnderTest,
			fileProvider,
			fileStream,
			USER_DEFINED_MIN_PART_SIZE,
			0L
	);
}
 
Example 10
private void uploadCurrentAndOpenNewPart(long fileLength) throws IOException {
	bytesBeforeCurrentPart += fileLength;
	closeAndUploadPart();

	// initialize a new temp file
	fileStream = RefCountedBufferingFileStream.openNew(tmpFileProvider);
}
 
Example 11
private static RefCountedBufferingFileStream boundedBufferingFileStream(
		final FunctionWithException<File, RefCountedFile, IOException> tmpFileCreator,
		final Optional<File> incompletePart) throws IOException {

	if (!incompletePart.isPresent()) {
		return RefCountedBufferingFileStream.openNew(tmpFileCreator);
	}

	final File file = incompletePart.get();
	return RefCountedBufferingFileStream.restore(tmpFileCreator, file);
}
 
Example 12
@Test
public void discardingUnpersistedUploadedData() throws IOException {
	streamUnderTest.write(bytesOf("hello"));

	streamUnderTest.persist();
	streamUnderTest.write(randomBuffer(RefCountedBufferingFileStream.BUFFER_SIZE + 1));
	streamUnderTest = reopenStreamUnderTestAfterRecovery();

	streamUnderTest.write(bytesOf(" world"));
	streamUnderTest.closeForCommit().commit();

	assertThat(multipartUploadUnderTest, hasContent(bytesOf("hello world")));
}
 
Example 13
private S3RecoverableFsDataOutputStream reopenStreamUnderTestAfterRecovery() throws IOException {
	final long bytesBeforeCurrentPart = multipartUploadUnderTest.numBytes;
	final Optional<File> incompletePart = multipartUploadUnderTest.getIncompletePart();

	RefCountedBufferingFileStream fileStream = RefCountedBufferingFileStream.restore(fileProvider, incompletePart.get());
	multipartUploadUnderTest.discardUnpersistedData();

	return new S3RecoverableFsDataOutputStream(
			multipartUploadUnderTest,
			fileProvider,
			fileStream,
			USER_DEFINED_MIN_PART_SIZE,
			bytesBeforeCurrentPart
	);
}
 
Example 14
@Test(expected = IllegalStateException.class)
public void uploadingNonClosedFileAsCompleteShouldThroughException() throws IOException {
	final byte[] incompletePart = bytesOf("!!!");

	final RefCountedBufferingFileStream incompletePartFile =
			writeContent(incompletePart);

	multiPartUploadUnderTest.uploadPart(incompletePartFile);
}
 
Example 15
private S3Recoverable uploadObject(byte[] content) throws IOException {
	final RefCountedBufferingFileStream incompletePartFile = writeContent(content);
	incompletePartFile.flush();

	// as in the production code, we assume that a file containing
	// a in-progress part is flushed but not closed before being passed
	// to the uploader.

	return multiPartUploadUnderTest.snapshotAndGetRecoverable(incompletePartFile);
}
 
Example 16
private RefCountedBufferingFileStream writeContent(byte[] content) throws IOException {
	final File newFile = new File(temporaryFolder.getRoot(), ".tmp_" + UUID.randomUUID());
	final OutputStream out = Files.newOutputStream(newFile.toPath(), StandardOpenOption.CREATE_NEW);

	final RefCountedBufferingFileStream testStream =
			new RefCountedBufferingFileStream(RefCountedFile.newFile(newFile, out), BUFFER_SIZE);

	testStream.write(content, 0, content.length);
	return testStream;
}
 
Example 17
Source Project: flink   Source File: S3RecoverableFsDataOutputStream.java    License: Apache License 2.0 5 votes vote down vote up
private void uploadCurrentAndOpenNewPart(long fileLength) throws IOException {
	bytesBeforeCurrentPart += fileLength;
	closeAndUploadPart();

	// initialize a new temp file
	fileStream = RefCountedBufferingFileStream.openNew(tmpFileProvider);
}
 
Example 18
Source Project: flink   Source File: S3RecoverableFsDataOutputStream.java    License: Apache License 2.0 5 votes vote down vote up
private static RefCountedBufferingFileStream boundedBufferingFileStream(
		final FunctionWithException<File, RefCountedFile, IOException> tmpFileCreator,
		final Optional<File> incompletePart) throws IOException {

	if (!incompletePart.isPresent()) {
		return RefCountedBufferingFileStream.openNew(tmpFileCreator);
	}

	final File file = incompletePart.get();
	return RefCountedBufferingFileStream.restore(tmpFileCreator, file);
}
 
Example 19
Source Project: flink   Source File: S3RecoverableFsDataOutputStreamTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void discardingUnpersistedUploadedData() throws IOException {
	streamUnderTest.write(bytesOf("hello"));

	streamUnderTest.persist();
	streamUnderTest.write(randomBuffer(RefCountedBufferingFileStream.BUFFER_SIZE + 1));
	streamUnderTest = reopenStreamUnderTestAfterRecovery();

	streamUnderTest.write(bytesOf(" world"));
	streamUnderTest.closeForCommit().commit();

	assertThat(multipartUploadUnderTest, hasContent(bytesOf("hello world")));
}
 
Example 20
Source Project: flink   Source File: S3RecoverableFsDataOutputStreamTest.java    License: Apache License 2.0 5 votes vote down vote up
private S3RecoverableFsDataOutputStream reopenStreamUnderTestAfterRecovery() throws IOException {
	final long bytesBeforeCurrentPart = multipartUploadUnderTest.numBytes;
	final Optional<File> incompletePart = multipartUploadUnderTest.getIncompletePart();

	RefCountedBufferingFileStream fileStream = RefCountedBufferingFileStream.restore(fileProvider, incompletePart.get());
	multipartUploadUnderTest.discardUnpersistedData();

	return new S3RecoverableFsDataOutputStream(
			multipartUploadUnderTest,
			fileProvider,
			fileStream,
			USER_DEFINED_MIN_PART_SIZE,
			bytesBeforeCurrentPart
	);
}
 
Example 21
Source Project: flink   Source File: RecoverableMultiPartUploadImplTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test(expected = IllegalStateException.class)
public void uploadingNonClosedFileAsCompleteShouldThroughException() throws IOException {
	final byte[] incompletePart = bytesOf("!!!");

	final RefCountedBufferingFileStream incompletePartFile =
			writeContent(incompletePart);

	multiPartUploadUnderTest.uploadPart(incompletePartFile);
}
 
Example 22
Source Project: flink   Source File: RecoverableMultiPartUploadImplTest.java    License: Apache License 2.0 5 votes vote down vote up
private S3Recoverable uploadObject(byte[] content) throws IOException {
	final RefCountedBufferingFileStream incompletePartFile = writeContent(content);
	incompletePartFile.flush();

	// as in the production code, we assume that a file containing
	// a in-progress part is flushed but not closed before being passed
	// to the uploader.

	return multiPartUploadUnderTest.snapshotAndGetRecoverable(incompletePartFile);
}
 
Example 23
Source Project: flink   Source File: RecoverableMultiPartUploadImplTest.java    License: Apache License 2.0 5 votes vote down vote up
private RefCountedBufferingFileStream writeContent(byte[] content) throws IOException {
	final File newFile = new File(temporaryFolder.getRoot(), ".tmp_" + UUID.randomUUID());
	final OutputStream out = Files.newOutputStream(newFile.toPath(), StandardOpenOption.CREATE_NEW);

	final RefCountedBufferingFileStream testStream =
			new RefCountedBufferingFileStream(RefCountedFile.newFile(newFile, out), BUFFER_SIZE);

	testStream.write(content, 0, content.length);
	return testStream;
}
 
Example 24
Source Project: flink   Source File: S3RecoverableFsDataOutputStream.java    License: Apache License 2.0 5 votes vote down vote up
private void uploadCurrentAndOpenNewPart(long fileLength) throws IOException {
	bytesBeforeCurrentPart += fileLength;
	closeAndUploadPart();

	// initialize a new temp file
	fileStream = RefCountedBufferingFileStream.openNew(tmpFileProvider);
}
 
Example 25
Source Project: flink   Source File: S3RecoverableFsDataOutputStream.java    License: Apache License 2.0 5 votes vote down vote up
private static RefCountedBufferingFileStream boundedBufferingFileStream(
		final FunctionWithException<File, RefCountedFileWithStream, IOException> tmpFileCreator,
		final Optional<File> incompletePart) throws IOException {

	if (!incompletePart.isPresent()) {
		return RefCountedBufferingFileStream.openNew(tmpFileCreator);
	}

	final File file = incompletePart.get();
	return RefCountedBufferingFileStream.restore(tmpFileCreator, file);
}
 
Example 26
Source Project: flink   Source File: S3RecoverableFsDataOutputStreamTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void discardingUnpersistedUploadedData() throws IOException {
	streamUnderTest.write(bytesOf("hello"));

	streamUnderTest.persist();
	streamUnderTest.write(randomBuffer(RefCountedBufferingFileStream.BUFFER_SIZE + 1));
	streamUnderTest = reopenStreamUnderTestAfterRecovery();

	streamUnderTest.write(bytesOf(" world"));
	streamUnderTest.closeForCommit().commit();

	assertThat(multipartUploadUnderTest, hasContent(bytesOf("hello world")));
}
 
Example 27
Source Project: flink   Source File: S3RecoverableFsDataOutputStreamTest.java    License: Apache License 2.0 5 votes vote down vote up
private S3RecoverableFsDataOutputStream reopenStreamUnderTestAfterRecovery() throws IOException {
	final long bytesBeforeCurrentPart = multipartUploadUnderTest.numBytes;
	final Optional<File> incompletePart = multipartUploadUnderTest.getIncompletePart();

	RefCountedBufferingFileStream fileStream = RefCountedBufferingFileStream.restore(fileProvider, incompletePart.get());
	multipartUploadUnderTest.discardUnpersistedData();

	return new S3RecoverableFsDataOutputStream(
			multipartUploadUnderTest,
			fileProvider,
			fileStream,
			USER_DEFINED_MIN_PART_SIZE,
			bytesBeforeCurrentPart
	);
}
 
Example 28
Source Project: flink   Source File: RecoverableMultiPartUploadImplTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test(expected = IllegalStateException.class)
public void uploadingNonClosedFileAsCompleteShouldThroughException() throws IOException {
	final byte[] incompletePart = bytesOf("!!!");

	final RefCountedBufferingFileStream incompletePartFile =
			writeContent(incompletePart);

	multiPartUploadUnderTest.uploadPart(incompletePartFile);
}
 
Example 29
Source Project: flink   Source File: RecoverableMultiPartUploadImplTest.java    License: Apache License 2.0 5 votes vote down vote up
private S3Recoverable uploadObject(byte[] content) throws IOException {
	final RefCountedBufferingFileStream incompletePartFile = writeContent(content);
	incompletePartFile.flush();

	// as in the production code, we assume that a file containing
	// a in-progress part is flushed but not closed before being passed
	// to the uploader.

	return multiPartUploadUnderTest.snapshotAndGetRecoverable(incompletePartFile);
}
 
Example 30
Source Project: flink   Source File: RecoverableMultiPartUploadImplTest.java    License: Apache License 2.0 5 votes vote down vote up
private RefCountedBufferingFileStream writeContent(byte[] content) throws IOException {
	final File newFile = new File(temporaryFolder.getRoot(), ".tmp_" + UUID.randomUUID());
	final OutputStream out = Files.newOutputStream(newFile.toPath(), StandardOpenOption.CREATE_NEW);

	final RefCountedBufferingFileStream testStream =
			new RefCountedBufferingFileStream(RefCountedFileWithStream.newFile(newFile, out), BUFFER_SIZE);

	testStream.write(content, 0, content.length);
	return testStream;
}