Java Code Examples for org.apache.flink.fs.s3.common.utils.RefCountedFSOutputStream#getPos()

The following examples show how to use org.apache.flink.fs.s3.common.utils.RefCountedFSOutputStream#getPos() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
/**
 * Adds a part to the uploads without any size limitations.
 *
 * <p>This method is non-blocking and does not wait for the part upload to complete.
 *
 * @param file The file with the part data.
 *
 * @throws IOException If this method throws an exception, the RecoverableS3MultiPartUpload
 *                     should not be used any more, but recovered instead.
 */
@Override
public void uploadPart(RefCountedFSOutputStream file) throws IOException {
	// this is to guarantee that nobody is
	// writing to the file we are uploading.
	checkState(file.isClosed());

	final CompletableFuture<PartETag> future = new CompletableFuture<>();
	uploadsInProgress.add(future);

	final long partLength = file.getPos();
	currentUploadInfo.registerNewPart(partLength);

	file.retain(); // keep the file while the async upload still runs
	uploadThreadPool.execute(new UploadTask(s3AccessHelper, currentUploadInfo, file, future));
}
 
Example 2
/**
 * Creates a snapshot of this MultiPartUpload, from which the upload can be resumed.
 *
 * <p>Data buffered locally which is less than
 * {@link org.apache.flink.fs.s3.common.FlinkS3FileSystem#S3_MULTIPART_MIN_PART_SIZE S3_MULTIPART_MIN_PART_SIZE},
 * and cannot be uploaded as part of the MPU and set to S3 as independent objects.
 *
 * <p>This implementation currently blocks until all part uploads are complete and returns
 * a completed future.
 */
@Override
public S3Recoverable snapshotAndGetRecoverable(@Nullable final RefCountedFSOutputStream incompletePartFile) throws IOException {

	final String incompletePartObjectName = safelyUploadSmallPart(incompletePartFile);

	// make sure all other uploads are complete
	// this currently makes the method blocking,
	// to be made non-blocking in the future
	awaitPendingPartsUpload();

	final String objectName = currentUploadInfo.getObjectName();
	final String uploadId = currentUploadInfo.getUploadId();
	final List<PartETag> completedParts = currentUploadInfo.getCopyOfEtagsOfCompleteParts();
	final long sizeInBytes = currentUploadInfo.getExpectedSizeInBytes();

	if (incompletePartObjectName == null) {
		return new S3Recoverable(objectName, uploadId, completedParts, sizeInBytes);
	} else {
		return new S3Recoverable(objectName, uploadId, completedParts, sizeInBytes, incompletePartObjectName, incompletePartFile.getPos());
	}
}
 
Example 3
@Nullable
private String safelyUploadSmallPart(@Nullable RefCountedFSOutputStream file) throws IOException {

	if (file == null || file.getPos() == 0L) {
		return null;
	}

	// first, upload the trailing data file. during that time, other in-progress uploads may complete.
	final String incompletePartObjectName = createIncompletePartObjectName();
	file.retain();
	try {
		s3AccessHelper.putObject(incompletePartObjectName, file.getInputFile());
	}
	finally {
		file.release();
	}
	return incompletePartObjectName;
}
 
Example 4
/**
 * Adds a part to the uploads without any size limitations.
 *
 * <p>This method is non-blocking and does not wait for the part upload to complete.
 *
 * @param file The file with the part data.
 *
 * @throws IOException If this method throws an exception, the RecoverableS3MultiPartUpload
 *                     should not be used any more, but recovered instead.
 */
@Override
public void uploadPart(RefCountedFSOutputStream file) throws IOException {
	// this is to guarantee that nobody is
	// writing to the file we are uploading.
	checkState(file.isClosed());

	final CompletableFuture<PartETag> future = new CompletableFuture<>();
	uploadsInProgress.add(future);

	final long partLength = file.getPos();
	currentUploadInfo.registerNewPart(partLength);

	file.retain(); // keep the file while the async upload still runs
	uploadThreadPool.execute(new UploadTask(s3AccessHelper, currentUploadInfo, file, future));
}
 
Example 5
/**
 * Creates a snapshot of this MultiPartUpload, from which the upload can be resumed.
 *
 * <p>Data buffered locally which is less than
 * {@link org.apache.flink.fs.s3.common.FlinkS3FileSystem#S3_MULTIPART_MIN_PART_SIZE S3_MULTIPART_MIN_PART_SIZE},
 * and cannot be uploaded as part of the MPU and set to S3 as independent objects.
 *
 * <p>This implementation currently blocks until all part uploads are complete and returns
 * a completed future.
 */
@Override
public S3Recoverable snapshotAndGetRecoverable(@Nullable final RefCountedFSOutputStream incompletePartFile) throws IOException {

	final String incompletePartObjectName = safelyUploadSmallPart(incompletePartFile);

	// make sure all other uploads are complete
	// this currently makes the method blocking,
	// to be made non-blocking in the future
	awaitPendingPartsUpload();

	final String objectName = currentUploadInfo.getObjectName();
	final String uploadId = currentUploadInfo.getUploadId();
	final List<PartETag> completedParts = currentUploadInfo.getCopyOfEtagsOfCompleteParts();
	final long sizeInBytes = currentUploadInfo.getExpectedSizeInBytes();

	if (incompletePartObjectName == null) {
		return new S3Recoverable(objectName, uploadId, completedParts, sizeInBytes);
	} else {
		return new S3Recoverable(objectName, uploadId, completedParts, sizeInBytes, incompletePartObjectName, incompletePartFile.getPos());
	}
}
 
Example 6
@Nullable
private String safelyUploadSmallPart(@Nullable RefCountedFSOutputStream file) throws IOException {

	if (file == null || file.getPos() == 0L) {
		return null;
	}

	// first, upload the trailing data file. during that time, other in-progress uploads may complete.
	final String incompletePartObjectName = createIncompletePartObjectName();
	file.retain();
	try {
		s3AccessHelper.putObject(incompletePartObjectName, file.getInputFile());
	}
	finally {
		file.release();
	}
	return incompletePartObjectName;
}
 
Example 7
/**
 * Adds a part to the uploads without any size limitations.
 *
 * <p>This method is non-blocking and does not wait for the part upload to complete.
 *
 * @param file The file with the part data.
 *
 * @throws IOException If this method throws an exception, the RecoverableS3MultiPartUpload
 *                     should not be used any more, but recovered instead.
 */
@Override
public void uploadPart(RefCountedFSOutputStream file) throws IOException {
	// this is to guarantee that nobody is
	// writing to the file we are uploading.
	checkState(file.isClosed());

	final CompletableFuture<PartETag> future = new CompletableFuture<>();
	uploadsInProgress.add(future);

	final long partLength = file.getPos();
	currentUploadInfo.registerNewPart(partLength);

	file.retain(); // keep the file while the async upload still runs
	uploadThreadPool.execute(new UploadTask(s3AccessHelper, currentUploadInfo, file, future));
}
 
Example 8
/**
 * Creates a snapshot of this MultiPartUpload, from which the upload can be resumed.
 *
 * <p>Data buffered locally which is less than
 * {@link org.apache.flink.fs.s3.common.FlinkS3FileSystem#S3_MULTIPART_MIN_PART_SIZE S3_MULTIPART_MIN_PART_SIZE},
 * and cannot be uploaded as part of the MPU and set to S3 as independent objects.
 *
 * <p>This implementation currently blocks until all part uploads are complete and returns
 * a completed future.
 */
@Override
public S3Recoverable snapshotAndGetRecoverable(@Nullable final RefCountedFSOutputStream incompletePartFile) throws IOException {

	final String incompletePartObjectName = safelyUploadSmallPart(incompletePartFile);

	// make sure all other uploads are complete
	// this currently makes the method blocking,
	// to be made non-blocking in the future
	awaitPendingPartsUpload();

	final String objectName = currentUploadInfo.getObjectName();
	final String uploadId = currentUploadInfo.getUploadId();
	final List<PartETag> completedParts = currentUploadInfo.getCopyOfEtagsOfCompleteParts();
	final long sizeInBytes = currentUploadInfo.getExpectedSizeInBytes();

	if (incompletePartObjectName == null) {
		return new S3Recoverable(objectName, uploadId, completedParts, sizeInBytes);
	} else {
		return new S3Recoverable(objectName, uploadId, completedParts, sizeInBytes, incompletePartObjectName, incompletePartFile.getPos());
	}
}
 
Example 9
@Nullable
private String safelyUploadSmallPart(@Nullable RefCountedFSOutputStream file) throws IOException {

	if (file == null || file.getPos() == 0L) {
		return null;
	}

	// first, upload the trailing data file. during that time, other in-progress uploads may complete.
	final String incompletePartObjectName = createIncompletePartObjectName();
	file.retain();
	try {
		s3AccessHelper.putObject(incompletePartObjectName, file.getInputFile());
	}
	finally {
		file.release();
	}
	return incompletePartObjectName;
}
 
Example 10
@Override
public RecoverableWriter.ResumeRecoverable snapshotAndGetRecoverable(RefCountedFSOutputStream incompletePartFile) throws IOException {
	lastPersistedIndex = uploadedContent.size();

	if (incompletePartFile.getPos() >= 0L) {
		byte[] bytes = readFileContents(incompletePartFile);
		uncompleted = Optional.of(bytes);
	}

	return null;
}
 
Example 11
@Override
public void uploadPart(RefCountedFSOutputStream file) throws IOException {
	numParts++;
	numBytes += file.getPos();

	uploadedContent.add(readFileContents(file));
}
 
Example 12
@Override
public RecoverableWriter.ResumeRecoverable snapshotAndGetRecoverable(RefCountedFSOutputStream incompletePartFile) throws IOException {
	lastPersistedIndex = uploadedContent.size();

	if (incompletePartFile.getPos() >= 0L) {
		byte[] bytes = readFileContents(incompletePartFile);
		uncompleted = Optional.of(bytes);
	}

	return null;
}
 
Example 13
@Override
public void uploadPart(RefCountedFSOutputStream file) throws IOException {
	numParts++;
	numBytes += file.getPos();

	uploadedContent.add(readFileContents(file));
}
 
Example 14
@Override
public RecoverableWriter.ResumeRecoverable snapshotAndGetRecoverable(RefCountedFSOutputStream incompletePartFile) throws IOException {
	lastPersistedIndex = uploadedContent.size();

	if (incompletePartFile.getPos() >= 0L) {
		byte[] bytes = readFileContents(incompletePartFile);
		uncompleted = Optional.of(bytes);
	}

	return null;
}
 
Example 15
@Override
public void uploadPart(RefCountedFSOutputStream file) throws IOException {
	numParts++;
	numBytes += file.getPos();

	uploadedContent.add(readFileContents(file));
}