org.apache.flink.fs.s3.common.utils.RefCountedTmpFileCreator Java Examples

The following examples show how to use org.apache.flink.fs.s3.common.utils.RefCountedTmpFileCreator. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FlinkS3FileSystem.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a FlinkS3FileSystem based on the given Hadoop S3 file system.
 * The given Hadoop file system object is expected to be initialized already.
 *
 * <p>This constructor additionally configures the entropy injection for the file system.
 *
 * @param hadoopS3FileSystem The Hadoop FileSystem that will be used under the hood.
 * @param entropyInjectionKey The substring that will be replaced by entropy or removed.
 * @param entropyLength The number of random alphanumeric characters to inject as entropy.
 */
public FlinkS3FileSystem(
		org.apache.hadoop.fs.FileSystem hadoopS3FileSystem,
		String localTmpDirectory,
		@Nullable String entropyInjectionKey,
		int entropyLength,
		@Nullable S3AccessHelper s3UploadHelper,
		long s3uploadPartSize,
		int maxConcurrentUploadsPerStream) {

	super(hadoopS3FileSystem);

	if (entropyInjectionKey != null && entropyLength <= 0) {
		throw new IllegalArgumentException("Entropy length must be >= 0 when entropy injection key is set");
	}

	this.entropyInjectionKey = entropyInjectionKey;
	this.entropyLength = entropyLength;

	// recoverable writer parameter configuration initialization
	this.localTmpDir = Preconditions.checkNotNull(localTmpDirectory);
	this.tmpFileCreator = RefCountedTmpFileCreator.inDirectories(new File(localTmpDirectory));
	this.s3AccessHelper = s3UploadHelper;
	this.uploadThreadPool = Executors.newCachedThreadPool();

	Preconditions.checkArgument(s3uploadPartSize >= S3_MULTIPART_MIN_PART_SIZE);
	this.s3uploadPartSize = s3uploadPartSize;
	this.maxConcurrentUploadsPerStream = maxConcurrentUploadsPerStream;
}
 
Example #2
Source File: FlinkS3FileSystem.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a FlinkS3FileSystem based on the given Hadoop S3 file system.
 * The given Hadoop file system object is expected to be initialized already.
 *
 * <p>This constructor additionally configures the entropy injection for the file system.
 *
 * @param hadoopS3FileSystem The Hadoop FileSystem that will be used under the hood.
 * @param entropyInjectionKey The substring that will be replaced by entropy or removed.
 * @param entropyLength The number of random alphanumeric characters to inject as entropy.
 */
public FlinkS3FileSystem(
		org.apache.hadoop.fs.FileSystem hadoopS3FileSystem,
		String localTmpDirectory,
		@Nullable String entropyInjectionKey,
		int entropyLength,
		@Nullable S3AccessHelper s3UploadHelper,
		long s3uploadPartSize,
		int maxConcurrentUploadsPerStream) {

	super(hadoopS3FileSystem);

	if (entropyInjectionKey != null && entropyLength <= 0) {
		throw new IllegalArgumentException("Entropy length must be >= 0 when entropy injection key is set");
	}

	this.entropyInjectionKey = entropyInjectionKey;
	this.entropyLength = entropyLength;

	// recoverable writer parameter configuration initialization
	this.localTmpDir = Preconditions.checkNotNull(localTmpDirectory);
	this.tmpFileCreator = RefCountedTmpFileCreator.inDirectories(new File(localTmpDirectory));
	this.s3AccessHelper = s3UploadHelper;
	this.uploadThreadPool = Executors.newCachedThreadPool();

	Preconditions.checkArgument(s3uploadPartSize >= S3_MULTIPART_MIN_PART_SIZE);
	this.s3uploadPartSize = s3uploadPartSize;
	this.maxConcurrentUploadsPerStream = maxConcurrentUploadsPerStream;
}
 
Example #3
Source File: FlinkS3FileSystem.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a FlinkS3FileSystem based on the given Hadoop S3 file system.
 * The given Hadoop file system object is expected to be initialized already.
 *
 * <p>This constructor additionally configures the entropy injection for the file system.
 *
 * @param hadoopS3FileSystem The Hadoop FileSystem that will be used under the hood.
 * @param entropyInjectionKey The substring that will be replaced by entropy or removed.
 * @param entropyLength The number of random alphanumeric characters to inject as entropy.
 */
public FlinkS3FileSystem(
		org.apache.hadoop.fs.FileSystem hadoopS3FileSystem,
		String localTmpDirectory,
		@Nullable String entropyInjectionKey,
		int entropyLength,
		@Nullable S3AccessHelper s3UploadHelper,
		long s3uploadPartSize,
		int maxConcurrentUploadsPerStream) {

	super(hadoopS3FileSystem);

	if (entropyInjectionKey != null && entropyLength <= 0) {
		throw new IllegalArgumentException("Entropy length must be >= 0 when entropy injection key is set");
	}

	this.entropyInjectionKey = entropyInjectionKey;
	this.entropyLength = entropyLength;

	// recoverable writer parameter configuration initialization
	this.localTmpDir = Preconditions.checkNotNull(localTmpDirectory);
	this.tmpFileCreator = RefCountedTmpFileCreator.inDirectories(new File(localTmpDirectory));
	this.s3AccessHelper = s3UploadHelper;
	this.uploadThreadPool = Executors.newCachedThreadPool();

	Preconditions.checkArgument(s3uploadPartSize >= S3_MULTIPART_MIN_PART_SIZE);
	this.s3uploadPartSize = s3uploadPartSize;
	this.maxConcurrentUploadsPerStream = maxConcurrentUploadsPerStream;
}