org.apache.flink.streaming.connectors.fs.Clock Java Examples

The following examples show how to use org.apache.flink.streaming.connectors.fs.Clock. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BucketingSink.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);

	state = new State<>();

	processingTimeService =
			((StreamingRuntimeContext) getRuntimeContext()).getProcessingTimeService();

	long currentProcessingTime = processingTimeService.getCurrentProcessingTime();

	processingTimeService.registerTimer(currentProcessingTime + inactiveBucketCheckInterval, this);

	this.clock = new Clock() {
		@Override
		public long currentTimeMillis() {
			return processingTimeService.getCurrentProcessingTime();
		}
	};
}
Example #2
Source File: BucketingSinkTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private OneInputStreamOperatorTestHarness<String, Object> createRescalingTestSinkWithRollover(
	File outDir, int totalParallelism, int taskIdx, long inactivityInterval, long rolloverInterval) throws Exception {

	BucketingSink<String> sink = new BucketingSink<String>(outDir.getAbsolutePath())
		.setBucketer(new Bucketer<String>() {
			private static final long serialVersionUID = 1L;

			@Override
			public Path getBucketPath(Clock clock, Path basePath, String element) {
				return new Path(basePath, element);
			}
		})
		.setWriter(new StringWriter<String>())
		.setInactiveBucketCheckInterval(inactivityInterval)
		.setInactiveBucketThreshold(inactivityInterval)
		.setPartPrefix(PART_PREFIX)
		.setInProgressPrefix("")
		.setPendingPrefix("")
		.setValidLengthPrefix("")
		.setInProgressSuffix(IN_PROGRESS_SUFFIX)
		.setPendingSuffix(PENDING_SUFFIX)
		.setValidLengthSuffix(VALID_LENGTH_SUFFIX)
		.setBatchRolloverInterval(rolloverInterval);

	return createTestSink(sink, totalParallelism, taskIdx);
}
Example #3
Source File: BucketingSinkTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private OneInputStreamOperatorTestHarness<String, Object> createTestSink(File dataDir, int totalParallelism, int taskIdx) throws Exception {
	BucketingSink<String> sink = new BucketingSink<String>(dataDir.getAbsolutePath())
		.setBucketer(new Bucketer<String>() {
			private static final long serialVersionUID = 1L;

			@Override
			public Path getBucketPath(Clock clock, Path basePath, String element) {
				return new Path(basePath, element);
			}
		})
		.setWriter(new StringWriter<String>())
		.setPartPrefix(PART_PREFIX)
		.setPendingPrefix("")
		.setInactiveBucketCheckInterval(5 * 60 * 1000L)
		.setInactiveBucketThreshold(5 * 60 * 1000L)
		.setPendingSuffix(PENDING_SUFFIX)
		.setInProgressSuffix(IN_PROGRESS_SUFFIX);

	return createTestSink(sink, totalParallelism, taskIdx);
}
Example #4
Source File: BucketingSinkTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private OneInputStreamOperatorTestHarness<String, Object> createRescalingTestSink(
	File outDir, int totalParallelism, int taskIdx, long inactivityInterval) throws Exception {

	BucketingSink<String> sink = new BucketingSink<String>(outDir.getAbsolutePath())
		.setBucketer(new Bucketer<String>() {
			private static final long serialVersionUID = 1L;

			@Override
			public Path getBucketPath(Clock clock, Path basePath, String element) {
				return new Path(basePath, element);
			}
		})
		.setWriter(new StringWriter<String>())
		.setInactiveBucketCheckInterval(inactivityInterval)
		.setInactiveBucketThreshold(inactivityInterval)
		.setPartPrefix(PART_PREFIX)
		.setInProgressPrefix("")
		.setPendingPrefix("")
		.setValidLengthPrefix("")
		.setInProgressSuffix(IN_PROGRESS_SUFFIX)
		.setPendingSuffix(PENDING_SUFFIX)
		.setValidLengthSuffix(VALID_LENGTH_SUFFIX);

	return createTestSink(sink, totalParallelism, taskIdx);
}
Example #5
Source File: BucketingSinkTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private OneInputStreamOperatorTestHarness<String, Object> createTestSink(File dataDir, int totalParallelism, int taskIdx) throws Exception {
	BucketingSink<String> sink = new BucketingSink<String>(dataDir.getAbsolutePath())
		.setBucketer(new Bucketer<String>() {
			private static final long serialVersionUID = 1L;

			@Override
			public Path getBucketPath(Clock clock, Path basePath, String element) {
				return new Path(basePath, element);
			}
		})
		.setWriter(new StringWriter<String>())
		.setPartPrefix(PART_PREFIX)
		.setPendingPrefix("")
		.setInactiveBucketCheckInterval(5 * 60 * 1000L)
		.setInactiveBucketThreshold(5 * 60 * 1000L)
		.setPendingSuffix(PENDING_SUFFIX)
		.setInProgressSuffix(IN_PROGRESS_SUFFIX);

	return createTestSink(sink, totalParallelism, taskIdx);
}
Example #6
Source File: BucketingSinkTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private OneInputStreamOperatorTestHarness<String, Object> createRescalingTestSinkWithRollover(
	File outDir, int totalParallelism, int taskIdx, long inactivityInterval, long rolloverInterval) throws Exception {

	BucketingSink<String> sink = new BucketingSink<String>(outDir.getAbsolutePath())
		.setBucketer(new Bucketer<String>() {
			private static final long serialVersionUID = 1L;

			@Override
			public Path getBucketPath(Clock clock, Path basePath, String element) {
				return new Path(basePath, element);
			}
		})
		.setWriter(new StringWriter<String>())
		.setInactiveBucketCheckInterval(inactivityInterval)
		.setInactiveBucketThreshold(inactivityInterval)
		.setPartPrefix(PART_PREFIX)
		.setInProgressPrefix("")
		.setPendingPrefix("")
		.setValidLengthPrefix("")
		.setInProgressSuffix(IN_PROGRESS_SUFFIX)
		.setPendingSuffix(PENDING_SUFFIX)
		.setValidLengthSuffix(VALID_LENGTH_SUFFIX)
		.setBatchRolloverInterval(rolloverInterval);

	return createTestSink(sink, totalParallelism, taskIdx);
}
Example #7
Source File: BucketingSinkTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private OneInputStreamOperatorTestHarness<String, Object> createRescalingTestSink(
	File outDir, int totalParallelism, int taskIdx, long inactivityInterval) throws Exception {

	BucketingSink<String> sink = new BucketingSink<String>(outDir.getAbsolutePath())
		.setBucketer(new Bucketer<String>() {
			private static final long serialVersionUID = 1L;

			@Override
			public Path getBucketPath(Clock clock, Path basePath, String element) {
				return new Path(basePath, element);
			}
		})
		.setWriter(new StringWriter<String>())
		.setInactiveBucketCheckInterval(inactivityInterval)
		.setInactiveBucketThreshold(inactivityInterval)
		.setPartPrefix(PART_PREFIX)
		.setInProgressPrefix("")
		.setPendingPrefix("")
		.setValidLengthPrefix("")
		.setInProgressSuffix(IN_PROGRESS_SUFFIX)
		.setPendingSuffix(PENDING_SUFFIX)
		.setValidLengthSuffix(VALID_LENGTH_SUFFIX);

	return createTestSink(sink, totalParallelism, taskIdx);
}
Example #8
Source File: BucketingSink.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);

	state = new State<>();

	processingTimeService =
			((StreamingRuntimeContext) getRuntimeContext()).getProcessingTimeService();

	long currentProcessingTime = processingTimeService.getCurrentProcessingTime();

	processingTimeService.registerTimer(currentProcessingTime + inactiveBucketCheckInterval, this);

	this.clock = new Clock() {
		@Override
		public long currentTimeMillis() {
			return processingTimeService.getCurrentProcessingTime();
		}
	};
}
Example #9
Source File: BucketingSink.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);

	state = new State<>();

	processingTimeService =
			((StreamingRuntimeContext) getRuntimeContext()).getProcessingTimeService();

	long currentProcessingTime = processingTimeService.getCurrentProcessingTime();

	processingTimeService.registerTimer(currentProcessingTime + inactiveBucketCheckInterval, this);

	this.clock = new Clock() {
		@Override
		public long currentTimeMillis() {
			return processingTimeService.getCurrentProcessingTime();
		}
	};
}
Example #10
Source File: BucketingSinkTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private OneInputStreamOperatorTestHarness<String, Object> createRescalingTestSink(
	File outDir, int totalParallelism, int taskIdx, long inactivityInterval) throws Exception {

	BucketingSink<String> sink = new BucketingSink<String>(outDir.getAbsolutePath())
		.setBucketer(new Bucketer<String>() {
			private static final long serialVersionUID = 1L;

			@Override
			public Path getBucketPath(Clock clock, Path basePath, String element) {
				return new Path(basePath, element);
			}
		})
		.setWriter(new StringWriter<String>())
		.setInactiveBucketCheckInterval(inactivityInterval)
		.setInactiveBucketThreshold(inactivityInterval)
		.setPartPrefix(PART_PREFIX)
		.setInProgressPrefix("")
		.setPendingPrefix("")
		.setValidLengthPrefix("")
		.setInProgressSuffix(IN_PROGRESS_SUFFIX)
		.setPendingSuffix(PENDING_SUFFIX)
		.setValidLengthSuffix(VALID_LENGTH_SUFFIX);

	return createTestSink(sink, totalParallelism, taskIdx);
}
Example #11
Source File: BucketingSinkTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private OneInputStreamOperatorTestHarness<String, Object> createTestSink(File dataDir, int totalParallelism, int taskIdx) throws Exception {
	BucketingSink<String> sink = new BucketingSink<String>(dataDir.getAbsolutePath())
		.setBucketer(new Bucketer<String>() {
			private static final long serialVersionUID = 1L;

			@Override
			public Path getBucketPath(Clock clock, Path basePath, String element) {
				return new Path(basePath, element);
			}
		})
		.setWriter(new StringWriter<String>())
		.setPartPrefix(PART_PREFIX)
		.setPendingPrefix("")
		.setInactiveBucketCheckInterval(5 * 60 * 1000L)
		.setInactiveBucketThreshold(5 * 60 * 1000L)
		.setPendingSuffix(PENDING_SUFFIX)
		.setInProgressSuffix(IN_PROGRESS_SUFFIX);

	return createTestSink(sink, totalParallelism, taskIdx);
}
Example #12
Source File: BucketingSinkTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private OneInputStreamOperatorTestHarness<String, Object> createRescalingTestSinkWithRollover(
	File outDir, int totalParallelism, int taskIdx, long inactivityInterval, long rolloverInterval) throws Exception {

	BucketingSink<String> sink = new BucketingSink<String>(outDir.getAbsolutePath())
		.setBucketer(new Bucketer<String>() {
			private static final long serialVersionUID = 1L;

			@Override
			public Path getBucketPath(Clock clock, Path basePath, String element) {
				return new Path(basePath, element);
			}
		})
		.setWriter(new StringWriter<String>())
		.setInactiveBucketCheckInterval(inactivityInterval)
		.setInactiveBucketThreshold(inactivityInterval)
		.setPartPrefix(PART_PREFIX)
		.setInProgressPrefix("")
		.setPendingPrefix("")
		.setValidLengthPrefix("")
		.setInProgressSuffix(IN_PROGRESS_SUFFIX)
		.setPendingSuffix(PENDING_SUFFIX)
		.setValidLengthSuffix(VALID_LENGTH_SUFFIX)
		.setBatchRolloverInterval(rolloverInterval);

	return createTestSink(sink, totalParallelism, taskIdx);
}
Example #13
Source File: BucketingSinkTestProgram.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public Path getBucketPath(Clock clock, Path basePath, Tuple4<Integer, Long, Integer, String> element) {
	return basePath.suffix(String.valueOf(element.f0));
}
Example #14
Source File: DateTimeBucketer.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public Path getBucketPath(Clock clock, Path basePath, T element) {
	String newDateTimeString = dateTimeFormatter.format(Instant.ofEpochMilli(clock.currentTimeMillis()));
	return new Path(basePath + "/" + newDateTimeString);
}
Example #15
Source File: BasePathBucketer.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public Path getBucketPath(Clock clock, Path basePath, T element) {
	return basePath;
}
Example #16
Source File: BucketingSinkTestProgram.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public Path getBucketPath(Clock clock, Path basePath, Tuple4<Integer, Long, Integer, String> element) {
	return basePath.suffix(String.valueOf(element.f0));
}
Example #17
Source File: DateTimeBucketer.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public Path getBucketPath(Clock clock, Path basePath, T element) {
	String newDateTimeString = dateTimeFormatter.format(Instant.ofEpochMilli(clock.currentTimeMillis()));
	return new Path(basePath + "/" + newDateTimeString);
}
Example #18
Source File: BasePathBucketer.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public Path getBucketPath(Clock clock, Path basePath, T element) {
	return basePath;
}
Example #19
Source File: BucketingSinkTestProgram.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Override
public Path getBucketPath(Clock clock, Path basePath, Tuple4<Integer, Long, Integer, String> element) {
	return basePath.suffix(String.valueOf(element.f0));
}
Example #20
Source File: DateTimeBucketer.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Override
public Path getBucketPath(Clock clock, Path basePath, T element) {
	String newDateTimeString = dateTimeFormatter.format(Instant.ofEpochMilli(clock.currentTimeMillis()));
	return new Path(basePath + "/" + newDateTimeString);
}
Example #21
Source File: BasePathBucketer.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Override
public Path getBucketPath(Clock clock, Path basePath, T element) {
	return basePath;
}
Example #22
Source File: Bucketer.java    From flink with Apache License 2.0 2 votes vote down vote up 
/**
 * Returns the {@link Path} of a bucket file.
 *
 * @param clock The current system time in milliseconds.
 * @param basePath The base path containing all the buckets.
 * @param element The current element being processed.
 *
 * @return The complete {@code Path} of the bucket which the provided element should fall in. This
 * should include the {@code basePath} and also the {@code subtaskIndex} to avoid clashes with
 * parallel sinks.
 */
Path getBucketPath(Clock clock, Path basePath, T element);
Example #23
Source File: Bucketer.java    From flink with Apache License 2.0 2 votes vote down vote up 
/**
 * Returns the {@link Path} of a bucket file.
 *
 * @param clock The current system time in milliseconds.
 * @param basePath The base path containing all the buckets.
 * @param element The current element being processed.
 *
 * @return The complete {@code Path} of the bucket which the provided element should fall in. This
 * should include the {@code basePath} and also the {@code subtaskIndex} to avoid clashes with
 * parallel sinks.
 */
Path getBucketPath(Clock clock, Path basePath, T element);
Example #24
Source File: Bucketer.java    From Flink-CEPplus with Apache License 2.0 2 votes vote down vote up 
/**
 * Returns the {@link Path} of a bucket file.
 *
 * @param clock The current system time in milliseconds.
 * @param basePath The base path containing all the buckets.
 * @param element The current element being processed.
 *
 * @return The complete {@code Path} of the bucket which the provided element should fall in. This
 * should include the {@code basePath} and also the {@code subtaskIndex} to avoid clashes with
 * parallel sinks.
 */
Path getBucketPath(Clock clock, Path basePath, T element);