Java Code Examples for org.apache.flink.streaming.api.watermark.Watermark

The following examples show how to use org.apache.flink.streaming.api.watermark.Watermark. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: AbstractFetcher.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Record emission, if a timestamp will be attached from an assigner that is
 * also a punctuated watermark generator.
 */
private void emitRecordWithTimestampAndPunctuatedWatermark(
		T record, KafkaTopicPartitionState<KPH> partitionState, long offset, long kafkaEventTimestamp) {
	@SuppressWarnings("unchecked")
	final KafkaTopicPartitionStateWithPunctuatedWatermarks<T, KPH> withWatermarksState =
			(KafkaTopicPartitionStateWithPunctuatedWatermarks<T, KPH>) partitionState;

	// only one thread ever works on accessing timestamps and watermarks
	// from the punctuated extractor
	final long timestamp = withWatermarksState.getTimestampForRecord(record, kafkaEventTimestamp);
	final Watermark newWatermark = withWatermarksState.checkAndGetNewWatermark(record, timestamp);

	// emit the record with timestamp, using the usual checkpoint lock to guarantee
	// atomicity of record emission and offset state update
	synchronized (checkpointLock) {
		sourceContext.collectWithTimestamp(record, timestamp);
		partitionState.setOffset(offset);
	}

	// if we also have a new per-partition watermark, check if that is also a
	// new cross-partition watermark
	if (newWatermark != null) {
		updateMinPunctuatedWatermark(newWatermark);
	}
}
 
Example 2
Source Project: flink   Source File: AbstractFetcher.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Record emission, if a timestamp will be attached from an assigner that is
 * also a punctuated watermark generator.
 */
private void emitRecordWithTimestampAndPunctuatedWatermark(
		T record, KafkaTopicPartitionState<KPH> partitionState, long offset, long kafkaEventTimestamp) {
	@SuppressWarnings("unchecked")
	final KafkaTopicPartitionStateWithPunctuatedWatermarks<T, KPH> withWatermarksState =
			(KafkaTopicPartitionStateWithPunctuatedWatermarks<T, KPH>) partitionState;

	// only one thread ever works on accessing timestamps and watermarks
	// from the punctuated extractor
	final long timestamp = withWatermarksState.getTimestampForRecord(record, kafkaEventTimestamp);
	final Watermark newWatermark = withWatermarksState.checkAndGetNewWatermark(record, timestamp);

	// emit the record with timestamp, using the usual checkpoint lock to guarantee
	// atomicity of record emission and offset state update
	synchronized (checkpointLock) {
		sourceContext.collectWithTimestamp(record, timestamp);
		partitionState.setOffset(offset);
	}

	// if we also have a new per-partition watermark, check if that is also a
	// new cross-partition watermark
	if (newWatermark != null) {
		updateMinPunctuatedWatermark(newWatermark);
	}
}
 
Example 3
Source Project: flink   Source File: WindowOperatorMigrationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public int compare(Object o1, Object o2) {
	if (o1 instanceof Watermark || o2 instanceof Watermark) {
		return 0;
	} else {
		StreamRecord<Tuple2<K, Integer>> sr0 = (StreamRecord<Tuple2<K, Integer>>) o1;
		StreamRecord<Tuple2<K, Integer>> sr1 = (StreamRecord<Tuple2<K, Integer>>) o2;
		if (sr0.getTimestamp() != sr1.getTimestamp()) {
			return (int) (sr0.getTimestamp() - sr1.getTimestamp());
		}
		int comparison = sr0.getValue().f0.compareTo(sr1.getValue().f0);
		if (comparison != 0) {
			return comparison;
		} else {
			return sr0.getValue().f1 - sr1.getValue().f1;
		}
	}
}
 
Example 4
Source Project: beam   Source File: ImpulseSourceFunctionTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test(timeout = 10_000)
public void testImpulseRestored() throws Exception {
  ImpulseSourceFunction source = new ImpulseSourceFunction(0);
  // Previous state available
  ListState<Object> mockListState = getMockListState(Collections.singletonList(true));
  source.initializeState(getInitializationContext(mockListState));

  // 1) Should finish
  source.run(sourceContext);
  // 2) Should keep checkpoint state
  verify(mockListState).get();
  verifyNoMoreInteractions(mockListState);
  // 3) Should always emit the final watermark
  verify(sourceContext).emitWatermark(Watermark.MAX_WATERMARK);
  // 4) Should _not_ emit impulse element
  verifyNoMoreInteractions(sourceContext);
}
 
Example 5
Source Project: flink   Source File: StatusWatermarkValve.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Feed a {@link Watermark} into the valve. If the input triggers the valve to output a new Watermark,
 * {@link ValveOutputHandler#handleWatermark(Watermark)} will be called to process the new Watermark.
 *
 * @param watermark the watermark to feed to the valve
 * @param channelIndex the index of the channel that the fed watermark belongs to (index starting from 0)
 */
public void inputWatermark(Watermark watermark, int channelIndex) {
	// ignore the input watermark if its input channel, or all input channels are idle (i.e. overall the valve is idle).
	if (lastOutputStreamStatus.isActive() && channelStatuses[channelIndex].streamStatus.isActive()) {
		long watermarkMillis = watermark.getTimestamp();

		// if the input watermark's value is less than the last received watermark for its input channel, ignore it also.
		if (watermarkMillis > channelStatuses[channelIndex].watermark) {
			channelStatuses[channelIndex].watermark = watermarkMillis;

			// previously unaligned input channels are now aligned if its watermark has caught up
			if (!channelStatuses[channelIndex].isWatermarkAligned && watermarkMillis >= lastOutputWatermark) {
				channelStatuses[channelIndex].isWatermarkAligned = true;
			}

			// now, attempt to find a new min watermark across all aligned channels
			findAndOutputNewMinWatermarkAcrossAlignedChannels();
		}
	}
}
 
Example 6
Source Project: Flink-CEPplus   Source File: MigrationTestUtils.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void run(SourceContext<Tuple2<Long, Long>> ctx) throws Exception {

	ctx.emitWatermark(new Watermark(0));

	synchronized (ctx.getCheckpointLock()) {
		for (long i = 0; i < numElements; i++) {
			if (i % getRuntimeContext().getNumberOfParallelSubtasks() == getRuntimeContext().getIndexOfThisSubtask()) {
				ctx.collect(new Tuple2<>(i, i));
			}
		}
	}

	// don't emit a final watermark so that we don't trigger the registered event-time
	// timers
	while (isRunning) {
		Thread.sleep(20);
	}
}
 
Example 7
Source Project: flink   Source File: StreamElementQueueTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testPut() {
	StreamElementQueue<Integer> queue = createStreamElementQueue(2);

	Watermark watermark = new Watermark(0L);
	StreamRecord<Integer> streamRecord = new StreamRecord<>(42, 1L);

	// add two elements to reach capacity
	assertTrue(queue.tryPut(watermark).isPresent());
	assertTrue(queue.tryPut(streamRecord).isPresent());

	assertEquals(2, queue.size());

	// queue full, cannot add new element
	assertFalse(queue.tryPut(new Watermark(2L)).isPresent());

	// check if expected values are returned (for checkpointing)
	assertEquals(Arrays.asList(watermark, streamRecord), queue.values());
}
 
Example 8
@Override
 public void processWatermark(Watermark watermark) throws Exception {
   // we'll keep state forever in the operator

/*	StreamRecord<Tuple3<String, Long, Long>> result = new StreamRecord<>(null, -1);

	Iterator<Map.Entry<String, Map<Long, CountAndAccessTime>>> iterator = windows.entrySet().iterator();
	while (iterator.hasNext()) {
		Map.Entry<String, Map<Long, CountAndAccessTime>> campaignWindows = iterator.next();
		for(Map.Entry<Long, CountAndAccessTime> window: campaignWindows.getValue().entrySet()) {
			if(window.getKey() < watermark.getTimestamp() && window.getKey() >= lastWatermark) {
				// emit window
				Tuple3<String, Long, Long> resultTuple = Tuple3.of(campaignWindows.getKey(), window.getKey(), window.getValue().count);
				output.collect(result.replace(resultTuple));
			}
		}
	}
	lastWatermark = watermark.getTimestamp(); **/
 }
 
Example 9
Source Project: flink   Source File: StreamElementQueueTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testPop() {
	StreamElementQueue<Integer> queue = createStreamElementQueue(2);

	// add two elements to reach capacity
	putSuccessfully(queue, new Watermark(0L));
	ResultFuture<Integer> recordResult = putSuccessfully(queue, new StreamRecord<>(42, 1L));

	assertEquals(2, queue.size());

	// remove completed elements (watermarks are always completed)
	assertEquals(Arrays.asList(new Watermark(0L)), popCompleted(queue));
	assertEquals(1, queue.size());

	// now complete the stream record
	recordResult.complete(Collections.singleton(43));

	assertEquals(Arrays.asList(new StreamRecord<>(43, 1L)), popCompleted(queue));
	assertEquals(0, queue.size());
	assertTrue(queue.isEmpty());
}
 
Example 10
Source Project: flink   Source File: StreamIterationHead.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public void init() {
	// offer the queue for the tail
	BlockingQueueBroker.INSTANCE.handIn(brokerID, dataChannel);
	LOG.info("Iteration head {} added feedback queue under {}", getName(), brokerID);

	this.streamOutputs = (RecordWriterOutput<OUT>[]) getStreamOutputs();

	// If timestamps are enabled we make sure to remove cyclic watermark dependencies
	if (isSerializingTimestamps()) {
		for (RecordWriterOutput<OUT> output : streamOutputs) {
			output.emitWatermark(new Watermark(Long.MAX_VALUE));
		}
	}
}
 
Example 11
Source Project: Flink-CEPplus   Source File: StatusWatermarkValveTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that for a multiple input valve, decreasing watermarks will yield no output.
 */
@Test
public void testMultipleInputDecreasingWatermarksYieldsNoOutput() {
	BufferedValveOutputHandler valveOutput = new BufferedValveOutputHandler();
	StatusWatermarkValve valve = new StatusWatermarkValve(3, valveOutput);

	valve.inputWatermark(new Watermark(25), 0);
	valve.inputWatermark(new Watermark(10), 1);
	valve.inputWatermark(new Watermark(17), 2);
	assertEquals(new Watermark(10), valveOutput.popLastSeenOutput());

	valve.inputWatermark(new Watermark(12), 0);
	valve.inputWatermark(new Watermark(8), 1);
	valve.inputWatermark(new Watermark(15), 2);
	assertEquals(null, valveOutput.popLastSeenOutput());
}
 
Example 12
@Override
public void run(SourceContext<Tuple2<Long, Long>> ctx) throws Exception {
	getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);

	// immediately trigger any set timers
	ctx.emitWatermark(new Watermark(1000));

	synchronized (ctx.getCheckpointLock()) {
		for (long i = 0; i < numElements; i++) {
			ctx.collect(new Tuple2<>(i, i));
		}
	}

	while (isRunning) {
		Thread.sleep(20);
	}
}
 
Example 13
Source Project: flink   Source File: StreamSourceContexts.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void emitWatermark(Watermark mark) {
	if (allowWatermark(mark)) {
		synchronized (checkpointLock) {
			streamStatusMaintainer.toggleStreamStatus(StreamStatus.ACTIVE);

			if (nextCheck != null) {
				this.failOnNextCheck = false;
			} else {
				scheduleNextIdleDetectionTask();
			}

			processAndEmitWatermark(mark);
		}
	}
}
 
Example 14
Source Project: flink   Source File: WatermarkAssignerOperatorTestBase.java    License: Apache License 2.0 5 votes vote down vote up
protected List<Watermark> extractWatermarks(Collection<Object> collection) {
	List<Watermark> watermarks = new ArrayList<>();
	for (Object obj : collection) {
		if (obj instanceof Watermark) {
			watermarks.add((Watermark) obj);
		}
	}
	return watermarks;
}
 
Example 15
Source Project: flink   Source File: KafkaShuffleFetcher.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void partitionConsumerRecordsHandler(
		List<ConsumerRecord<byte[], byte[]>> partitionRecords,
		KafkaTopicPartitionState<T, TopicPartition> partition) throws Exception {

	for (ConsumerRecord<byte[], byte[]> record : partitionRecords) {
		final KafkaShuffleElement element = kafkaShuffleDeserializer.deserialize(record);

		// TODO: Do we need to check the end of stream if reaching the end watermark
		// TODO: Currently, if one of the partition sends an end-of-stream signal the fetcher stops running.
		// The current "ending of stream" logic in KafkaFetcher a bit strange: if any partition has a record
		// signaled as "END_OF_STREAM", the fetcher will stop running. Notice that the signal is coming from
		// the deserializer, which means from Kafka data itself. But it is possible that other topics
		// and partitions still have data to read. Finishing reading Partition0 can not guarantee that Partition1
		// also finishes.
		if (element.isRecord()) {
			// timestamp is inherent from upstream
			// If using ProcessTime, timestamp is going to be ignored (upstream does not include timestamp as well)
			// If using IngestionTime, timestamp is going to be overwritten
			// If using EventTime, timestamp is going to be used
			synchronized (checkpointLock) {
				KafkaShuffleRecord<T> elementAsRecord = element.asRecord();
				sourceContext.collectWithTimestamp(
					elementAsRecord.value,
					elementAsRecord.timestamp == null ? record.timestamp() : elementAsRecord.timestamp);
				partition.setOffset(record.offset());
			}
		} else if (element.isWatermark()) {
			final KafkaShuffleWatermark watermark = element.asWatermark();
			Optional<Watermark> newWatermark = watermarkHandler.checkAndGetNewWatermark(watermark);
			newWatermark.ifPresent(sourceContext::emitWatermark);
		}
	}
}
 
Example 16
Source Project: flink   Source File: TimestampITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void run(SourceContext<Integer> ctx) throws Exception {
	for (int i = 0; i < numWatermarks; i++) {
		ctx.collectWithTimestamp(i, initialTime + i);
		ctx.emitWatermark(new Watermark(initialTime + i));
	}
}
 
Example 17
@Override
public void processWatermark(Watermark mark) throws Exception {
	Optional<InternalTimeServiceManager<?>> timeServiceManager = getTimeServiceManager();
	if (timeServiceManager.isPresent()) {
		timeServiceManager.get().advanceWatermark(mark);
	}
	emitter.accept(mark);
}
 
Example 18
@Override
public void processElement(StreamRecord<T> element) throws Exception {
	final T value = element.getValue();
	final long newTimestamp = userFunction.extractTimestamp(value,
			element.hasTimestamp() ? element.getTimestamp() : Long.MIN_VALUE);

	output.collect(element.replace(element.getValue(), newTimestamp));

	final Watermark nextWatermark = userFunction.checkAndGetNextWatermark(value, newTimestamp);
	if (nextWatermark != null && nextWatermark.getTimestamp() > currentWatermark) {
		currentWatermark = nextWatermark.getTimestamp();
		output.emitWatermark(nextWatermark);
	}
}
 
Example 19
Source Project: Flink-CEPplus   Source File: StreamFilterTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testFilter() throws Exception {
	StreamFilter<Integer> operator = new StreamFilter<Integer>(new MyFilter());

	OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<Integer, Integer>(operator);

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();

	testHarness.open();

	testHarness.processElement(new StreamRecord<Integer>(1, initialTime + 1));
	testHarness.processElement(new StreamRecord<Integer>(2, initialTime + 2));
	testHarness.processWatermark(new Watermark(initialTime + 2));
	testHarness.processElement(new StreamRecord<Integer>(3, initialTime + 3));
	testHarness.processElement(new StreamRecord<Integer>(4, initialTime + 4));
	testHarness.processElement(new StreamRecord<Integer>(5, initialTime + 5));
	testHarness.processElement(new StreamRecord<Integer>(6, initialTime + 6));
	testHarness.processElement(new StreamRecord<Integer>(7, initialTime + 7));

	expectedOutput.add(new StreamRecord<Integer>(2, initialTime + 2));
	expectedOutput.add(new Watermark(initialTime + 2));
	expectedOutput.add(new StreamRecord<Integer>(4, initialTime + 4));
	expectedOutput.add(new StreamRecord<Integer>(6, initialTime + 6));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}
 
Example 20
Source Project: flink   Source File: CoProcessOperatorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testTimestampAndWatermarkQuerying() throws Exception {

	CoProcessOperator<Integer, String, String> operator =
			new CoProcessOperator<>(new WatermarkQueryingProcessFunction());

	TwoInputStreamOperatorTestHarness<Integer, String, String> testHarness =
			new TwoInputStreamOperatorTestHarness<>(operator);

	testHarness.setup();
	testHarness.open();

	testHarness.processWatermark1(new Watermark(17));
	testHarness.processWatermark2(new Watermark(17));
	testHarness.processElement1(new StreamRecord<>(5, 12L));

	testHarness.processWatermark1(new Watermark(42));
	testHarness.processWatermark2(new Watermark(42));
	testHarness.processElement2(new StreamRecord<>("6", 13L));

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new Watermark(17L));
	expectedOutput.add(new StreamRecord<>("5WM:17 TS:12", 12L));
	expectedOutput.add(new Watermark(42L));
	expectedOutput.add(new StreamRecord<>("6WM:42 TS:13", 13L));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}
 
Example 21
Source Project: flink   Source File: CoBroadcastWithKeyedOperatorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSideOutput() throws Exception {
	try (
			TwoInputStreamOperatorTestHarness<String, Integer, String> testHarness = getInitializedTestHarness(
					BasicTypeInfo.STRING_TYPE_INFO,
					new IdentityKeySelector<>(),
					new FunctionWithSideOutput())
	) {

		testHarness.processWatermark1(new Watermark(10L));
		testHarness.processWatermark2(new Watermark(10L));
		testHarness.processElement2(new StreamRecord<>(5, 12L));

		testHarness.processWatermark1(new Watermark(40L));
		testHarness.processWatermark2(new Watermark(40L));
		testHarness.processElement1(new StreamRecord<>("6", 13L));
		testHarness.processElement1(new StreamRecord<>("6", 15L));

		testHarness.processWatermark1(new Watermark(50L));
		testHarness.processWatermark2(new Watermark(50L));

		Queue<StreamRecord<String>> expectedBr = new ConcurrentLinkedQueue<>();
		expectedBr.add(new StreamRecord<>("BR:5 WM:10 TS:12", 12L));

		Queue<StreamRecord<String>> expectedNonBr = new ConcurrentLinkedQueue<>();
		expectedNonBr.add(new StreamRecord<>("NON-BR:6 WM:40 TS:13", 13L));
		expectedNonBr.add(new StreamRecord<>("NON-BR:6 WM:40 TS:15", 15L));

		TestHarnessUtil.assertOutputEquals(
				"Wrong Side Output",
				expectedBr,
				testHarness.getSideOutput(FunctionWithSideOutput.BROADCAST_TAG));

		TestHarnessUtil.assertOutputEquals(
				"Wrong Side Output",
				expectedNonBr,
				testHarness.getSideOutput(FunctionWithSideOutput.NON_BROADCAST_TAG));
	}
}
 
Example 22
Source Project: flink   Source File: WatermarkAssignerOperatorTestBase.java    License: Apache License 2.0 5 votes vote down vote up
protected List<Watermark> extractWatermarks(Collection<Object> collection) {
	List<Watermark> watermarks = new ArrayList<>();
	for (Object obj : collection) {
		if (obj instanceof Watermark) {
			watermarks.add((Watermark) obj);
		}
	}
	return watermarks;
}
 
Example 23
Source Project: flink   Source File: IngestionTimeExtractor.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Watermark getCurrentWatermark() {
	// make sure timestamps are monotonously increasing, even when the system clock re-syncs
	final long now = Math.max(System.currentTimeMillis(), maxTimestamp);
	maxTimestamp = now;
	return new Watermark(now - 1);
}
 
Example 24
Source Project: flink   Source File: TimestampITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void processWatermark(Watermark mark) throws Exception {
	super.processWatermark(mark);

	for (Watermark previousMark: watermarks) {
		assertTrue(previousMark.getTimestamp() < mark.getTimestamp());
	}
	watermarks.add(mark);
	latch.trigger();
	output.emitWatermark(mark);
}
 
Example 25
Source Project: flink   Source File: DirectedOutput.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void emitWatermark(Watermark mark) {
	watermarkGauge.setCurrentWatermark(mark.getTimestamp());
	for (Output<StreamRecord<OUT>> out : allOutputs) {
		out.emitWatermark(mark);
	}
}
 
Example 26
Source Project: flink   Source File: LegacyKeyedCoProcessOperatorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testEventTimeTimers() throws Exception {

	LegacyKeyedCoProcessOperator<String, Integer, String, String> operator =
			new LegacyKeyedCoProcessOperator<>(new EventTimeTriggeringProcessFunction());

	TwoInputStreamOperatorTestHarness<Integer, String, String> testHarness =
			new KeyedTwoInputStreamOperatorTestHarness<>(
					operator,
					new IntToStringKeySelector<>(),
					new IdentityKeySelector<String>(),
					BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.processElement1(new StreamRecord<>(17, 42L));
	testHarness.processElement2(new StreamRecord<>("18", 42L));

	testHarness.processWatermark1(new Watermark(5));
	testHarness.processWatermark2(new Watermark(5));

	testHarness.processWatermark1(new Watermark(6));
	testHarness.processWatermark2(new Watermark(6));

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new StreamRecord<>("INPUT1:17", 42L));
	expectedOutput.add(new StreamRecord<>("INPUT2:18", 42L));
	expectedOutput.add(new StreamRecord<>("1777", 5L));
	expectedOutput.add(new Watermark(5L));
	expectedOutput.add(new StreamRecord<>("1777", 6L));
	expectedOutput.add(new Watermark(6L));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}
 
Example 27
@Test
public void testBroadcastState() throws Exception {

	final Set<String> keysToRegister = new HashSet<>();
	keysToRegister.add("test1");
	keysToRegister.add("test2");
	keysToRegister.add("test3");

	try (
			TwoInputStreamOperatorTestHarness<String, Integer, String> testHarness = getInitializedTestHarness(
					new TestFunction(keysToRegister), STATE_DESCRIPTOR)
	) {
		testHarness.processWatermark1(new Watermark(10L));
		testHarness.processWatermark2(new Watermark(10L));
		testHarness.processElement2(new StreamRecord<>(5, 12L));

		testHarness.processWatermark1(new Watermark(40L));
		testHarness.processWatermark2(new Watermark(40L));
		testHarness.processElement1(new StreamRecord<>("6", 13L));
		testHarness.processElement1(new StreamRecord<>("6", 15L));

		testHarness.processWatermark1(new Watermark(50L));
		testHarness.processWatermark2(new Watermark(50L));

		Queue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

		expectedOutput.add(new Watermark(10L));
		expectedOutput.add(new StreamRecord<>("5WM:10 TS:12", 12L));
		expectedOutput.add(new Watermark(40L));
		expectedOutput.add(new StreamRecord<>("6WM:40 TS:13", 13L));
		expectedOutput.add(new StreamRecord<>("6WM:40 TS:15", 15L));
		expectedOutput.add(new Watermark(50L));

		TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
	}
}
 
Example 28
Source Project: flink   Source File: StreamProjectTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testProject() throws Exception {

	TypeInformation<Tuple5<Integer, String, Integer, String, Integer>> inType = TypeExtractor
			.getForObject(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "b", 4));

	int[] fields = new int[]{4, 4, 3};

	TupleSerializer<Tuple3<Integer, Integer, String>> serializer =
			new TupleTypeInfo<Tuple3<Integer, Integer, String>>(StreamProjection.extractFieldTypes(fields, inType))
					.createSerializer(new ExecutionConfig());
	@SuppressWarnings("unchecked")
	StreamProject<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>> operator =
			new StreamProject<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>>(
					fields, serializer);

	OneInputStreamOperatorTestHarness<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>> testHarness = new OneInputStreamOperatorTestHarness<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>>(operator);

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();

	testHarness.open();

	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "b", 4), initialTime + 1));
	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "s", 3, "c", 2), initialTime + 2));
	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "c", 2), initialTime + 3));
	testHarness.processWatermark(new Watermark(initialTime + 2));
	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "a", 7), initialTime + 4));

	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(4, 4, "b"), initialTime + 1));
	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(2, 2, "c"), initialTime + 2));
	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(2, 2, "c"), initialTime + 3));
	expectedOutput.add(new Watermark(initialTime + 2));
	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(7, 7, "a"), initialTime + 4));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}
 
Example 29
Source Project: Flink-CEPplus   Source File: TimestampITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void run(SourceContext<Integer> ctx) throws Exception {
	for (int i = 0; i < numWatermarks; i++) {
		ctx.collectWithTimestamp(i, initialTime + i);
		ctx.emitWatermark(new Watermark(initialTime + i));
	}
}
 
Example 30
@Override
public final Watermark getCurrentWatermark() {
	// this guarantees that the watermark never goes backwards.
	long potentialWM = currentMaxTimestamp - maxOutOfOrderness;
	if (potentialWM >= lastEmittedWatermark) {
		lastEmittedWatermark = potentialWM;
	}
	return new Watermark(lastEmittedWatermark);
}