Java Code Examples for org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness#setProcessingTime()

The following examples show how to use org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness#setProcessingTime() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: AbstractStreamOperatorTest.java From Flink-CEPplus with Apache License 2.0

5 votes

/**
 * Verify that timers for the different time domains don't clash.
 */
@Test
public void testProcessingTimeAndEventTimeDontInterfere() throws Exception {
	TestOperator testOperator = new TestOperator();

	KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, String> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(testOperator, new TestKeySelector(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.open();

	testHarness.setProcessingTime(0L);
	testHarness.processWatermark(0L);

	testHarness.processElement(new Tuple2<>(0, "SET_PROC_TIME_TIMER:10"), 0);
	testHarness.processElement(new Tuple2<>(0, "SET_EVENT_TIME_TIMER:20"), 0);

	testHarness.processElement(new Tuple2<>(0, "SET_STATE:HELLO"), 0);

	testHarness.processWatermark(20L);

	assertThat(
			extractResult(testHarness),
			contains("ON_EVENT_TIME:HELLO"));

	testHarness.setProcessingTime(10L);

	assertThat(
			extractResult(testHarness),
			contains("ON_PROC_TIME:HELLO"));
}

Example 2

Source File: WindowTranslationTest.java From Flink-CEPplus with Apache License 2.0

5 votes

/**
 * Ensure that we get some output from the given operator when pushing in an element and
 * setting watermark and processing time to {@code Long.MAX_VALUE}.
 */
private static <K, IN, OUT> void processElementAndEnsureOutput(
		OneInputStreamOperator<IN, OUT> operator,
		KeySelector<IN, K> keySelector,
		TypeInformation<K> keyType,
		IN element) throws Exception {

	KeyedOneInputStreamOperatorTestHarness<K, IN, OUT> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(
					operator,
					keySelector,
					keyType);

	if (operator instanceof OutputTypeConfigurable) {
		// use a dummy type since window functions just need the ExecutionConfig
		// this is also only needed for Fold, which we're getting rid off soon.
		((OutputTypeConfigurable) operator).setOutputType(BasicTypeInfo.STRING_TYPE_INFO, new ExecutionConfig());
	}

	testHarness.open();

	testHarness.setProcessingTime(0);
	testHarness.processWatermark(Long.MIN_VALUE);

	testHarness.processElement(new StreamRecord<>(element, 0));

	// provoke any processing-time/event-time triggers
	testHarness.setProcessingTime(Long.MAX_VALUE);
	testHarness.processWatermark(Long.MAX_VALUE);

	// we at least get the two watermarks and should also see an output element
	assertTrue(testHarness.getOutput().size() >= 3);

	testHarness.close();
}

Example 3

Source File: AllWindowTranslationTest.java From Flink-CEPplus with Apache License 2.0

5 votes

/**
 * Ensure that we get some output from the given operator when pushing in an element and
 * setting watermark and processing time to {@code Long.MAX_VALUE}.
 */
private static <K, IN, OUT> void processElementAndEnsureOutput(
		OneInputStreamOperator<IN, OUT> operator,
		KeySelector<IN, K> keySelector,
		TypeInformation<K> keyType,
		IN element) throws Exception {

	KeyedOneInputStreamOperatorTestHarness<K, IN, OUT> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(
					operator,
					keySelector,
					keyType);

	testHarness.open();

	testHarness.setProcessingTime(0);
	testHarness.processWatermark(Long.MIN_VALUE);

	testHarness.processElement(new StreamRecord<>(element, 0));

	// provoke any processing-time/event-time triggers
	testHarness.setProcessingTime(Long.MAX_VALUE);
	testHarness.processWatermark(Long.MAX_VALUE);

	// we at least get the two watermarks and should also see an output element
	assertTrue(testHarness.getOutput().size() >= 3);

	testHarness.close();
}

Example 4

Source File: AbstractStreamOperatorTest.java From flink with Apache License 2.0

5 votes

/**
 * Verify that timers for the different time domains don't clash.
 */
@Test
public void testProcessingTimeAndEventTimeDontInterfere() throws Exception {
	TestOperator testOperator = new TestOperator();

	KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, String> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(testOperator, new TestKeySelector(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.open();

	testHarness.setProcessingTime(0L);
	testHarness.processWatermark(0L);

	testHarness.processElement(new Tuple2<>(0, "SET_PROC_TIME_TIMER:10"), 0);
	testHarness.processElement(new Tuple2<>(0, "SET_EVENT_TIME_TIMER:20"), 0);

	testHarness.processElement(new Tuple2<>(0, "SET_STATE:HELLO"), 0);

	testHarness.processWatermark(20L);

	assertThat(
			extractResult(testHarness),
			contains("ON_EVENT_TIME:HELLO"));

	testHarness.setProcessingTime(10L);

	assertThat(
			extractResult(testHarness),
			contains("ON_PROC_TIME:HELLO"));
}

Example 5

Source File: WindowTranslationTest.java From flink with Apache License 2.0

5 votes

/**
 * Ensure that we get some output from the given operator when pushing in an element and
 * setting watermark and processing time to {@code Long.MAX_VALUE}.
 */
private static <K, IN, OUT> void processElementAndEnsureOutput(
		OneInputStreamOperator<IN, OUT> operator,
		KeySelector<IN, K> keySelector,
		TypeInformation<K> keyType,
		IN element) throws Exception {

	KeyedOneInputStreamOperatorTestHarness<K, IN, OUT> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(
					operator,
					keySelector,
					keyType);

	if (operator instanceof OutputTypeConfigurable) {
		// use a dummy type since window functions just need the ExecutionConfig
		// this is also only needed for Fold, which we're getting rid off soon.
		((OutputTypeConfigurable) operator).setOutputType(BasicTypeInfo.STRING_TYPE_INFO, new ExecutionConfig());
	}

	testHarness.open();

	testHarness.setProcessingTime(0);
	testHarness.processWatermark(Long.MIN_VALUE);

	testHarness.processElement(new StreamRecord<>(element, 0));

	// provoke any processing-time/event-time triggers
	testHarness.setProcessingTime(Long.MAX_VALUE);
	testHarness.processWatermark(Long.MAX_VALUE);

	// we at least get the two watermarks and should also see an output element
	assertTrue(testHarness.getOutput().size() >= 3);

	testHarness.close();
}

Example 6

Source File: AllWindowTranslationTest.java From flink with Apache License 2.0

5 votes

/**
 * Ensure that we get some output from the given operator when pushing in an element and
 * setting watermark and processing time to {@code Long.MAX_VALUE}.
 */
private static <K, IN, OUT> void processElementAndEnsureOutput(
		OneInputStreamOperator<IN, OUT> operator,
		KeySelector<IN, K> keySelector,
		TypeInformation<K> keyType,
		IN element) throws Exception {

	KeyedOneInputStreamOperatorTestHarness<K, IN, OUT> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(
					operator,
					keySelector,
					keyType);

	testHarness.open();

	testHarness.setProcessingTime(0);
	testHarness.processWatermark(Long.MIN_VALUE);

	testHarness.processElement(new StreamRecord<>(element, 0));

	// provoke any processing-time/event-time triggers
	testHarness.setProcessingTime(Long.MAX_VALUE);
	testHarness.processWatermark(Long.MAX_VALUE);

	// we at least get the two watermarks and should also see an output element
	assertTrue(testHarness.getOutput().size() >= 3);

	testHarness.close();
}

Example 7

Source File: WindowTranslationTest.java From flink with Apache License 2.0

5 votes

/**
 * Ensure that we get some output from the given operator when pushing in an element and
 * setting watermark and processing time to {@code Long.MAX_VALUE}.
 */
private static <K, IN, OUT> void processElementAndEnsureOutput(
		OneInputStreamOperator<IN, OUT> operator,
		KeySelector<IN, K> keySelector,
		TypeInformation<K> keyType,
		IN element) throws Exception {

	KeyedOneInputStreamOperatorTestHarness<K, IN, OUT> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(
					operator,
					keySelector,
					keyType);

	if (operator instanceof OutputTypeConfigurable) {
		// use a dummy type since window functions just need the ExecutionConfig
		// this is also only needed for Fold, which we're getting rid off soon.
		((OutputTypeConfigurable) operator).setOutputType(BasicTypeInfo.STRING_TYPE_INFO, new ExecutionConfig());
	}

	testHarness.open();

	testHarness.setProcessingTime(0);
	testHarness.processWatermark(Long.MIN_VALUE);

	testHarness.processElement(new StreamRecord<>(element, 0));

	// provoke any processing-time/event-time triggers
	testHarness.setProcessingTime(Long.MAX_VALUE);
	testHarness.processWatermark(Long.MAX_VALUE);

	// we at least get the two watermarks and should also see an output element
	assertTrue(testHarness.getOutput().size() >= 3);

	testHarness.close();
}

Example 8

Source File: AllWindowTranslationTest.java From flink with Apache License 2.0

5 votes

/**
 * Ensure that we get some output from the given operator when pushing in an element and
 * setting watermark and processing time to {@code Long.MAX_VALUE}.
 */
private static <K, IN, OUT> void processElementAndEnsureOutput(
		OneInputStreamOperator<IN, OUT> operator,
		KeySelector<IN, K> keySelector,
		TypeInformation<K> keyType,
		IN element) throws Exception {

	KeyedOneInputStreamOperatorTestHarness<K, IN, OUT> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(
					operator,
					keySelector,
					keyType);

	testHarness.open();

	testHarness.setProcessingTime(0);
	testHarness.processWatermark(Long.MIN_VALUE);

	testHarness.processElement(new StreamRecord<>(element, 0));

	// provoke any processing-time/event-time triggers
	testHarness.setProcessingTime(Long.MAX_VALUE);
	testHarness.processWatermark(Long.MAX_VALUE);

	// we at least get the two watermarks and should also see an output element
	assertTrue(testHarness.getOutput().size() >= 3);

	testHarness.close();
}

Example 9

Source File: AbstractStreamOperatorTest.java From Flink-CEPplus with Apache License 2.0

4 votes

/**
 * Verify that a low-level timer is set for processing-time timers in case of restore.
 */
@Test
public void testEnsureProcessingTimeTimerRegisteredOnRestore() throws Exception {
	TestOperator testOperator = new TestOperator();

	KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, String> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(testOperator, new TestKeySelector(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.open();

	testHarness.setProcessingTime(0L);

	testHarness.processElement(new Tuple2<>(1, "SET_PROC_TIME_TIMER:20"), 0);

	testHarness.processElement(new Tuple2<>(0, "SET_STATE:HELLO"), 0);
	testHarness.processElement(new Tuple2<>(1, "SET_STATE:CIAO"), 0);

	testHarness.processElement(new Tuple2<>(0, "SET_PROC_TIME_TIMER:10"), 0);

	OperatorSubtaskState snapshot = testHarness.snapshot(0, 0);

	TestOperator testOperator1 = new TestOperator();

	KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, String> testHarness1 =
			new KeyedOneInputStreamOperatorTestHarness<>(
					testOperator1,
					new TestKeySelector(),
					BasicTypeInfo.INT_TYPE_INFO);

	testHarness1.setProcessingTime(0L);

	testHarness1.setup();
	testHarness1.initializeState(snapshot);
	testHarness1.open();

	testHarness1.setProcessingTime(10L);

	assertThat(
			extractResult(testHarness1),
			contains("ON_PROC_TIME:HELLO"));

	testHarness1.setProcessingTime(20L);

	assertThat(
			extractResult(testHarness1),
			contains("ON_PROC_TIME:CIAO"));
}

Example 10

Source File: AbstractStreamOperatorTest.java From flink with Apache License 2.0

4 votes

/**
 * Verify that a low-level timer is set for processing-time timers in case of restore.
 */
@Test
public void testEnsureProcessingTimeTimerRegisteredOnRestore() throws Exception {
	TestOperator testOperator = new TestOperator();

	KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, String> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(testOperator, new TestKeySelector(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.open();

	testHarness.setProcessingTime(0L);

	testHarness.processElement(new Tuple2<>(1, "SET_PROC_TIME_TIMER:20"), 0);

	testHarness.processElement(new Tuple2<>(0, "SET_STATE:HELLO"), 0);
	testHarness.processElement(new Tuple2<>(1, "SET_STATE:CIAO"), 0);

	testHarness.processElement(new Tuple2<>(0, "SET_PROC_TIME_TIMER:10"), 0);

	OperatorSubtaskState snapshot = testHarness.snapshot(0, 0);

	TestOperator testOperator1 = new TestOperator();

	KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, String> testHarness1 =
			new KeyedOneInputStreamOperatorTestHarness<>(
					testOperator1,
					new TestKeySelector(),
					BasicTypeInfo.INT_TYPE_INFO);

	testHarness1.setProcessingTime(0L);

	testHarness1.setup();
	testHarness1.initializeState(snapshot);
	testHarness1.open();

	testHarness1.setProcessingTime(10L);

	assertThat(
			extractResult(testHarness1),
			contains("ON_PROC_TIME:HELLO"));

	testHarness1.setProcessingTime(20L);

	assertThat(
			extractResult(testHarness1),
			contains("ON_PROC_TIME:CIAO"));
}

Example 11

Source File: AbstractStreamOperatorTest.java From Flink-CEPplus with Apache License 2.0

3 votes

/**
 * Verify that firing processing-time timers see the state of the key that was active
 * when the timer was set.
 */
@Test
public void testProcessingTimeTimersDontInterfere() throws Exception {
	TestOperator testOperator = new TestOperator();

	KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, String> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(testOperator, new TestKeySelector(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.open();

	testHarness.setProcessingTime(0L);

	testHarness.processElement(new Tuple2<>(1, "SET_PROC_TIME_TIMER:20"), 0);

	testHarness.processElement(new Tuple2<>(0, "SET_STATE:HELLO"), 0);
	testHarness.processElement(new Tuple2<>(1, "SET_STATE:CIAO"), 0);

	testHarness.processElement(new Tuple2<>(0, "SET_PROC_TIME_TIMER:10"), 0);

	testHarness.setProcessingTime(10L);

	assertThat(
			extractResult(testHarness),
			contains("ON_PROC_TIME:HELLO"));

	testHarness.setProcessingTime(20L);

	assertThat(
			extractResult(testHarness),
			contains("ON_PROC_TIME:CIAO"));
}

Example 12

Source File: WindowOperatorContractTest.java From Flink-CEPplus with Apache License 2.0

3 votes

@Test
public void testProcessingTimeGarbageCollectionTimerIsAlwaysWindowMaxTimestamp() throws Exception {

	WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner();
	when(mockAssigner.isEventTime()).thenReturn(false);
	Trigger<Integer, TimeWindow> mockTrigger = mockTrigger();
	InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction();

	KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness =
			createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction);

	testHarness.open();

	when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext()))
			.thenReturn(Arrays.asList(new TimeWindow(0, Long.MAX_VALUE - 10)));

	assertEquals(0, testHarness.getOutput().size());
	assertEquals(0, testHarness.numKeyedStateEntries());

	testHarness.processElement(new StreamRecord<>(0, 0L));

	// just the window contents
	assertEquals(1, testHarness.numKeyedStateEntries());

	// no GC timer
	assertEquals(0, testHarness.numEventTimeTimers());
	assertEquals(1, testHarness.numProcessingTimeTimers());

	verify(mockTrigger, never()).clear(anyTimeWindow(), anyTriggerContext());

	testHarness.setProcessingTime(Long.MAX_VALUE - 10);

	verify(mockTrigger, times(1)).clear(anyTimeWindow(), anyTriggerContext());

	assertEquals(0, testHarness.numEventTimeTimers());
	assertEquals(0, testHarness.numProcessingTimeTimers());
}

Example 13

Source File: AbstractStreamOperatorTest.java From flink with Apache License 2.0

3 votes

/**
 * Verify that firing processing-time timers see the state of the key that was active
 * when the timer was set.
 */
@Test
public void testProcessingTimeTimersDontInterfere() throws Exception {
	TestOperator testOperator = new TestOperator();

	KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, String> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(testOperator, new TestKeySelector(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.open();

	testHarness.setProcessingTime(0L);

	testHarness.processElement(new Tuple2<>(1, "SET_PROC_TIME_TIMER:20"), 0);

	testHarness.processElement(new Tuple2<>(0, "SET_STATE:HELLO"), 0);
	testHarness.processElement(new Tuple2<>(1, "SET_STATE:CIAO"), 0);

	testHarness.processElement(new Tuple2<>(0, "SET_PROC_TIME_TIMER:10"), 0);

	testHarness.setProcessingTime(10L);

	assertThat(
			extractResult(testHarness),
			contains("ON_PROC_TIME:HELLO"));

	testHarness.setProcessingTime(20L);

	assertThat(
			extractResult(testHarness),
			contains("ON_PROC_TIME:CIAO"));
}

Example 14

Source File: WindowOperatorContractTest.java From flink with Apache License 2.0

3 votes

@Test
public void testProcessingTimeGarbageCollectionTimerIsAlwaysWindowMaxTimestamp() throws Exception {

	WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner();
	when(mockAssigner.isEventTime()).thenReturn(false);
	Trigger<Integer, TimeWindow> mockTrigger = mockTrigger();
	InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction();

	KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness =
			createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction);

	testHarness.open();

	when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext()))
			.thenReturn(Arrays.asList(new TimeWindow(0, Long.MAX_VALUE - 10)));

	assertEquals(0, testHarness.getOutput().size());
	assertEquals(0, testHarness.numKeyedStateEntries());

	testHarness.processElement(new StreamRecord<>(0, 0L));

	// just the window contents
	assertEquals(1, testHarness.numKeyedStateEntries());

	// no GC timer
	assertEquals(0, testHarness.numEventTimeTimers());
	assertEquals(1, testHarness.numProcessingTimeTimers());

	verify(mockTrigger, never()).clear(anyTimeWindow(), anyTriggerContext());

	testHarness.setProcessingTime(Long.MAX_VALUE - 10);

	verify(mockTrigger, times(1)).clear(anyTimeWindow(), anyTriggerContext());

	assertEquals(0, testHarness.numEventTimeTimers());
	assertEquals(0, testHarness.numProcessingTimeTimers());
}

Example 15

Source File: WindowOperatorContractTest.java From flink with Apache License 2.0

3 votes

@Test
public void testProcessingTimeGarbageCollectionTimerIsAlwaysWindowMaxTimestamp() throws Exception {

	WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner();
	when(mockAssigner.isEventTime()).thenReturn(false);
	Trigger<Integer, TimeWindow> mockTrigger = mockTrigger();
	InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction();

	KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness =
			createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction);

	testHarness.open();

	when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext()))
			.thenReturn(Arrays.asList(new TimeWindow(0, Long.MAX_VALUE - 10)));

	assertEquals(0, testHarness.getOutput().size());
	assertEquals(0, testHarness.numKeyedStateEntries());

	testHarness.processElement(new StreamRecord<>(0, 0L));

	// just the window contents
	assertEquals(1, testHarness.numKeyedStateEntries());

	// no GC timer
	assertEquals(0, testHarness.numEventTimeTimers());
	assertEquals(1, testHarness.numProcessingTimeTimers());

	verify(mockTrigger, never()).clear(anyTimeWindow(), anyTriggerContext());

	testHarness.setProcessingTime(Long.MAX_VALUE - 10);

	verify(mockTrigger, times(1)).clear(anyTimeWindow(), anyTriggerContext());

	assertEquals(0, testHarness.numEventTimeTimers());
	assertEquals(0, testHarness.numProcessingTimeTimers());
}