Java Code Examples for org.apache.flink.streaming.api.windowing.assigners.SlidingEventTimeWindows#of()

The following examples show how to use org.apache.flink.streaming.api.windowing.assigners.SlidingEventTimeWindows#of() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: WindowOperatorTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
@SuppressWarnings("unchecked")
public void testSlidingEventTimeWindowsReduce() throws Exception {
	closeCalled.set(0);

	final int windowSize = 3;
	final int windowSlide = 1;

	ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
			new SumReducer(),
			STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
			SlidingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS), Time.of(windowSlide, TimeUnit.SECONDS)),
			new TimeWindow.Serializer(),
			new TupleKeySelector(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			stateDesc,
			new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
			EventTimeTrigger.create(),
			0,
			null /* late data output tag */);

	testSlidingEventTimeWindows(operator);
}

Example 2

Source File: WindowOperatorTest.java From flink with Apache License 2.0

6 votes

@Test
public void testSlidingEventTimeWindowsApply() throws Exception {
	closeCalled.set(0);

	final int windowSize = 3;
	final int windowSlide = 1;

	ListStateDescriptor<Tuple2<String, Integer>> stateDesc = new ListStateDescriptor<>("window-contents",
			STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
			SlidingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS), Time.of(windowSlide, TimeUnit.SECONDS)),
			new TimeWindow.Serializer(),
			new TupleKeySelector(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			stateDesc,
			new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>()),
			EventTimeTrigger.create(),
			0,
			null /* late data output tag */);

	testSlidingEventTimeWindows(operator);

	// we close once in the rest...
	Assert.assertEquals("Close was not called.", 2, closeCalled.get());
}

Example 3

Source File: WindowOperatorTest.java From flink with Apache License 2.0

6 votes

@Test
public void testSlidingEventTimeWindowsApply() throws Exception {
	closeCalled.set(0);

	final int windowSize = 3;
	final int windowSlide = 1;

	ListStateDescriptor<Tuple2<String, Integer>> stateDesc = new ListStateDescriptor<>("window-contents",
			STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
			SlidingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS), Time.of(windowSlide, TimeUnit.SECONDS)),
			new TimeWindow.Serializer(),
			new TupleKeySelector(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			stateDesc,
			new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>()),
			EventTimeTrigger.create(),
			0,
			null /* late data output tag */);

	testSlidingEventTimeWindows(operator);

	// we close once in the rest...
	Assert.assertEquals("Close was not called.", 2, closeCalled.get());
}

Example 4

Source File: WindowOperatorTest.java From flink with Apache License 2.0

6 votes

@Test
@SuppressWarnings("unchecked")
public void testSlidingEventTimeWindowsReduce() throws Exception {
	closeCalled.set(0);

	final int windowSize = 3;
	final int windowSlide = 1;

	ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
			new SumReducer(),
			STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
			SlidingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS), Time.of(windowSlide, TimeUnit.SECONDS)),
			new TimeWindow.Serializer(),
			new TupleKeySelector(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			stateDesc,
			new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
			EventTimeTrigger.create(),
			0,
			null /* late data output tag */);

	testSlidingEventTimeWindows(operator);
}

Example 5

Source File: WindowOperatorTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testSlidingEventTimeWindowsApply() throws Exception {
	closeCalled.set(0);

	final int windowSize = 3;
	final int windowSlide = 1;

	ListStateDescriptor<Tuple2<String, Integer>> stateDesc = new ListStateDescriptor<>("window-contents",
			STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
			SlidingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS), Time.of(windowSlide, TimeUnit.SECONDS)),
			new TimeWindow.Serializer(),
			new TupleKeySelector(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			stateDesc,
			new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>()),
			EventTimeTrigger.create(),
			0,
			null /* late data output tag */);

	testSlidingEventTimeWindows(operator);

	// we close once in the rest...
	Assert.assertEquals("Close was not called.", 2, closeCalled.get());
}

Example 6

Source File: SlidingEventTimeWindowsTest.java From flink with Apache License 2.0

5 votes

@Test
public void testWindowAssignmentWithNegativeOffset() {
	WindowAssigner.WindowAssignerContext mockContext =
		mock(WindowAssigner.WindowAssignerContext.class);

	SlidingEventTimeWindows assigner =
		SlidingEventTimeWindows.of(Time.milliseconds(5000), Time.milliseconds(1000), Time.milliseconds(-100));

	assertThat(assigner.assignWindows("String", 0L, mockContext), containsInAnyOrder(
		timeWindow(-4100, 900),
		timeWindow(-3100, 1900),
		timeWindow(-2100, 2900),
		timeWindow(-1100, 3900),
		timeWindow(-100, 4900)));

	assertThat(assigner.assignWindows("String", 4899L, mockContext), containsInAnyOrder(
		timeWindow(-100, 4900),
		timeWindow(900, 5900),
		timeWindow(1900, 6900),
		timeWindow(2900, 7900),
		timeWindow(3900, 8900)));

	assertThat(assigner.assignWindows("String", 4900L, mockContext), containsInAnyOrder(
		timeWindow(900, 5900),
		timeWindow(1900, 6900),
		timeWindow(2900, 7900),
		timeWindow(3900, 8900),
		timeWindow(4900, 9900)));
}

Example 7

Source File: SlidingEventTimeWindowsTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testWindowAssignmentWithOffset() {
	WindowAssigner.WindowAssignerContext mockContext =
			mock(WindowAssigner.WindowAssignerContext.class);

	SlidingEventTimeWindows assigner =
			SlidingEventTimeWindows.of(Time.milliseconds(5000), Time.milliseconds(1000), Time.milliseconds(100));

	assertThat(assigner.assignWindows("String", 100L, mockContext), containsInAnyOrder(
			timeWindow(-3900, 1100),
			timeWindow(-2900, 2100),
			timeWindow(-1900, 3100),
			timeWindow(-900, 4100),
			timeWindow(100, 5100)));

	assertThat(assigner.assignWindows("String", 5099L, mockContext), containsInAnyOrder(
			timeWindow(100, 5100),
			timeWindow(1100, 6100),
			timeWindow(2100, 7100),
			timeWindow(3100, 8100),
			timeWindow(4100, 9100)));

	assertThat(assigner.assignWindows("String", 5100L, mockContext), containsInAnyOrder(
			timeWindow(1100, 6100),
			timeWindow(2100, 7100),
			timeWindow(3100, 8100),
			timeWindow(4100, 9100),
			timeWindow(5100, 10100)));
}

Example 8

Source File: SlidingEventTimeWindowsTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testWindowAssignmentWithNegativeOffset() {
	WindowAssigner.WindowAssignerContext mockContext =
		mock(WindowAssigner.WindowAssignerContext.class);

	SlidingEventTimeWindows assigner =
		SlidingEventTimeWindows.of(Time.milliseconds(5000), Time.milliseconds(1000), Time.milliseconds(-100));

	assertThat(assigner.assignWindows("String", 0L, mockContext), containsInAnyOrder(
		timeWindow(-4100, 900),
		timeWindow(-3100, 1900),
		timeWindow(-2100, 2900),
		timeWindow(-1100, 3900),
		timeWindow(-100, 4900)));

	assertThat(assigner.assignWindows("String", 4899L, mockContext), containsInAnyOrder(
		timeWindow(-100, 4900),
		timeWindow(900, 5900),
		timeWindow(1900, 6900),
		timeWindow(2900, 7900),
		timeWindow(3900, 8900)));

	assertThat(assigner.assignWindows("String", 4900L, mockContext), containsInAnyOrder(
		timeWindow(900, 5900),
		timeWindow(1900, 6900),
		timeWindow(2900, 7900),
		timeWindow(3900, 8900),
		timeWindow(4900, 9900)));
}

Example 9

Source File: SlidingEventTimeWindowsTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testTimeUnits() {
	// sanity check with one other time unit

	WindowAssigner.WindowAssignerContext mockContext =
			mock(WindowAssigner.WindowAssignerContext.class);

	SlidingEventTimeWindows assigner = SlidingEventTimeWindows.of(Time.seconds(5), Time.seconds(1), Time.milliseconds(500));

	assertThat(assigner.assignWindows("String", 100L, mockContext), containsInAnyOrder(
			timeWindow(-4500, 500),
			timeWindow(-3500, 1500),
			timeWindow(-2500, 2500),
			timeWindow(-1500, 3500),
			timeWindow(-500, 4500)));

	assertThat(assigner.assignWindows("String", 5499L, mockContext), containsInAnyOrder(
			timeWindow(500, 5500),
			timeWindow(1500, 6500),
			timeWindow(2500, 7500),
			timeWindow(3500, 8500),
			timeWindow(4500, 9500)));

	assertThat(assigner.assignWindows("String", 5100L, mockContext), containsInAnyOrder(
			timeWindow(500, 5500),
			timeWindow(1500, 6500),
			timeWindow(2500, 7500),
			timeWindow(3500, 8500),
			timeWindow(4500, 9500)));
}

Example 10

Source File: DataStreamAllroundTestJobFactory.java From flink with Apache License 2.0

5 votes

static SlidingEventTimeWindows createSlidingWindow(ParameterTool pt) {
	long slideSize = pt.getLong(
		TEST_SLIDE_SIZE.key(),
		TEST_SLIDE_SIZE.defaultValue());

	long slideFactor = pt.getInt(
		TEST_SLIDE_FACTOR.key(),
		TEST_SLIDE_FACTOR.defaultValue()
	);

	return SlidingEventTimeWindows.of(Time.milliseconds(slideSize * slideFactor), Time.milliseconds(slideSize));
}

Example 11

Source File: SlidingEventTimeWindowsTest.java From flink with Apache License 2.0

5 votes

@Test
public void testWindowAssignmentWithOffset() {
	WindowAssigner.WindowAssignerContext mockContext =
			mock(WindowAssigner.WindowAssignerContext.class);

	SlidingEventTimeWindows assigner =
			SlidingEventTimeWindows.of(Time.milliseconds(5000), Time.milliseconds(1000), Time.milliseconds(100));

	assertThat(assigner.assignWindows("String", 100L, mockContext), containsInAnyOrder(
			timeWindow(-3900, 1100),
			timeWindow(-2900, 2100),
			timeWindow(-1900, 3100),
			timeWindow(-900, 4100),
			timeWindow(100, 5100)));

	assertThat(assigner.assignWindows("String", 5099L, mockContext), containsInAnyOrder(
			timeWindow(100, 5100),
			timeWindow(1100, 6100),
			timeWindow(2100, 7100),
			timeWindow(3100, 8100),
			timeWindow(4100, 9100)));

	assertThat(assigner.assignWindows("String", 5100L, mockContext), containsInAnyOrder(
			timeWindow(1100, 6100),
			timeWindow(2100, 7100),
			timeWindow(3100, 8100),
			timeWindow(4100, 9100),
			timeWindow(5100, 10100)));
}

Example 12

Source File: SlidingEventTimeWindowsTest.java From flink with Apache License 2.0

5 votes

@Test
public void testWindowAssignmentWithOffset() {
	WindowAssigner.WindowAssignerContext mockContext =
			mock(WindowAssigner.WindowAssignerContext.class);

	SlidingEventTimeWindows assigner =
			SlidingEventTimeWindows.of(Time.milliseconds(5000), Time.milliseconds(1000), Time.milliseconds(100));

	assertThat(assigner.assignWindows("String", 100L, mockContext), containsInAnyOrder(
			timeWindow(-3900, 1100),
			timeWindow(-2900, 2100),
			timeWindow(-1900, 3100),
			timeWindow(-900, 4100),
			timeWindow(100, 5100)));

	assertThat(assigner.assignWindows("String", 5099L, mockContext), containsInAnyOrder(
			timeWindow(100, 5100),
			timeWindow(1100, 6100),
			timeWindow(2100, 7100),
			timeWindow(3100, 8100),
			timeWindow(4100, 9100)));

	assertThat(assigner.assignWindows("String", 5100L, mockContext), containsInAnyOrder(
			timeWindow(1100, 6100),
			timeWindow(2100, 7100),
			timeWindow(3100, 8100),
			timeWindow(4100, 9100),
			timeWindow(5100, 10100)));
}

Example 13

Source File: SlidingEventTimeWindowsTest.java From flink with Apache License 2.0

5 votes

@Test
public void testWindowAssignmentWithNegativeOffset() {
	WindowAssigner.WindowAssignerContext mockContext =
		mock(WindowAssigner.WindowAssignerContext.class);

	SlidingEventTimeWindows assigner =
		SlidingEventTimeWindows.of(Time.milliseconds(5000), Time.milliseconds(1000), Time.milliseconds(-100));

	assertThat(assigner.assignWindows("String", 0L, mockContext), containsInAnyOrder(
		timeWindow(-4100, 900),
		timeWindow(-3100, 1900),
		timeWindow(-2100, 2900),
		timeWindow(-1100, 3900),
		timeWindow(-100, 4900)));

	assertThat(assigner.assignWindows("String", 4899L, mockContext), containsInAnyOrder(
		timeWindow(-100, 4900),
		timeWindow(900, 5900),
		timeWindow(1900, 6900),
		timeWindow(2900, 7900),
		timeWindow(3900, 8900)));

	assertThat(assigner.assignWindows("String", 4900L, mockContext), containsInAnyOrder(
		timeWindow(900, 5900),
		timeWindow(1900, 6900),
		timeWindow(2900, 7900),
		timeWindow(3900, 8900),
		timeWindow(4900, 9900)));
}

Example 14

Source File: SlidingEventTimeWindowsTest.java From flink with Apache License 2.0

5 votes

@Test
public void testTimeUnits() {
	// sanity check with one other time unit

	WindowAssigner.WindowAssignerContext mockContext =
			mock(WindowAssigner.WindowAssignerContext.class);

	SlidingEventTimeWindows assigner = SlidingEventTimeWindows.of(Time.seconds(5), Time.seconds(1), Time.milliseconds(500));

	assertThat(assigner.assignWindows("String", 100L, mockContext), containsInAnyOrder(
			timeWindow(-4500, 500),
			timeWindow(-3500, 1500),
			timeWindow(-2500, 2500),
			timeWindow(-1500, 3500),
			timeWindow(-500, 4500)));

	assertThat(assigner.assignWindows("String", 5499L, mockContext), containsInAnyOrder(
			timeWindow(500, 5500),
			timeWindow(1500, 6500),
			timeWindow(2500, 7500),
			timeWindow(3500, 8500),
			timeWindow(4500, 9500)));

	assertThat(assigner.assignWindows("String", 5100L, mockContext), containsInAnyOrder(
			timeWindow(500, 5500),
			timeWindow(1500, 6500),
			timeWindow(2500, 7500),
			timeWindow(3500, 8500),
			timeWindow(4500, 9500)));
}

Example 15

Source File: SlidingEventTimeWindowsTest.java From flink with Apache License 2.0

5 votes

@Test
public void testProperties() {
	SlidingEventTimeWindows assigner = SlidingEventTimeWindows.of(Time.seconds(5), Time.milliseconds(100));

	assertTrue(assigner.isEventTime());
	assertEquals(new TimeWindow.Serializer(), assigner.getWindowSerializer(new ExecutionConfig()));
	assertThat(assigner.getDefaultTrigger(mock(StreamExecutionEnvironment.class)), instanceOf(EventTimeTrigger.class));
}

Example 16

Source File: DataStreamAllroundTestJobFactory.java From flink with Apache License 2.0

5 votes

static SlidingEventTimeWindows createSlidingWindow(ParameterTool pt) {
	long slideSize = pt.getLong(
		TEST_SLIDE_SIZE.key(),
		TEST_SLIDE_SIZE.defaultValue());

	long slideFactor = pt.getInt(
		TEST_SLIDE_FACTOR.key(),
		TEST_SLIDE_FACTOR.defaultValue()
	);

	return SlidingEventTimeWindows.of(Time.milliseconds(slideSize * slideFactor), Time.milliseconds(slideSize));
}

Example 17

Source File: SlidingEventTimeWindowsTest.java From flink with Apache License 2.0

5 votes

@Test
public void testTimeUnits() {
	// sanity check with one other time unit

	WindowAssigner.WindowAssignerContext mockContext =
			mock(WindowAssigner.WindowAssignerContext.class);

	SlidingEventTimeWindows assigner = SlidingEventTimeWindows.of(Time.seconds(5), Time.seconds(1), Time.milliseconds(500));

	assertThat(assigner.assignWindows("String", 100L, mockContext), containsInAnyOrder(
			timeWindow(-4500, 500),
			timeWindow(-3500, 1500),
			timeWindow(-2500, 2500),
			timeWindow(-1500, 3500),
			timeWindow(-500, 4500)));

	assertThat(assigner.assignWindows("String", 5499L, mockContext), containsInAnyOrder(
			timeWindow(500, 5500),
			timeWindow(1500, 6500),
			timeWindow(2500, 7500),
			timeWindow(3500, 8500),
			timeWindow(4500, 9500)));

	assertThat(assigner.assignWindows("String", 5100L, mockContext), containsInAnyOrder(
			timeWindow(500, 5500),
			timeWindow(1500, 6500),
			timeWindow(2500, 7500),
			timeWindow(3500, 8500),
			timeWindow(4500, 9500)));
}

Example 18

Source File: WindowOperatorTest.java From flink with Apache License 2.0

4 votes

@Test
public void testSideOutputDueToLatenessSliding() throws Exception {
	final int windowSize = 3;
	final int windowSlide = 1;
	final long lateness = 0;

	ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
		new SumReducer(),
		STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator =
		new WindowOperator<>(
			SlidingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS), Time.of(windowSlide, TimeUnit.SECONDS)),
			new TimeWindow.Serializer(),
			new TupleKeySelector(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			stateDesc,
			new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
			EventTimeTrigger.create(),
			lateness,
			lateOutputTag /* late data output tag */);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
		createTestHarness(operator);

	testHarness.open();

	ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
	ConcurrentLinkedQueue<Object> sideExpected = new ConcurrentLinkedQueue<>();

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
	testHarness.processWatermark(new Watermark(1999));

	expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 1999));
	expected.add(new Watermark(1999));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2000));
	testHarness.processWatermark(new Watermark(3000));

	expected.add(new StreamRecord<>(new Tuple2<>("key2", 2), 2999));
	expected.add(new Watermark(3000));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 3001));

	// lateness is set to 0 and window size = 3 sec and slide 1, the following 2 elements (2400)
	// are assigned to windows ending at 2999, 3999, 4999.
	// The 2999 is dropped because it is already late (WM = 2999) but the rest are kept.

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2400));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2400));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 3001));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3900));
	testHarness.processWatermark(new Watermark(6000));

	expected.add(new StreamRecord<>(new Tuple2<>("key2", 5), 3999));
	expected.add(new StreamRecord<>(new Tuple2<>("key1", 2), 3999));

	expected.add(new StreamRecord<>(new Tuple2<>("key2", 4), 4999));
	expected.add(new StreamRecord<>(new Tuple2<>("key1", 2), 4999));

	expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 5999));
	expected.add(new StreamRecord<>(new Tuple2<>("key1", 2), 5999));

	expected.add(new Watermark(6000));

	// sideoutput element due to lateness
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 3001));
	sideExpected.add(new StreamRecord<>(new Tuple2<>("key1", 1), 3001));

	testHarness.processWatermark(new Watermark(25000));

	expected.add(new Watermark(25000));

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, testHarness.getOutput(), new Tuple2ResultSortComparator());
	TestHarnessUtil.assertOutputEqualsSorted("SideOutput was not correct.", sideExpected, (Iterable) testHarness.getSideOutput(lateOutputTag), new Tuple2ResultSortComparator());
	testHarness.close();
}

Example 19

Source File: WindowOperatorTest.java From Flink-CEPplus with Apache License 2.0

4 votes

@Test
public void testSideOutputDueToLatenessSliding() throws Exception {
	final int windowSize = 3;
	final int windowSlide = 1;
	final long lateness = 0;

	ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
		new SumReducer(),
		STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator =
		new WindowOperator<>(
			SlidingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS), Time.of(windowSlide, TimeUnit.SECONDS)),
			new TimeWindow.Serializer(),
			new TupleKeySelector(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			stateDesc,
			new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
			EventTimeTrigger.create(),
			lateness,
			lateOutputTag /* late data output tag */);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
		createTestHarness(operator);

	testHarness.open();

	ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
	ConcurrentLinkedQueue<Object> sideExpected = new ConcurrentLinkedQueue<>();

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
	testHarness.processWatermark(new Watermark(1999));

	expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 1999));
	expected.add(new Watermark(1999));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2000));
	testHarness.processWatermark(new Watermark(3000));

	expected.add(new StreamRecord<>(new Tuple2<>("key2", 2), 2999));
	expected.add(new Watermark(3000));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 3001));

	// lateness is set to 0 and window size = 3 sec and slide 1, the following 2 elements (2400)
	// are assigned to windows ending at 2999, 3999, 4999.
	// The 2999 is dropped because it is already late (WM = 2999) but the rest are kept.

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2400));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2400));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 3001));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3900));
	testHarness.processWatermark(new Watermark(6000));

	expected.add(new StreamRecord<>(new Tuple2<>("key2", 5), 3999));
	expected.add(new StreamRecord<>(new Tuple2<>("key1", 2), 3999));

	expected.add(new StreamRecord<>(new Tuple2<>("key2", 4), 4999));
	expected.add(new StreamRecord<>(new Tuple2<>("key1", 2), 4999));

	expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 5999));
	expected.add(new StreamRecord<>(new Tuple2<>("key1", 2), 5999));

	expected.add(new Watermark(6000));

	// sideoutput element due to lateness
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 3001));
	sideExpected.add(new StreamRecord<>(new Tuple2<>("key1", 1), 3001));

	testHarness.processWatermark(new Watermark(25000));

	expected.add(new Watermark(25000));

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, testHarness.getOutput(), new Tuple2ResultSortComparator());
	TestHarnessUtil.assertOutputEqualsSorted("SideOutput was not correct.", sideExpected, (Iterable) testHarness.getSideOutput(lateOutputTag), new Tuple2ResultSortComparator());
	testHarness.close();
}

Example 20

Source File: WindowOperatorTest.java From flink with Apache License 2.0

4 votes

@Test
public void testSideOutputDueToLatenessSliding() throws Exception {
	final int windowSize = 3;
	final int windowSlide = 1;
	final long lateness = 0;

	ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
		new SumReducer(),
		STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator =
		new WindowOperator<>(
			SlidingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS), Time.of(windowSlide, TimeUnit.SECONDS)),
			new TimeWindow.Serializer(),
			new TupleKeySelector(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			stateDesc,
			new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
			EventTimeTrigger.create(),
			lateness,
			lateOutputTag /* late data output tag */);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
		createTestHarness(operator);

	testHarness.open();

	ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
	ConcurrentLinkedQueue<Object> sideExpected = new ConcurrentLinkedQueue<>();

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
	testHarness.processWatermark(new Watermark(1999));

	expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 1999));
	expected.add(new Watermark(1999));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2000));
	testHarness.processWatermark(new Watermark(3000));

	expected.add(new StreamRecord<>(new Tuple2<>("key2", 2), 2999));
	expected.add(new Watermark(3000));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 3001));

	// lateness is set to 0 and window size = 3 sec and slide 1, the following 2 elements (2400)
	// are assigned to windows ending at 2999, 3999, 4999.
	// The 2999 is dropped because it is already late (WM = 2999) but the rest are kept.

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2400));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2400));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 3001));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3900));
	testHarness.processWatermark(new Watermark(6000));

	expected.add(new StreamRecord<>(new Tuple2<>("key2", 5), 3999));
	expected.add(new StreamRecord<>(new Tuple2<>("key1", 2), 3999));

	expected.add(new StreamRecord<>(new Tuple2<>("key2", 4), 4999));
	expected.add(new StreamRecord<>(new Tuple2<>("key1", 2), 4999));

	expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 5999));
	expected.add(new StreamRecord<>(new Tuple2<>("key1", 2), 5999));

	expected.add(new Watermark(6000));

	// sideoutput element due to lateness
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 3001));
	sideExpected.add(new StreamRecord<>(new Tuple2<>("key1", 1), 3001));

	testHarness.processWatermark(new Watermark(25000));

	expected.add(new Watermark(25000));

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, testHarness.getOutput(), new Tuple2ResultSortComparator());
	TestHarnessUtil.assertOutputEqualsSorted("SideOutput was not correct.", sideExpected, (Iterable) testHarness.getSideOutput(lateOutputTag), new Tuple2ResultSortComparator());
	testHarness.close();
}