org.apache.flink.streaming.api.windowing.assigners.EventTimeSessionWindows Java Examples

The following examples show how to use org.apache.flink.streaming.api.windowing.assigners.EventTimeSessionWindows. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: AllWindowTranslationTest.java From flink with Apache License 2.0

6 votes

@Test
public void testSessionWithFoldFails() throws Exception {
	// verify that fold does not work with merging windows

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	AllWindowedStream<String, TimeWindow> windowedStream = env.fromElements("Hello", "Ciao")
			.windowAll(EventTimeSessionWindows.withGap(Time.seconds(5)));

	try {
		windowedStream.fold("", new FoldFunction<String, String>() {
			private static final long serialVersionUID = -4567902917104921706L;

			@Override
			public String fold(String accumulator, String value) throws Exception {
				return accumulator;
			}
		});
	} catch (UnsupportedOperationException e) {
		// expected
		// use a catch to ensure that the exception is thrown by the fold
		return;
	}

	fail("The fold call should fail.");
}

Example #2

Source File: EventTimeSessionWindowsTest.java From flink with Apache License 2.0

6 votes

@Test
public void testMergeCoveringWindow() {
	MergingWindowAssigner.MergeCallback callback = mock(MergingWindowAssigner.MergeCallback.class);

	EventTimeSessionWindows assigner = EventTimeSessionWindows.withGap(Time.milliseconds(5000));

	assigner.mergeWindows(
			Lists.newArrayList(
					new TimeWindow(1, 1),
					new TimeWindow(0, 2),
					new TimeWindow(4, 7),
					new TimeWindow(5, 6)),
			callback);

	verify(callback, times(1)).merge(
			(Collection<TimeWindow>) argThat(containsInAnyOrder(new TimeWindow(1, 1), new TimeWindow(0, 2))),
			eq(new TimeWindow(0, 2)));

	verify(callback, times(1)).merge(
			(Collection<TimeWindow>) argThat(containsInAnyOrder(new TimeWindow(5, 6), new TimeWindow(4, 7))),
			eq(new TimeWindow(4, 7)));

	verify(callback, times(2)).merge(anyCollection(), Matchers.anyObject());
}

Example #3

Source File: AllWindowTranslationTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testSessionWithFoldFails() throws Exception {
	// verify that fold does not work with merging windows

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	AllWindowedStream<String, TimeWindow> windowedStream = env.fromElements("Hello", "Ciao")
			.windowAll(EventTimeSessionWindows.withGap(Time.seconds(5)));

	try {
		windowedStream.fold("", new FoldFunction<String, String>() {
			private static final long serialVersionUID = -4567902917104921706L;

			@Override
			public String fold(String accumulator, String value) throws Exception {
				return accumulator;
			}
		});
	} catch (UnsupportedOperationException e) {
		// expected
		// use a catch to ensure that the exception is thrown by the fold
		return;
	}

	fail("The fold call should fail.");
}

Example #4

Source File: EventTimeSessionWindowsTest.java From flink with Apache License 2.0

6 votes

@Test
public void testMergeConsecutiveWindows() {
	MergingWindowAssigner.MergeCallback callback = mock(MergingWindowAssigner.MergeCallback.class);

	EventTimeSessionWindows assigner = EventTimeSessionWindows.withGap(Time.milliseconds(5000));

	assigner.mergeWindows(
			Lists.newArrayList(
					new TimeWindow(0, 1),
					new TimeWindow(1, 2),
					new TimeWindow(2, 3),
					new TimeWindow(4, 5),
					new TimeWindow(5, 6)),
			callback);

	verify(callback, times(1)).merge(
			(Collection<TimeWindow>) argThat(containsInAnyOrder(new TimeWindow(0, 1), new TimeWindow(1, 2), new TimeWindow(2, 3))),
			eq(new TimeWindow(0, 3)));

	verify(callback, times(1)).merge(
			(Collection<TimeWindow>) argThat(containsInAnyOrder(new TimeWindow(4, 5), new TimeWindow(5, 6))),
			eq(new TimeWindow(4, 6)));

	verify(callback, times(2)).merge(anyCollection(), Matchers.anyObject());
}

Example #5

Source File: EventTimeSessionWindowsTest.java From flink with Apache License 2.0

6 votes

@Test
public void testMergeCoveringWindow() {
	MergingWindowAssigner.MergeCallback callback = mock(MergingWindowAssigner.MergeCallback.class);

	EventTimeSessionWindows assigner = EventTimeSessionWindows.withGap(Time.milliseconds(5000));

	assigner.mergeWindows(
			Lists.newArrayList(
					new TimeWindow(1, 1),
					new TimeWindow(0, 2),
					new TimeWindow(4, 7),
					new TimeWindow(5, 6)),
			callback);

	verify(callback, times(1)).merge(
			(Collection<TimeWindow>) argThat(containsInAnyOrder(new TimeWindow(1, 1), new TimeWindow(0, 2))),
			eq(new TimeWindow(0, 2)));

	verify(callback, times(1)).merge(
			(Collection<TimeWindow>) argThat(containsInAnyOrder(new TimeWindow(5, 6), new TimeWindow(4, 7))),
			eq(new TimeWindow(4, 7)));

	verify(callback, times(2)).merge(anyCollection(), Matchers.anyObject());
}

Example #6

Source File: MergingWindowSetTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testPersistOnlyIfHaveUpdates() throws Exception {
	@SuppressWarnings("unchecked")
	ListState<Tuple2<TimeWindow, TimeWindow>> mockState = mock(ListState.class);
	when(mockState.get()).thenReturn(Lists.newArrayList(
			new Tuple2<>(new TimeWindow(17, 42), new TimeWindow(42, 17)),
			new Tuple2<>(new TimeWindow(1, 2), new TimeWindow(3, 4))
	));

	MergingWindowSet<TimeWindow> windowSet = new MergingWindowSet<>(EventTimeSessionWindows.withGap(Time.milliseconds(3)), mockState);

	assertEquals(new TimeWindow(42, 17), windowSet.getStateWindow(new TimeWindow(17, 42)));
	assertEquals(new TimeWindow(3, 4), windowSet.getStateWindow(new TimeWindow(1, 2)));

	windowSet.persist();

	verify(mockState, times(0)).add(Matchers.<Tuple2<TimeWindow, TimeWindow>>anyObject());

}

Example #7

Source File: MergingWindowSetTest.java From flink with Apache License 2.0

6 votes

/**
 * Test adding a new window that is identical to an existing window. This should not cause
 * a merge.
 */
@Test
public void testAddingIdenticalWindows() throws Exception {
	@SuppressWarnings("unchecked")
	ListState<Tuple2<TimeWindow, TimeWindow>> mockState = mock(ListState.class);

	MergingWindowSet<TimeWindow> windowSet = new MergingWindowSet<>(EventTimeSessionWindows.withGap(Time.milliseconds(3)), mockState);

	TestingMergeFunction mergeFunction = new TestingMergeFunction();

	mergeFunction.reset();
	assertEquals(new TimeWindow(1, 2), windowSet.addWindow(new TimeWindow(1, 2), mergeFunction));
	assertFalse(mergeFunction.hasMerged());
	assertEquals(new TimeWindow(1, 2), windowSet.getStateWindow(new TimeWindow(1, 2)));

	mergeFunction.reset();
	assertEquals(new TimeWindow(1, 2), windowSet.addWindow(new TimeWindow(1, 2), mergeFunction));
	assertFalse(mergeFunction.hasMerged());
	assertEquals(new TimeWindow(1, 2), windowSet.getStateWindow(new TimeWindow(1, 2)));
}

Example #8

Source File: MergingWindowSetTest.java From flink with Apache License 2.0

6 votes

/**
 * Test merging of a large new window that covers one existing windows.
 */
@Test
public void testMergeLargeWindowCoveringSingleWindow() throws Exception {
	@SuppressWarnings("unchecked")
	ListState<Tuple2<TimeWindow, TimeWindow>> mockState = mock(ListState.class);

	MergingWindowSet<TimeWindow> windowSet = new MergingWindowSet<>(EventTimeSessionWindows.withGap(Time.milliseconds(3)), mockState);

	TestingMergeFunction mergeFunction = new TestingMergeFunction();

	// add an initial small window

	mergeFunction.reset();
	assertEquals(new TimeWindow(1, 2), windowSet.addWindow(new TimeWindow(1, 2), mergeFunction));
	assertFalse(mergeFunction.hasMerged());
	assertEquals(new TimeWindow(1, 2), windowSet.getStateWindow(new TimeWindow(1, 2)));

	// add a new window that completely covers the existing window

	mergeFunction.reset();
	assertEquals(new TimeWindow(0, 3), windowSet.addWindow(new TimeWindow(0, 3), mergeFunction));
	assertTrue(mergeFunction.hasMerged());
	assertEquals(new TimeWindow(1, 2), windowSet.getStateWindow(new TimeWindow(0, 3)));
}

Example #9

Source File: MergingWindowSetTest.java From Flink-CEPplus with Apache License 2.0

6 votes

/**
 * Test adding a new window that is identical to an existing window. This should not cause
 * a merge.
 */
@Test
public void testAddingIdenticalWindows() throws Exception {
	@SuppressWarnings("unchecked")
	ListState<Tuple2<TimeWindow, TimeWindow>> mockState = mock(ListState.class);

	MergingWindowSet<TimeWindow> windowSet = new MergingWindowSet<>(EventTimeSessionWindows.withGap(Time.milliseconds(3)), mockState);

	TestingMergeFunction mergeFunction = new TestingMergeFunction();

	mergeFunction.reset();
	assertEquals(new TimeWindow(1, 2), windowSet.addWindow(new TimeWindow(1, 2), mergeFunction));
	assertFalse(mergeFunction.hasMerged());
	assertEquals(new TimeWindow(1, 2), windowSet.getStateWindow(new TimeWindow(1, 2)));

	mergeFunction.reset();
	assertEquals(new TimeWindow(1, 2), windowSet.addWindow(new TimeWindow(1, 2), mergeFunction));
	assertFalse(mergeFunction.hasMerged());
	assertEquals(new TimeWindow(1, 2), windowSet.getStateWindow(new TimeWindow(1, 2)));
}

Example #10

Source File: MergingWindowSetTest.java From Flink-CEPplus with Apache License 2.0

6 votes

/**
 * Test merging of a large new window that covers one existing windows.
 */
@Test
public void testMergeLargeWindowCoveringSingleWindow() throws Exception {
	@SuppressWarnings("unchecked")
	ListState<Tuple2<TimeWindow, TimeWindow>> mockState = mock(ListState.class);

	MergingWindowSet<TimeWindow> windowSet = new MergingWindowSet<>(EventTimeSessionWindows.withGap(Time.milliseconds(3)), mockState);

	TestingMergeFunction mergeFunction = new TestingMergeFunction();

	// add an initial small window

	mergeFunction.reset();
	assertEquals(new TimeWindow(1, 2), windowSet.addWindow(new TimeWindow(1, 2), mergeFunction));
	assertFalse(mergeFunction.hasMerged());
	assertEquals(new TimeWindow(1, 2), windowSet.getStateWindow(new TimeWindow(1, 2)));

	// add a new window that completely covers the existing window

	mergeFunction.reset();
	assertEquals(new TimeWindow(0, 3), windowSet.addWindow(new TimeWindow(0, 3), mergeFunction));
	assertTrue(mergeFunction.hasMerged());
	assertEquals(new TimeWindow(1, 2), windowSet.getStateWindow(new TimeWindow(0, 3)));
}

Example #11

Source File: AllWindowTranslationTest.java From flink with Apache License 2.0

6 votes

@Test
@SuppressWarnings("rawtypes")
public void testMergingWindowsWithEvictor() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DataStream<Tuple3<String, String, Integer>> window1 = source
			.windowAll(EventTimeSessionWindows.withGap(Time.seconds(5)))
			.evictor(CountEvictor.of(5))
			.process(new TestProcessAllWindowFunction());

	OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple3<String, String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof WindowOperator);
	WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof EventTimeSessionWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);

	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}

Example #12

Source File: AllWindowTranslationTest.java From flink with Apache License 2.0

6 votes

@Test
@SuppressWarnings("rawtypes")
public void testMergingWindowsWithEvictor() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DataStream<Tuple3<String, String, Integer>> window1 = source
			.windowAll(EventTimeSessionWindows.withGap(Time.seconds(5)))
			.evictor(CountEvictor.of(5))
			.process(new TestProcessAllWindowFunction());

	OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple3<String, String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof WindowOperator);
	WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof EventTimeSessionWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);

	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}

Example #13

Source File: EventTimeSessionWindowsTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testMergeConsecutiveWindows() {
	MergingWindowAssigner.MergeCallback callback = mock(MergingWindowAssigner.MergeCallback.class);

	EventTimeSessionWindows assigner = EventTimeSessionWindows.withGap(Time.milliseconds(5000));

	assigner.mergeWindows(
			Lists.newArrayList(
					new TimeWindow(0, 1),
					new TimeWindow(1, 2),
					new TimeWindow(2, 3),
					new TimeWindow(4, 5),
					new TimeWindow(5, 6)),
			callback);

	verify(callback, times(1)).merge(
			(Collection<TimeWindow>) argThat(containsInAnyOrder(new TimeWindow(0, 1), new TimeWindow(1, 2), new TimeWindow(2, 3))),
			eq(new TimeWindow(0, 3)));

	verify(callback, times(1)).merge(
			(Collection<TimeWindow>) argThat(containsInAnyOrder(new TimeWindow(4, 5), new TimeWindow(5, 6))),
			eq(new TimeWindow(4, 6)));

	verify(callback, times(2)).merge(anyCollection(), Matchers.anyObject());
}

Example #14

Source File: MergingWindowSetTest.java From flink with Apache License 2.0

6 votes

@Test
public void testPersistOnlyIfHaveUpdates() throws Exception {
	@SuppressWarnings("unchecked")
	ListState<Tuple2<TimeWindow, TimeWindow>> mockState = mock(ListState.class);
	when(mockState.get()).thenReturn(Lists.newArrayList(
			new Tuple2<>(new TimeWindow(17, 42), new TimeWindow(42, 17)),
			new Tuple2<>(new TimeWindow(1, 2), new TimeWindow(3, 4))
	));

	MergingWindowSet<TimeWindow> windowSet = new MergingWindowSet<>(EventTimeSessionWindows.withGap(Time.milliseconds(3)), mockState);

	assertEquals(new TimeWindow(42, 17), windowSet.getStateWindow(new TimeWindow(17, 42)));
	assertEquals(new TimeWindow(3, 4), windowSet.getStateWindow(new TimeWindow(1, 2)));

	windowSet.persist();

	verify(mockState, times(0)).add(Matchers.<Tuple2<TimeWindow, TimeWindow>>anyObject());

}

Example #15

Source File: MergingWindowSetTest.java From flink with Apache License 2.0

6 votes

/**
 * Test adding a new window that is identical to an existing window. This should not cause
 * a merge.
 */
@Test
public void testAddingIdenticalWindows() throws Exception {
	@SuppressWarnings("unchecked")
	ListState<Tuple2<TimeWindow, TimeWindow>> mockState = mock(ListState.class);

	MergingWindowSet<TimeWindow> windowSet = new MergingWindowSet<>(EventTimeSessionWindows.withGap(Time.milliseconds(3)), mockState);

	TestingMergeFunction mergeFunction = new TestingMergeFunction();

	mergeFunction.reset();
	assertEquals(new TimeWindow(1, 2), windowSet.addWindow(new TimeWindow(1, 2), mergeFunction));
	assertFalse(mergeFunction.hasMerged());
	assertEquals(new TimeWindow(1, 2), windowSet.getStateWindow(new TimeWindow(1, 2)));

	mergeFunction.reset();
	assertEquals(new TimeWindow(1, 2), windowSet.addWindow(new TimeWindow(1, 2), mergeFunction));
	assertFalse(mergeFunction.hasMerged());
	assertEquals(new TimeWindow(1, 2), windowSet.getStateWindow(new TimeWindow(1, 2)));
}

Example #16

Source File: EventTimeSessionWindowsTest.java From flink with Apache License 2.0

5 votes

@Test
public void testWindowAssignment() {
	final int sessionGap = 5000;

	WindowAssigner.WindowAssignerContext mockContext =
			mock(WindowAssigner.WindowAssignerContext.class);

	EventTimeSessionWindows assigner = EventTimeSessionWindows.withGap(Time.milliseconds(sessionGap));

	assertThat(assigner.assignWindows("String", 0L, mockContext), contains(timeWindow(0, 0 + sessionGap)));
	assertThat(assigner.assignWindows("String", 4999L, mockContext), contains(timeWindow(4999, 4999 + sessionGap)));
	assertThat(assigner.assignWindows("String", 5000L, mockContext), contains(timeWindow(5000, 5000 + sessionGap)));
}

Example #17

Source File: EventTimeSessionWindowsTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testDynamicGapProperties() {
	SessionWindowTimeGapExtractor<String> extractor = mock(SessionWindowTimeGapExtractor.class);
	DynamicEventTimeSessionWindows<String> assigner = EventTimeSessionWindows.withDynamicGap(extractor);

	assertNotNull(assigner);
	assertTrue(assigner.isEventTime());
}

Example #18

Source File: MergingWindowSetTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testRestoreFromState() throws Exception {
	@SuppressWarnings("unchecked")
	ListState<Tuple2<TimeWindow, TimeWindow>> mockState = mock(ListState.class);
	when(mockState.get()).thenReturn(Lists.newArrayList(
			new Tuple2<>(new TimeWindow(17, 42), new TimeWindow(42, 17)),
			new Tuple2<>(new TimeWindow(1, 2), new TimeWindow(3, 4))
	));

	MergingWindowSet<TimeWindow> windowSet = new MergingWindowSet<>(EventTimeSessionWindows.withGap(Time.milliseconds(3)), mockState);

	assertEquals(new TimeWindow(42, 17), windowSet.getStateWindow(new TimeWindow(17, 42)));
	assertEquals(new TimeWindow(3, 4), windowSet.getStateWindow(new TimeWindow(1, 2)));
}

Example #19

Source File: WindowTranslationTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
@SuppressWarnings("rawtypes")
public void testMergingWindowsWithEvictor() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Integer> source = env.fromElements(1, 2);

	DataStream<String> window1 = source
			.keyBy(new KeySelector<Integer, String>() {
				@Override
				public String getKey(Integer value) throws Exception {
					return value.toString();
				}
			})
			.window(EventTimeSessionWindows.withGap(Time.seconds(5)))
			.evictor(CountEvictor.of(5))
			.process(new TestProcessWindowFunction());

	final OneInputTransformation<Integer, String> transform = (OneInputTransformation<Integer, String>) window1.getTransformation();
	final OneInputStreamOperator<Integer, String> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof WindowOperator);
	WindowOperator<String, Integer, ?, ?, ?> winOperator = (WindowOperator<String, Integer, ?, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof EventTimeSessionWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);

	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO,  1);
}

Example #20

Source File: Main4.java From flink-learning with Apache License 2.0

5 votes

public static void main(String[] args) throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
    env.setParallelism(2);
    DataStreamSource<String> data = env.socketTextStream("localhost", 9001);

    data.map(new MapFunction<String, Tuple2<String, Long>>() {
        @Override
        public Tuple2<String, Long> map(String s) throws Exception {
            String[] split = s.split(",");
            return new Tuple2<>(split[0], Long.valueOf(split[1]));
        }
    }).assignTimestampsAndWatermarks(new AssignerWithPeriodicWatermarks<Tuple2<String, Long>>() {
        private long currentTimestamp;

        @Nullable
        @Override
        public Watermark getCurrentWatermark() {
            return new Watermark(currentTimestamp);
        }

        @Override
        public long extractTimestamp(Tuple2<String, Long> tuple2, long l) {
            long timestamp = tuple2.f1;
            currentTimestamp = Math.max(timestamp, currentTimestamp);
            return timestamp;
        }
    }).keyBy(0)
            .window(EventTimeSessionWindows.withGap(Time.minutes(5)))
            .sum(1)
            .print("session ");
    System.out.println(env.getExecutionPlan());
    env.execute();
}

Example #21

Source File: WindowOperatorTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testCleanupTimerWithEmptyListStateForSessionWindows() throws Exception {
	final int gapSize = 3;
	final long lateness = 10;

	ListStateDescriptor<Tuple2<String, Integer>> windowStateDesc =
		new ListStateDescriptor<>("window-contents", STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator =
		new WindowOperator<>(
			EventTimeSessionWindows.withGap(Time.seconds(gapSize)),
			new TimeWindow.Serializer(),
			new TupleKeySelector(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			windowStateDesc,
			new InternalIterableWindowFunction<>(new PassThroughFunction()),
			EventTimeTrigger.create(),
			lateness,
			null /* late data output tag */);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
		createTestHarness(operator);

	testHarness.open();

	ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
	testHarness.processWatermark(new Watermark(4998));

	expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 3999));
	expected.add(new Watermark(4998));

	testHarness.processWatermark(new Watermark(14600));
	expected.add(new Watermark(14600));

	ConcurrentLinkedQueue<Object> actual = testHarness.getOutput();
	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple2ResultSortComparator());
	testHarness.close();
}

Example #22

Source File: EventTimeSessionWindowsTest.java From flink with Apache License 2.0

5 votes

@Test
public void testProperties() {
	EventTimeSessionWindows assigner = EventTimeSessionWindows.withGap(Time.seconds(5));

	assertTrue(assigner.isEventTime());
	assertEquals(new TimeWindow.Serializer(), assigner.getWindowSerializer(new ExecutionConfig()));
	assertThat(assigner.getDefaultTrigger(mock(StreamExecutionEnvironment.class)), instanceOf(EventTimeTrigger.class));
}

Example #23

Source File: SessionWindowITCase.java From Flink-CEPplus with Apache License 2.0

5 votes

private void runTest(
		SourceFunction<SessionEvent<Integer, TestEventPayload>> dataSource,
		WindowFunction<SessionEvent<Integer, TestEventPayload>,
				String, Tuple, TimeWindow> windowFunction) throws Exception {

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
	WindowedStream<SessionEvent<Integer, TestEventPayload>, Tuple, TimeWindow> windowedStream =
			env.addSource(dataSource).keyBy("sessionKey")
			.window(EventTimeSessionWindows.withGap(Time.milliseconds(MAX_SESSION_EVENT_GAP_MS)));

	if (ALLOWED_LATENESS_MS != Long.MAX_VALUE) {
		windowedStream = windowedStream.allowedLateness(Time.milliseconds(ALLOWED_LATENESS_MS));
	}

	if (PURGE_WINDOW_ON_FIRE) {
		windowedStream = windowedStream.trigger(PurgingTrigger.of(EventTimeTrigger.create()));
	}

	windowedStream.apply(windowFunction).print();
	JobExecutionResult result = env.execute();

	// check that overall event counts match with our expectations. remember that late events within lateness will
	// each trigger a window!
	Assert.assertEquals(
		(LATE_EVENTS_PER_SESSION + 1) * NUMBER_OF_SESSIONS * EVENTS_PER_SESSION,
		(long) result.getAccumulatorResult(SESSION_COUNTER_ON_TIME_KEY));
	Assert.assertEquals(
		NUMBER_OF_SESSIONS * (LATE_EVENTS_PER_SESSION * (LATE_EVENTS_PER_SESSION + 1) / 2),
		(long) result.getAccumulatorResult(SESSION_COUNTER_LATE_KEY));
}

Example #24

Source File: SessionWindowITCase.java From flink with Apache License 2.0

5 votes

private void runTest(
		SourceFunction<SessionEvent<Integer, TestEventPayload>> dataSource,
		WindowFunction<SessionEvent<Integer, TestEventPayload>,
				String, Tuple, TimeWindow> windowFunction) throws Exception {

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
	WindowedStream<SessionEvent<Integer, TestEventPayload>, Tuple, TimeWindow> windowedStream =
			env.addSource(dataSource).keyBy("sessionKey")
			.window(EventTimeSessionWindows.withGap(Time.milliseconds(MAX_SESSION_EVENT_GAP_MS)));

	if (ALLOWED_LATENESS_MS != Long.MAX_VALUE) {
		windowedStream = windowedStream.allowedLateness(Time.milliseconds(ALLOWED_LATENESS_MS));
	}

	if (PURGE_WINDOW_ON_FIRE) {
		windowedStream = windowedStream.trigger(PurgingTrigger.of(EventTimeTrigger.create()));
	}

	windowedStream.apply(windowFunction).print();
	JobExecutionResult result = env.execute();

	// check that overall event counts match with our expectations. remember that late events within lateness will
	// each trigger a window!
	Assert.assertEquals(
		(LATE_EVENTS_PER_SESSION + 1) * NUMBER_OF_SESSIONS * EVENTS_PER_SESSION,
		(long) result.getAccumulatorResult(SESSION_COUNTER_ON_TIME_KEY));
	Assert.assertEquals(
		NUMBER_OF_SESSIONS * (LATE_EVENTS_PER_SESSION * (LATE_EVENTS_PER_SESSION + 1) / 2),
		(long) result.getAccumulatorResult(SESSION_COUNTER_LATE_KEY));
}

Example #25

Source File: MergingWindowSetTest.java From flink with Apache License 2.0

5 votes

@Test
public void testRestoreFromState() throws Exception {
	@SuppressWarnings("unchecked")
	ListState<Tuple2<TimeWindow, TimeWindow>> mockState = mock(ListState.class);
	when(mockState.get()).thenReturn(Lists.newArrayList(
			new Tuple2<>(new TimeWindow(17, 42), new TimeWindow(42, 17)),
			new Tuple2<>(new TimeWindow(1, 2), new TimeWindow(3, 4))
	));

	MergingWindowSet<TimeWindow> windowSet = new MergingWindowSet<>(EventTimeSessionWindows.withGap(Time.milliseconds(3)), mockState);

	assertEquals(new TimeWindow(42, 17), windowSet.getStateWindow(new TimeWindow(17, 42)));
	assertEquals(new TimeWindow(3, 4), windowSet.getStateWindow(new TimeWindow(1, 2)));
}

Example #26

Source File: EventTimeSessionWindowsTest.java From flink with Apache License 2.0

5 votes

@Test
public void testTimeUnits() {
	// sanity check with one other time unit

	final int sessionGap = 5000;

	WindowAssigner.WindowAssignerContext mockContext =
			mock(WindowAssigner.WindowAssignerContext.class);

	EventTimeSessionWindows assigner = EventTimeSessionWindows.withGap(Time.seconds(sessionGap / 1000));

	assertThat(assigner.assignWindows("String", 0L, mockContext), contains(timeWindow(0, 0 + sessionGap)));
	assertThat(assigner.assignWindows("String", 4999L, mockContext), contains(timeWindow(4999, 4999 + sessionGap)));
	assertThat(assigner.assignWindows("String", 5000L, mockContext), contains(timeWindow(5000, 5000 + sessionGap)));
}

Example #27

Source File: EventTimeSessionWindowsTest.java From flink with Apache License 2.0

5 votes

@Test
public void testProperties() {
	EventTimeSessionWindows assigner = EventTimeSessionWindows.withGap(Time.seconds(5));

	assertTrue(assigner.isEventTime());
	assertEquals(new TimeWindow.Serializer(), assigner.getWindowSerializer(new ExecutionConfig()));
	assertThat(assigner.getDefaultTrigger(mock(StreamExecutionEnvironment.class)), instanceOf(EventTimeTrigger.class));
}

Example #28

Source File: EventTimeSessionWindowsTest.java From flink with Apache License 2.0

5 votes

@Test
public void testTimeUnits() {
	// sanity check with one other time unit

	final int sessionGap = 5000;

	WindowAssigner.WindowAssignerContext mockContext =
			mock(WindowAssigner.WindowAssignerContext.class);

	EventTimeSessionWindows assigner = EventTimeSessionWindows.withGap(Time.seconds(sessionGap / 1000));

	assertThat(assigner.assignWindows("String", 0L, mockContext), contains(timeWindow(0, 0 + sessionGap)));
	assertThat(assigner.assignWindows("String", 4999L, mockContext), contains(timeWindow(4999, 4999 + sessionGap)));
	assertThat(assigner.assignWindows("String", 5000L, mockContext), contains(timeWindow(5000, 5000 + sessionGap)));
}

Example #29

Source File: EventTimeSessionWindowsTest.java From flink with Apache License 2.0

5 votes

@Test
public void testDynamicGapProperties() {
	SessionWindowTimeGapExtractor<String> extractor = mock(SessionWindowTimeGapExtractor.class);
	DynamicEventTimeSessionWindows<String> assigner = EventTimeSessionWindows.withDynamicGap(extractor);

	assertNotNull(assigner);
	assertTrue(assigner.isEventTime());
}

Example #30

Source File: WindowTranslationTest.java From flink with Apache License 2.0

5 votes

@Test
@SuppressWarnings("rawtypes")
public void testMergingWindowsWithEvictor() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Integer> source = env.fromElements(1, 2);

	DataStream<String> window1 = source
			.keyBy(new KeySelector<Integer, String>() {
				@Override
				public String getKey(Integer value) throws Exception {
					return value.toString();
				}
			})
			.window(EventTimeSessionWindows.withGap(Time.seconds(5)))
			.evictor(CountEvictor.of(5))
			.process(new TestProcessWindowFunction());

	final OneInputTransformation<Integer, String> transform = (OneInputTransformation<Integer, String>) window1.getTransformation();
	final OneInputStreamOperator<Integer, String> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof WindowOperator);
	WindowOperator<String, Integer, ?, ?, ?> winOperator = (WindowOperator<String, Integer, ?, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof EventTimeSessionWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);

	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO,  1);
}