org.apache.flink.streaming.api.windowing.windows.TimeWindow Java Examples
The following examples show how to use
org.apache.flink.streaming.api.windowing.windows.TimeWindow.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ProcessingTimeSessionWindowsTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testMergeCoveringWindow() { MergingWindowAssigner.MergeCallback callback = mock(MergingWindowAssigner.MergeCallback.class); ProcessingTimeSessionWindows assigner = ProcessingTimeSessionWindows.withGap(Time.milliseconds(5000)); assigner.mergeWindows( Lists.newArrayList( new TimeWindow(1, 1), new TimeWindow(0, 2), new TimeWindow(4, 7), new TimeWindow(5, 6)), callback); verify(callback, times(1)).merge( (Collection<TimeWindow>) argThat(containsInAnyOrder(new TimeWindow(1, 1), new TimeWindow(0, 2))), eq(new TimeWindow(0, 2))); verify(callback, times(1)).merge( (Collection<TimeWindow>) argThat(containsInAnyOrder(new TimeWindow(5, 6), new TimeWindow(4, 7))), eq(new TimeWindow(4, 7))); verify(callback, times(2)).merge(anyCollection(), Matchers.anyObject()); }
Example #2
Source File: ProcessingTimeSessionWindowsTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testMergeConsecutiveWindows() { MergingWindowAssigner.MergeCallback callback = mock(MergingWindowAssigner.MergeCallback.class); ProcessingTimeSessionWindows assigner = ProcessingTimeSessionWindows.withGap(Time.milliseconds(5000)); assigner.mergeWindows( Lists.newArrayList( new TimeWindow(0, 1), new TimeWindow(1, 2), new TimeWindow(2, 3), new TimeWindow(4, 5), new TimeWindow(5, 6)), callback); verify(callback, times(1)).merge( (Collection<TimeWindow>) argThat(containsInAnyOrder(new TimeWindow(0, 1), new TimeWindow(1, 2), new TimeWindow(2, 3))), eq(new TimeWindow(0, 3))); verify(callback, times(1)).merge( (Collection<TimeWindow>) argThat(containsInAnyOrder(new TimeWindow(4, 5), new TimeWindow(5, 6))), eq(new TimeWindow(4, 6))); verify(callback, times(2)).merge(anyCollection(), Matchers.anyObject()); }
Example #3
Source File: WindowTriangles.java From gelly-streaming with Apache License 2.0 | 6 votes |
@Override public void apply(Tuple key, TimeWindow window, Iterable<Tuple3<Long, Long, Boolean>> values, Collector<Tuple2<Integer, Long>> out) throws Exception { int candidates = 0; int edges = 0; for (Tuple3<Long, Long, Boolean> t: values) { if (t.f2) { // candidate candidates++; } else { edges++; } } if (edges > 0) { out.collect(new Tuple2<Integer, Long>(candidates, window.maxTimestamp())); } }
Example #4
Source File: DataStreamAllroundTestJobFactory.java From flink with Apache License 2.0 | 6 votes |
static WindowedStream<Event, Integer, TimeWindow> applyTumblingWindows( KeyedStream<Event, Integer> keyedStream, ParameterTool pt) { long eventTimeProgressPerEvent = pt.getLong( SEQUENCE_GENERATOR_SRC_EVENT_TIME_CLOCK_PROGRESS.key(), SEQUENCE_GENERATOR_SRC_EVENT_TIME_CLOCK_PROGRESS.defaultValue()); return keyedStream.timeWindow( Time.milliseconds( pt.getLong( TUMBLING_WINDOW_OPERATOR_NUM_EVENTS.key(), TUMBLING_WINDOW_OPERATOR_NUM_EVENTS.defaultValue() ) * eventTimeProgressPerEvent ) ); }
Example #5
Source File: WindowOperatorContractTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testAssignerWithMultipleWindows() throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); OneInputStreamOperatorTestHarness<Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(2, 4), new TimeWindow(0, 2))); shouldFireOnElement(mockTrigger); testHarness.processElement(new StreamRecord<>(0, 0L)); verify(mockWindowFunction, times(2)).process(eq(0), anyTimeWindow(), anyInternalWindowContext(), anyIntIterable(), WindowOperatorContractTest.<Void>anyCollector()); verify(mockWindowFunction, times(1)).process(eq(0), eq((new TimeWindow(0, 2))), anyInternalWindowContext(), intIterable(0), WindowOperatorContractTest.<Void>anyCollector()); verify(mockWindowFunction, times(1)).process(eq(0), eq(new TimeWindow(2, 4)), anyInternalWindowContext(), intIterable(0), WindowOperatorContractTest.<Void>anyCollector()); }
Example #6
Source File: CustomTrigger.java From flink-learning with Apache License 2.0 | 6 votes |
@Override public TriggerResult onElement(WordEvent element, long timestamp, TimeWindow window, TriggerContext ctx) throws Exception { log.info("======onElement====window start = {}, window end = {}", window.getStart(), window.getEnd()); return TriggerResult.CONTINUE; // ReducingState<Long> fireTimestamp = ctx.getPartitionedState(stateDesc); // // timestamp = ctx.getCurrentProcessingTime(); // // if (fireTimestamp.get() == null) { // long start = timestamp - (timestamp % interval); // long nextFireTimestamp = start + interval; // // ctx.registerProcessingTimeTimer(nextFireTimestamp); // // fireTimestamp.add(nextFireTimestamp); // return TriggerResult.CONTINUE; // } // return TriggerResult.CONTINUE; }
Example #7
Source File: MergingWindowSetTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Test merging of a large new window that covers one existing windows. */ @Test public void testMergeLargeWindowCoveringSingleWindow() throws Exception { @SuppressWarnings("unchecked") ListState<Tuple2<TimeWindow, TimeWindow>> mockState = mock(ListState.class); MergingWindowSet<TimeWindow> windowSet = new MergingWindowSet<>(EventTimeSessionWindows.withGap(Time.milliseconds(3)), mockState); TestingMergeFunction mergeFunction = new TestingMergeFunction(); // add an initial small window mergeFunction.reset(); assertEquals(new TimeWindow(1, 2), windowSet.addWindow(new TimeWindow(1, 2), mergeFunction)); assertFalse(mergeFunction.hasMerged()); assertEquals(new TimeWindow(1, 2), windowSet.getStateWindow(new TimeWindow(1, 2))); // add a new window that completely covers the existing window mergeFunction.reset(); assertEquals(new TimeWindow(0, 3), windowSet.addWindow(new TimeWindow(0, 3), mergeFunction)); assertTrue(mergeFunction.hasMerged()); assertEquals(new TimeWindow(1, 2), windowSet.getStateWindow(new TimeWindow(0, 3))); }
Example #8
Source File: MergingWindowSetTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testPersistOnlyIfHaveUpdates() throws Exception { @SuppressWarnings("unchecked") ListState<Tuple2<TimeWindow, TimeWindow>> mockState = mock(ListState.class); when(mockState.get()).thenReturn(Lists.newArrayList( new Tuple2<>(new TimeWindow(17, 42), new TimeWindow(42, 17)), new Tuple2<>(new TimeWindow(1, 2), new TimeWindow(3, 4)) )); MergingWindowSet<TimeWindow> windowSet = new MergingWindowSet<>(EventTimeSessionWindows.withGap(Time.milliseconds(3)), mockState); assertEquals(new TimeWindow(42, 17), windowSet.getStateWindow(new TimeWindow(17, 42))); assertEquals(new TimeWindow(3, 4), windowSet.getStateWindow(new TimeWindow(1, 2))); windowSet.persist(); verify(mockState, times(0)).add(Matchers.<Tuple2<TimeWindow, TimeWindow>>anyObject()); }
Example #9
Source File: SexApply.java From wingcloud with Apache License 2.0 | 6 votes |
@Override public void apply(Tuple tuple, TimeWindow window, Iterable<Tuple2<Long, String>> input, Collector<String> out) throws Exception { //获取分组字段 String usergender = tuple.getField(0).toString(); Iterator<Tuple2<Long, String>> it = input.iterator(); //计数器 Long count = 0L; //bigcount = 0L; while (it.hasNext()) { Tuple2<Long, String> next = it.next(); count++; } System.err.println(Thread.currentThread().getId()+"【男女比例】window触发了=数据条数:"+count); //组装结果 String res = "{\"usergender\":\""+usergender+"\",\"count\":\""+count+"\"}"; System.err.println("【男女比例】窗口数据 : "+ res); out.collect(res); }
Example #10
Source File: TimerSerializerTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") @Override protected TimerHeapInternalTimer<Long, TimeWindow>[] getTestData() { return (TimerHeapInternalTimer<Long, TimeWindow>[]) new TimerHeapInternalTimer[]{ new TimerHeapInternalTimer<>(42L, 4711L, new TimeWindow(1000L, 2000L)), new TimerHeapInternalTimer<>(0L, 0L, new TimeWindow(0L, 0L)), new TimerHeapInternalTimer<>(1L, -1L, new TimeWindow(1L, -1L)), new TimerHeapInternalTimer<>(-1L, 1L, new TimeWindow(-1L, 1L)), new TimerHeapInternalTimer<>(Long.MAX_VALUE, Long.MIN_VALUE, new TimeWindow(Long.MAX_VALUE, Long.MIN_VALUE)), new TimerHeapInternalTimer<>(Long.MIN_VALUE, Long.MAX_VALUE, new TimeWindow(Long.MIN_VALUE, Long.MAX_VALUE)) }; }
Example #11
Source File: WindowOperatorTest.java From flink with Apache License 2.0 | 5 votes |
@Override @SuppressWarnings("unchecked") public Collection<TimeWindow> assignWindows(Object element, long timestamp, WindowAssignerContext ctx) { if (element instanceof Tuple2) { Tuple2<String, Integer> t2 = (Tuple2<String, Integer>) element; if (t2.f1 == 33) { return Collections.singletonList(new TimeWindow(timestamp, timestamp)); } } return Collections.singletonList(new TimeWindow(timestamp, timestamp + sessionTimeout)); }
Example #12
Source File: AllWindowTranslationTest.java From flink with Apache License 2.0 | 5 votes |
@Test @SuppressWarnings("rawtypes") public void testApplyEventTime() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime); DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); DataStream<Tuple2<String, Integer>> window1 = source .windowAll(TumblingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS))) .apply(new AllWindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow>() { private static final long serialVersionUID = 1L; @Override public void apply( TimeWindow window, Iterable<Tuple2<String, Integer>> values, Collector<Tuple2<String, Integer>> out) throws Exception { for (Tuple2<String, Integer> in : values) { out.collect(in); } } }); OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation(); OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator(); Assert.assertTrue(operator instanceof WindowOperator); WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator; Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger); Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingEventTimeWindows); Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor); processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); }
Example #13
Source File: TumblingEventTimeWindowsTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testProperties() { TumblingEventTimeWindows assigner = TumblingEventTimeWindows.of(Time.seconds(5), Time.milliseconds(100)); assertTrue(assigner.isEventTime()); assertEquals(new TimeWindow.Serializer(), assigner.getWindowSerializer(new ExecutionConfig())); assertThat(assigner.getDefaultTrigger(mock(StreamExecutionEnvironment.class)), instanceOf(EventTimeTrigger.class)); }
Example #14
Source File: WindowOperatorContractTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testOnElementPurgeDoesNotCleanupMergingSet() throws Exception { MergingWindowAssigner<Integer, TimeWindow> mockAssigner = mockMergingAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, 2))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); doAnswer(new Answer<TriggerResult>() { @Override public TriggerResult answer(InvocationOnMock invocation) throws Exception { return TriggerResult.PURGE; } }).when(mockTrigger).onElement(Matchers.<Integer>anyObject(), anyLong(), anyTimeWindow(), anyTriggerContext()); testHarness.processElement(new StreamRecord<>(0, 0L)); assertEquals(1, testHarness.numKeyedStateEntries()); // the merging window set assertEquals(1, testHarness.numEventTimeTimers()); // one cleanup timer assertEquals(0, testHarness.getOutput().size()); }
Example #15
Source File: DynamicProcessingTimeSessionWindowsTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testMergeConsecutiveWindows() { MergingWindowAssigner.MergeCallback callback = mock(MergingWindowAssigner.MergeCallback.class); SessionWindowTimeGapExtractor extractor = mock(SessionWindowTimeGapExtractor.class); when(extractor.extract(any())).thenReturn(5000L); DynamicProcessingTimeSessionWindows assigner = DynamicProcessingTimeSessionWindows.withDynamicGap(extractor); assigner.mergeWindows( Lists.newArrayList( new TimeWindow(0, 1), new TimeWindow(1, 2), new TimeWindow(2, 3), new TimeWindow(4, 5), new TimeWindow(5, 6)), callback); verify(callback, times(1)).merge( (Collection<TimeWindow>) argThat(containsInAnyOrder(new TimeWindow(0, 1), new TimeWindow(1, 2), new TimeWindow(2, 3))), eq(new TimeWindow(0, 3))); verify(callback, times(1)).merge( (Collection<TimeWindow>) argThat(containsInAnyOrder(new TimeWindow(4, 5), new TimeWindow(5, 6))), eq(new TimeWindow(4, 6))); verify(callback, times(2)).merge(anyCollection(), Matchers.anyObject()); }
Example #16
Source File: TimeWindowTranslationTest.java From flink with Apache License 2.0 | 5 votes |
/** * Verifies that calls to timeWindow() instantiate a regular * windowOperator instead of an aligned one. */ @Test public void testAlignedWindowDeprecation() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime); DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); DummyReducer reducer = new DummyReducer(); DataStream<Tuple2<String, Integer>> window1 = source .keyBy(0) .timeWindow(Time.of(1000, TimeUnit.MILLISECONDS), Time.of(100, TimeUnit.MILLISECONDS)) .reduce(reducer); OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform1 = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation(); OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator1 = transform1.getOperator(); Assert.assertTrue(operator1 instanceof WindowOperator); DataStream<Tuple2<String, Integer>> window2 = source .keyBy(0) .timeWindow(Time.of(1000, TimeUnit.MILLISECONDS)) .apply(new WindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple, TimeWindow>() { private static final long serialVersionUID = 1L; @Override public void apply(Tuple tuple, TimeWindow window, Iterable<Tuple2<String, Integer>> values, Collector<Tuple2<String, Integer>> out) throws Exception { } }); OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform2 = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window2.getTransformation(); OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator2 = transform2.getOperator(); Assert.assertTrue(operator2 instanceof WindowOperator); }
Example #17
Source File: WindowOperatorTest.java From flink with Apache License 2.0 | 5 votes |
@Override public void process(String key, Context context, Iterable<Tuple2<String, Integer>> values, Collector<Tuple3<String, Long, Long>> out) throws Exception { int sum = 0; for (Tuple2<String, Integer> i: values) { sum += i.f1; } String resultString = key + "-" + sum; TimeWindow window = context.window(); out.collect(new Tuple3<>(resultString, window.getStart(), window.getEnd())); }
Example #18
Source File: WindowOperatorTest.java From flink with Apache License 2.0 | 5 votes |
@Override public void process(String key, Context context, Iterable<Tuple2<String, Integer>> values, Collector<Tuple3<String, Long, Long>> out) throws Exception { int sum = 0; for (Tuple2<String, Integer> i: values) { sum += i.f1; } String resultString = key + "-" + sum; TimeWindow window = context.window(); out.collect(new Tuple3<>(resultString, window.getStart(), window.getEnd())); }
Example #19
Source File: WindowOperatorContractTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testProcessingElementsWithinAllowedLateness() throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, 2))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); shouldFireOnElement(mockTrigger); // 20 is just at the limit, window.maxTime() is 1 and allowed lateness is 20 testHarness.processWatermark(new Watermark(20)); testHarness.processElement(new StreamRecord<>(0, 0L)); verify(mockWindowFunction, times(1)).process(eq(0), eq(new TimeWindow(0, 2)), anyInternalWindowContext(), intIterable(0), WindowOperatorContractTest.<Void>anyCollector()); // clear is only called at cleanup time/GC time verify(mockTrigger, never()).clear(anyTimeWindow(), anyTriggerContext()); // FIRE should not purge contents assertEquals(1, testHarness.numKeyedStateEntries()); // window contents plus trigger state assertEquals(1, testHarness.numEventTimeTimers()); // just the GC timer }
Example #20
Source File: WindowTranslationTest.java From flink with Apache License 2.0 | 5 votes |
@Test @SuppressWarnings("rawtypes") public void testReduceWithProcessWindowFunctionProcessingTime() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime); DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); DataStream<Tuple3<String, String, Integer>> window = source .keyBy(new TupleKeySelector()) .window(TumblingProcessingTimeWindows.of(Time.of(1, TimeUnit.SECONDS))) .reduce(new DummyReducer(), new ProcessWindowFunction<Tuple2<String, Integer>, Tuple3<String, String, Integer>, String, TimeWindow>() { private static final long serialVersionUID = 1L; @Override public void process(String tuple, Context ctx, Iterable<Tuple2<String, Integer>> values, Collector<Tuple3<String, String, Integer>> out) throws Exception { for (Tuple2<String, Integer> in : values) { out.collect(new Tuple3<>(in.f0, in.f0, in.f1)); } } }); OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>>) window.getTransformation(); OneInputStreamOperator<Tuple2<String, Integer>, Tuple3<String, String, Integer>> operator = transform.getOperator(); Assert.assertTrue(operator instanceof WindowOperator); WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator; Assert.assertTrue(winOperator.getTrigger() instanceof ProcessingTimeTrigger); Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingProcessingTimeWindows); Assert.assertTrue(winOperator.getStateDescriptor() instanceof ReducingStateDescriptor); processElementAndEnsureOutput(operator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); }
Example #21
Source File: JoinedStreamsTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testDelegateToCoGrouped() { Time lateness = Time.milliseconds(42L); JoinedStreams.WithWindow<String, String, String, TimeWindow> withLateness = dataStream1 .join(dataStream2) .where(keySelector) .equalTo(keySelector) .window(tsAssigner) .allowedLateness(lateness); withLateness.apply(joinFunction, BasicTypeInfo.STRING_TYPE_INFO); Assert.assertEquals(lateness.toMilliseconds(), withLateness.getCoGroupedWindowedStream().getAllowedLateness().toMilliseconds()); }
Example #22
Source File: WindowOperatorContractTest.java From flink with Apache License 2.0 | 5 votes |
private static <T> void shouldRegisterProcessingTimeTimerOnElement(Trigger<T, TimeWindow> mockTrigger, final long timestamp) throws Exception { doAnswer(new Answer<TriggerResult>() { @Override public TriggerResult answer(InvocationOnMock invocation) throws Exception { @SuppressWarnings("unchecked") Trigger.TriggerContext context = (Trigger.TriggerContext) invocation.getArguments()[3]; context.registerProcessingTimeTimer(timestamp); return TriggerResult.CONTINUE; } }) .when(mockTrigger).onElement(Matchers.<T>anyObject(), anyLong(), anyTimeWindow(), anyTriggerContext()); }
Example #23
Source File: WindowTranslationTest.java From flink with Apache License 2.0 | 5 votes |
@Test @SuppressWarnings("rawtypes") public void testReduceWithProcesWindowFunctionEventTime() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime); DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); DummyReducer reducer = new DummyReducer(); DataStream<Tuple3<String, String, Integer>> window = source .keyBy(new TupleKeySelector()) .window(TumblingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS))) .reduce(reducer, new ProcessWindowFunction<Tuple2<String, Integer>, Tuple3<String, String, Integer>, String, TimeWindow>() { private static final long serialVersionUID = 1L; @Override public void process(String key, Context ctx, Iterable<Tuple2<String, Integer>> values, Collector<Tuple3<String, String, Integer>> out) throws Exception { for (Tuple2<String, Integer> in : values) { out.collect(new Tuple3<>(in.f0, in.f0, in.f1)); } } }); OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>>) window.getTransformation(); OneInputStreamOperator<Tuple2<String, Integer>, Tuple3<String, String, Integer>> operator = transform.getOperator(); Assert.assertTrue(operator instanceof WindowOperator); WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator; Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger); Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingEventTimeWindows); Assert.assertTrue(winOperator.getStateDescriptor() instanceof ReducingStateDescriptor); processElementAndEnsureOutput(operator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); }
Example #24
Source File: EventTimeTriggerTest.java From flink with Apache License 2.0 | 5 votes |
/** * Merging a late window should not register a timer, otherwise we would get two firings: * one from onElement() on the merged window and one from the timer. */ @Test public void testMergingLateWindows() throws Exception { TriggerTestHarness<Object, TimeWindow> testHarness = new TriggerTestHarness<>(EventTimeTrigger.create(), new TimeWindow.Serializer()); assertTrue(EventTimeTrigger.create().canMerge()); assertEquals(TriggerResult.CONTINUE, testHarness.processElement(new StreamRecord<Object>(1), new TimeWindow(0, 2))); assertEquals(TriggerResult.CONTINUE, testHarness.processElement(new StreamRecord<Object>(1), new TimeWindow(2, 4))); assertEquals(0, testHarness.numStateEntries()); assertEquals(0, testHarness.numProcessingTimeTimers()); assertEquals(2, testHarness.numEventTimeTimers()); assertEquals(1, testHarness.numEventTimeTimers(new TimeWindow(0, 2))); assertEquals(1, testHarness.numEventTimeTimers(new TimeWindow(2, 4))); testHarness.advanceWatermark(10); assertEquals(0, testHarness.numStateEntries()); assertEquals(0, testHarness.numProcessingTimeTimers()); assertEquals(0, testHarness.numEventTimeTimers()); assertEquals(0, testHarness.numEventTimeTimers(new TimeWindow(0, 2))); assertEquals(0, testHarness.numEventTimeTimers(new TimeWindow(2, 4))); testHarness.mergeWindows(new TimeWindow(0, 4), Lists.newArrayList(new TimeWindow(0, 2), new TimeWindow(2, 4))); assertEquals(0, testHarness.numStateEntries()); assertEquals(0, testHarness.numProcessingTimeTimers()); assertEquals(0, testHarness.numEventTimeTimers()); assertEquals(0, testHarness.numEventTimeTimers(new TimeWindow(0, 2))); assertEquals(0, testHarness.numEventTimeTimers(new TimeWindow(2, 4))); assertEquals(0, testHarness.numEventTimeTimers(new TimeWindow(0, 4))); }
Example #25
Source File: AllWindowTranslationTest.java From flink with Apache License 2.0 | 5 votes |
@Test @SuppressWarnings("rawtypes") public void testApplyWithEvictor() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime); DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); DataStream<Tuple2<String, Integer>> window1 = source .windowAll(TumblingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS))) .trigger(CountTrigger.of(1)) .evictor(TimeEvictor.of(Time.of(100, TimeUnit.MILLISECONDS))) .apply(new AllWindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow>() { private static final long serialVersionUID = 1L; @Override public void apply( TimeWindow window, Iterable<Tuple2<String, Integer>> values, Collector<Tuple2<String, Integer>> out) throws Exception { for (Tuple2<String, Integer> in : values) { out.collect(in); } } }); OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation(); OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator(); Assert.assertTrue(operator instanceof EvictingWindowOperator); EvictingWindowOperator<String, Tuple2<String, Integer>, ?, ?> winOperator = (EvictingWindowOperator<String, Tuple2<String, Integer>, ?, ?>) operator; Assert.assertTrue(winOperator.getTrigger() instanceof CountTrigger); Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingEventTimeWindows); Assert.assertTrue(winOperator.getEvictor() instanceof TimeEvictor); Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor); processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); }
Example #26
Source File: WindowOperatorContractTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testNoEventTimeGarbageCollectionTimerForLongMax() throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, Long.MAX_VALUE - 10))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); testHarness.processElement(new StreamRecord<>(0, 0L)); // just the window contents assertEquals(1, testHarness.numKeyedStateEntries()); // no GC timer assertEquals(0, testHarness.numEventTimeTimers()); assertEquals(0, testHarness.numProcessingTimeTimers()); }
Example #27
Source File: EventTimeTriggerTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testMergingWindows() throws Exception { TriggerTestHarness<Object, TimeWindow> testHarness = new TriggerTestHarness<>(EventTimeTrigger.create(), new TimeWindow.Serializer()); assertTrue(EventTimeTrigger.create().canMerge()); assertEquals(TriggerResult.CONTINUE, testHarness.processElement(new StreamRecord<Object>(1), new TimeWindow(0, 2))); assertEquals(TriggerResult.CONTINUE, testHarness.processElement(new StreamRecord<Object>(1), new TimeWindow(2, 4))); assertEquals(0, testHarness.numStateEntries()); assertEquals(0, testHarness.numProcessingTimeTimers()); assertEquals(2, testHarness.numEventTimeTimers()); assertEquals(1, testHarness.numEventTimeTimers(new TimeWindow(0, 2))); assertEquals(1, testHarness.numEventTimeTimers(new TimeWindow(2, 4))); testHarness.mergeWindows(new TimeWindow(0, 4), Lists.newArrayList(new TimeWindow(0, 2), new TimeWindow(2, 4))); assertEquals(0, testHarness.numStateEntries()); assertEquals(0, testHarness.numProcessingTimeTimers()); assertEquals(1, testHarness.numEventTimeTimers()); assertEquals(0, testHarness.numEventTimeTimers(new TimeWindow(0, 2))); assertEquals(0, testHarness.numEventTimeTimers(new TimeWindow(2, 4))); assertEquals(1, testHarness.numEventTimeTimers(new TimeWindow(0, 4))); assertEquals(TriggerResult.FIRE, testHarness.advanceWatermark(4, new TimeWindow(0, 4))); assertEquals(0, testHarness.numStateEntries()); assertEquals(0, testHarness.numProcessingTimeTimers()); assertEquals(0, testHarness.numEventTimeTimers()); }
Example #28
Source File: WindowTranslationTest.java From flink with Apache License 2.0 | 5 votes |
@Test @SuppressWarnings("rawtypes") public void testProcessWithEvictor() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime); DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); DataStream<Tuple2<String, Integer>> window1 = source .keyBy(new TupleKeySelector()) .window(TumblingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS))) .trigger(CountTrigger.of(1)) .evictor(TimeEvictor.of(Time.of(100, TimeUnit.MILLISECONDS))) .process(new ProcessWindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, String, TimeWindow>() { private static final long serialVersionUID = 1L; @Override public void process(String key, Context ctx, Iterable<Tuple2<String, Integer>> values, Collector<Tuple2<String, Integer>> out) throws Exception { for (Tuple2<String, Integer> in : values) { out.collect(in); } } }); OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation(); OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator(); Assert.assertTrue(operator instanceof EvictingWindowOperator); EvictingWindowOperator<String, Tuple2<String, Integer>, ?, ?> winOperator = (EvictingWindowOperator<String, Tuple2<String, Integer>, ?, ?>) operator; Assert.assertTrue(winOperator.getTrigger() instanceof CountTrigger); Assert.assertTrue(winOperator.getEvictor() instanceof TimeEvictor); Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingEventTimeWindows); Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor); processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); }
Example #29
Source File: DynamicProcessingTimeSessionWindowsTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testProperties() { SessionWindowTimeGapExtractor extractor = mock(SessionWindowTimeGapExtractor.class); when(extractor.extract(any())).thenReturn(5000L); DynamicProcessingTimeSessionWindows assigner = DynamicProcessingTimeSessionWindows.withDynamicGap(extractor); assertFalse(assigner.isEventTime()); assertEquals(new TimeWindow.Serializer(), assigner.getWindowSerializer(new ExecutionConfig())); assertThat(assigner.getDefaultTrigger(mock(StreamExecutionEnvironment.class)), instanceOf(ProcessingTimeTrigger.class)); }
Example #30
Source File: WindowTranslationTest.java From flink with Apache License 2.0 | 5 votes |
@Test @SuppressWarnings("rawtypes") public void testProcessWithCustomTrigger() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime); DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); DataStream<Tuple2<String, Integer>> window1 = source .keyBy(new TupleKeySelector()) .window(TumblingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS))) .trigger(CountTrigger.of(1)) .process(new ProcessWindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, String, TimeWindow>() { private static final long serialVersionUID = 1L; @Override public void process(String key, Context ctx, Iterable<Tuple2<String, Integer>> values, Collector<Tuple2<String, Integer>> out) throws Exception { for (Tuple2<String, Integer> in : values) { out.collect(in); } } }); OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation(); OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator(); Assert.assertTrue(operator instanceof WindowOperator); WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator; Assert.assertTrue(winOperator.getTrigger() instanceof CountTrigger); Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingEventTimeWindows); Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor); processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); }