org.apache.flink.api.common.state.ListStateDescriptor Java Examples
The following examples show how to use
org.apache.flink.api.common.state.ListStateDescriptor.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: StateBackendTestBase.java From flink with Apache License 2.0 | 6 votes |
/** * This test verifies that all ListState implementations are consistent in not allowing * adding {@code null}. */ @Test public void testListStateAddNull() throws Exception { AbstractKeyedStateBackend<String> keyedBackend = createKeyedBackend(StringSerializer.INSTANCE); final ListStateDescriptor<Long> stateDescr = new ListStateDescriptor<>("my-state", Long.class); try { ListState<Long> state = keyedBackend.getPartitionedState( VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, stateDescr); keyedBackend.setCurrentKey("abc"); assertNull(state.get()); expectedException.expect(NullPointerException.class); state.add(null); } finally { keyedBackend.close(); keyedBackend.dispose(); } }
Example #2
Source File: StateBackendMigrationTestBase.java From flink with Apache License 2.0 | 6 votes |
@Test public void testKeyedListStateRegistrationFailsIfNewStateSerializerIsIncompatible() { final String stateName = "test-name"; try { testKeyedListStateUpgrade( new ListStateDescriptor<>( stateName, new TestType.V1TestTypeSerializer()), new ListStateDescriptor<>( stateName, new TestType.IncompatibleTestTypeSerializer())); fail("should have failed"); } catch (Exception expected) { Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent()); } }
Example #3
Source File: StateBackendTestBase.java From flink with Apache License 2.0 | 6 votes |
@Test public void testApplyToAllKeysLambdaFunction() throws Exception { AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE); try { ListStateDescriptor<String> listStateDescriptor = new ListStateDescriptor<>("foo", StringSerializer.INSTANCE); ListState<String> listState = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, listStateDescriptor); for (int i = 0; i < 100; ++i) { backend.setCurrentKey(i); listState.add("Hello" + i); } // valid state value via applyToAllKeys(). backend.applyToAllKeys(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, listStateDescriptor, (Integer key, ListState<String> state) -> assertEquals("Hello" + key, state.get().iterator().next()) ); } finally { IOUtils.closeQuietly(backend); backend.dispose(); } }
Example #4
Source File: KeyedStateInputFormat.java From flink with Apache License 2.0 | 6 votes |
private Context(AbstractKeyedStateBackend<K> keyedStateBackend, InternalTimerService<VoidNamespace> timerService) throws Exception { eventTimers = keyedStateBackend.getPartitionedState( USER_TIMERS_NAME, StringSerializer.INSTANCE, new ListStateDescriptor<>(EVENT_TIMER_STATE, Types.LONG) ); timerService.forEachEventTimeTimer((namespace, timer) -> { if (namespace.equals(VoidNamespace.INSTANCE)) { eventTimers.add(timer); } }); procTimers = keyedStateBackend.getPartitionedState( USER_TIMERS_NAME, StringSerializer.INSTANCE, new ListStateDescriptor<>(PROC_TIMER_STATE, Types.LONG) ); timerService.forEachProcessingTimeTimer((namespace, timer) -> { if (namespace.equals(VoidNamespace.INSTANCE)) { procTimers.add(timer); } }); }
Example #5
Source File: AllWindowTranslationTest.java From flink with Apache License 2.0 | 6 votes |
@Test @SuppressWarnings("rawtypes") public void testMergingWindowsWithEvictor() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime); DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); DataStream<Tuple3<String, String, Integer>> window1 = source .windowAll(EventTimeSessionWindows.withGap(Time.seconds(5))) .evictor(CountEvictor.of(5)) .process(new TestProcessAllWindowFunction()); OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>>) window1.getTransformation(); OneInputStreamOperator<Tuple2<String, Integer>, Tuple3<String, String, Integer>> operator = transform.getOperator(); Assert.assertTrue(operator instanceof WindowOperator); WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator; Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger); Assert.assertTrue(winOperator.getWindowAssigner() instanceof EventTimeSessionWindows); Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor); processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); }
Example #6
Source File: SavepointWriterITCase.java From flink with Apache License 2.0 | 6 votes |
@Override public void initializeState(FunctionInitializationContext context) throws Exception { state = context.getOperatorStateStore().getUnionListState( new ListStateDescriptor<>("numbers", Types.INT) ); if (context.isRestored()) { Set<Integer> expected = new HashSet<>(); expected.add(1); expected.add(2); expected.add(3); for (Integer number : state.get()) { Assert.assertTrue("Duplicate state", expected.contains(number)); expected.remove(number); } Assert.assertTrue("Failed to bootstrap all state elements: " + Arrays.toString(expected.toArray()), expected.isEmpty()); } }
Example #7
Source File: AbstractDynamicParallelSource.java From alibaba-flink-connectors with Apache License 2.0 | 6 votes |
@Override public void initializeState(FunctionInitializationContext context) throws Exception { LOG.info("initializeState"); ParameterizedType p = (ParameterizedType) this.getClass().getGenericSuperclass(); TypeInformation type0 = TypeExtractor.createTypeInfo(InputSplit.class); TypeInformation type1 = TypeExtractor.createTypeInfo(p.getActualTypeArguments()[1]); // TypeInformation<Tuple2<InputSplit, CURSOR>> stateTypeInfo = new TupleTypeInfo<>(type0, type1); List<PojoField> pojoFields = new ArrayList<>(); pojoFields.add(new PojoField(InnerProgress.class.getField("inputSplit"), type0)); pojoFields.add(new PojoField(InnerProgress.class.getField("cursor"), type1)); TypeInformation<InnerProgress> stateTypeInfo = new PojoTypeInfo<>(InnerProgress.class, pojoFields); // ListStateDescriptor<Tuple2<InputSplit, CURSOR>> descriptor = new ListStateDescriptor<>(SOURCE_STATE_NAME, stateTypeInfo); ListStateDescriptor<InnerProgress<CURSOR>> descriptor = new ListStateDescriptor(SOURCE_STATE_NAME, stateTypeInfo); unionInitialProgress = context.getOperatorStateStore().getUnionListState(descriptor); LOG.info("Restoring state: {}", unionInitialProgress); allSplitsInCP = new ArrayList<>(); if (context.isRestored()) { recoryFromState = true; for (InnerProgress progress: unionInitialProgress.get()){ allSplitsInCP.add(new InnerProgress(progress.inputSplit, progress.cursor)); } } }
Example #8
Source File: WindowOperatorTest.java From flink with Apache License 2.0 | 6 votes |
@Test @SuppressWarnings("unchecked") public void testTumblingEventTimeWindowsApply() throws Exception { closeCalled.set(0); final int windowSize = 3; ListStateDescriptor<Tuple2<String, Integer>> stateDesc = new ListStateDescriptor<>("window-contents", STRING_INT_TUPLE.createSerializer(new ExecutionConfig())); WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>( TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)), new TimeWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), stateDesc, new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>()), EventTimeTrigger.create(), 0, null /* late data output tag */); testTumblingEventTimeWindows(operator); // we close once in the rest... Assert.assertEquals("Close was not called.", 2, closeCalled.get()); }
Example #9
Source File: SavepointWriterITCase.java From flink with Apache License 2.0 | 6 votes |
@Override public void initializeState(FunctionInitializationContext context) throws Exception { state = context.getOperatorStateStore().getUnionListState( new ListStateDescriptor<>("numbers", Types.INT) ); if (context.isRestored()) { Set<Integer> expected = new HashSet<>(); expected.add(1); expected.add(2); expected.add(3); for (Integer number : state.get()) { Assert.assertTrue("Duplicate state", expected.contains(number)); expected.remove(number); } Assert.assertTrue("Failed to bootstrap all state elements: " + Arrays.toString(expected.toArray()), expected.isEmpty()); } }
Example #10
Source File: HeavyDeploymentStressTestProgram.java From flink with Apache License 2.0 | 6 votes |
@Override public void initializeState(FunctionInitializationContext context) throws Exception { readyToFail = false; if (context.isRestored()) { isRunning = false; } else { isRunning = true; OperatorStateStore operatorStateStore = context.getOperatorStateStore(); for (int i = 0; i < numListStates; ++i) { ListStateDescriptor<String> listStateDescriptor = new ListStateDescriptor<>("test-list-state-" + i, String.class); ListState<String> unionListState = operatorStateStore.getUnionListState(listStateDescriptor); for (int j = 0; j < numPartitionsPerListState; ++j) { unionListState.add(String.valueOf(j)); } } } }
Example #11
Source File: DataStreamAllroundTestJobFactory.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
private static <IN, STATE> ArtificialStateBuilder<IN> createListStateBuilder( JoinFunction<IN, STATE, STATE> inputAndOldStateToNewState, ListStateDescriptor<STATE> listStateDescriptor) { JoinFunction<IN, Iterable<STATE>, List<STATE>> listStateGenerator = (first, second) -> { List<STATE> newState = new ArrayList<>(); for (STATE s : second) { newState.add(inputAndOldStateToNewState.join(first, s)); } return newState; }; return new ArtificialListStateBuilder<>( listStateDescriptor.getName(), listStateGenerator, listStateGenerator, listStateDescriptor); }
Example #12
Source File: FlinkLogConsumer.java From aliyun-log-flink-connector with Apache License 2.0 | 6 votes |
@Override public void initializeState(FunctionInitializationContext context) throws Exception { LOG.debug("Initializing state from Flink state"); TypeInformation<Tuple2<LogstoreShardMeta, String>> shardsStateTypeInfo = new TupleTypeInfo<Tuple2<LogstoreShardMeta, String>>( TypeInformation.of(LogstoreShardMeta.class), TypeInformation.of(String.class)); cursorStateForCheckpoint = context.getOperatorStateStore().getUnionListState( new ListStateDescriptor<>(CURSOR_STATE_STORE_NAME, shardsStateTypeInfo)); if (!context.isRestored()) { LOG.info("No state restored for FlinkLogConsumer."); return; } if (cursorsToRestore != null) { LOG.info("Flink state has been restored already."); return; } createClientIfNeeded(); cursorsToRestore = new HashMap<>(); for (Tuple2<LogstoreShardMeta, String> cursor : cursorStateForCheckpoint.get()) { final LogstoreShardMeta shardMeta = cursor.f0; final String checkpoint = cursor.f1; cursorsToRestore.put(shardMeta, checkpoint); } LOG.info("The following offsets restored from Flink state: {}", cursorsToRestore); }
Example #13
Source File: StateBackendMigrationTestBase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testKeyedListStateRegistrationFailsIfNewStateSerializerIsIncompatible() throws Exception { final String stateName = "test-name"; try { testKeyedListStateUpgrade( new ListStateDescriptor<>( stateName, new TestType.V1TestTypeSerializer()), new ListStateDescriptor<>( stateName, new TestType.IncompatibleTestTypeSerializer())); Assert.fail("should have failed"); } catch (Exception expected) { Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent()); } }
Example #14
Source File: StateBackendTestBase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * This test verifies that all ListState implementations are consistent in not allowing * {@link ListState#addAll(List)} to be called with {@code null}. */ @Test public void testListStateAddAllNull() throws Exception { AbstractKeyedStateBackend<String> keyedBackend = createKeyedBackend(StringSerializer.INSTANCE); final ListStateDescriptor<Long> stateDescr = new ListStateDescriptor<>("my-state", Long.class); try { ListState<Long> state = keyedBackend.getPartitionedState( VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, stateDescr); keyedBackend.setCurrentKey("abc"); assertNull(state.get()); expectedException.expect(NullPointerException.class); state.addAll(null); } finally { keyedBackend.close(); keyedBackend.dispose(); } }
Example #15
Source File: StateBackendTestBase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testApplyToAllKeysLambdaFunction() throws Exception { AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE); try { ListStateDescriptor<String> listStateDescriptor = new ListStateDescriptor<>("foo", StringSerializer.INSTANCE); ListState<String> listState = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, listStateDescriptor); for (int i = 0; i < 100; ++i) { backend.setCurrentKey(i); listState.add("Hello" + i); } // valid state value via applyToAllKeys(). backend.applyToAllKeys(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, listStateDescriptor, (Integer key, ListState<String> state) -> assertEquals("Hello" + key, state.get().iterator().next()) ); } finally { IOUtils.closeQuietly(backend); backend.dispose(); } }
Example #16
Source File: StateBackendTestBase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Verify that an empty {@code ListState} yields {@code null}. */ @Test public void testListStateDefaultValue() throws Exception { AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE); ListStateDescriptor<String> kvId = new ListStateDescriptor<>("id", String.class); ListState<String> state = backend.getPartitionedState( VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId); backend.setCurrentKey(1); assertNull(state.get()); state.update(Arrays.asList("Ciao", "Bello")); assertThat(state.get(), containsInAnyOrder("Ciao", "Bello")); state.clear(); assertNull(state.get()); backend.dispose(); }
Example #17
Source File: DefaultKeyedStateStore.java From flink with Apache License 2.0 | 5 votes |
@Override public <T> ListState<T> getListState(ListStateDescriptor<T> stateProperties) { requireNonNull(stateProperties, "The state properties must not be null"); try { stateProperties.initializeSerializerUnlessSet(executionConfig); ListState<T> originalState = getPartitionedState(stateProperties); return new UserFacingListState<>(originalState); } catch (Exception e) { throw new RuntimeException("Error while getting state", e); } }
Example #18
Source File: RestoreStreamTaskTest.java From flink with Apache License 2.0 | 5 votes |
@Override public void initializeState(StateInitializationContext context) throws Exception { super.initializeState(context); counterState = context .getOperatorStateStore() .getListState(new ListStateDescriptor<>("counter-state", LongSerializer.INSTANCE)); if (context.isRestored()) { for (Long value : counterState.get()) { counter += value; } counterState.clear(); } }
Example #19
Source File: AllWindowTranslationTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testAggregateWithEvictorAndProcessFunction() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime); DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); DataStream<Tuple2<String, Integer>> window1 = source .windowAll(SlidingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS))) .evictor(CountEvictor.of(100)) .aggregate( new DummyAggregationFunction(), new ProcessAllWindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow>() { @Override public void process( Context context, Iterable<Tuple2<String, Integer>> elements, Collector<Tuple2<String, Integer>> out) throws Exception { for (Tuple2<String, Integer> in : elements) { out.collect(in); } } }); OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation(); OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator(); Assert.assertTrue(operator instanceof WindowOperator); WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator; Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger); Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingEventTimeWindows); Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor); processElementAndEnsureOutput( winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); }
Example #20
Source File: RestoreStreamTaskTest.java From flink with Apache License 2.0 | 5 votes |
@Override public void initializeState(StateInitializationContext context) throws Exception { super.initializeState(context); counterState = context .getOperatorStateStore() .getListState(new ListStateDescriptor<>("counter-state", LongSerializer.INSTANCE)); if (context.isRestored()) { for (Long value : counterState.get()) { counter += value; } counterState.clear(); } }
Example #21
Source File: StateBackendMigrationTestBase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testOperatorUnionListStateSerializerReconfiguration() throws Exception { final String stateName = "union-list-state"; testOperatorUnionListStateUpgrade( new ListStateDescriptor<>( stateName, new TestType.V1TestTypeSerializer()), new ListStateDescriptor<>( stateName, // restore with a new serializer that requires reconfiguration new TestType.ReconfigurationRequiringTestTypeSerializer())); }
Example #22
Source File: StateBackendMigrationTestBase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testOperatorParitionableListStateSerializerReconfiguration() throws Exception { final String stateName = "partitionable-list-state"; testOperatorPartitionableListStateUpgrade( new ListStateDescriptor<>( stateName, new TestType.V1TestTypeSerializer()), new ListStateDescriptor<>( stateName, // restore with a new serializer that requires reconfiguration new TestType.ReconfigurationRequiringTestTypeSerializer())); }
Example #23
Source File: StateBackendMigrationTestBase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testOperatorParitionableListStateMigration() throws Exception { final String stateName = "partitionable-list-state"; testOperatorPartitionableListStateUpgrade( new ListStateDescriptor<>( stateName, new TestType.V1TestTypeSerializer()), new ListStateDescriptor<>( stateName, // restore with a V2 serializer that has a different schema new TestType.V2TestTypeSerializer())); }
Example #24
Source File: ReinterpretDataStreamAsKeyedStreamITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public void initializeState(FunctionInitializationContext context) throws Exception { canFail = false; position = 0L; isRestored = context.isRestored(); positionState = context.getOperatorStateStore().getListState( new ListStateDescriptor<>("posState", Long.class)); if (isRestored) { for (long value : positionState.get()) { position += value; } } }
Example #25
Source File: StateBackendMigrationTestBase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testKeyedListStateSerializerReconfiguration() throws Exception { final String stateName = "test-name"; testKeyedListStateUpgrade( new ListStateDescriptor<>( stateName, new TestType.V1TestTypeSerializer()), new ListStateDescriptor<>( stateName, // the test fails if this serializer is used instead of a reconfigured new serializer new TestType.ReconfigurationRequiringTestTypeSerializer())); }
Example #26
Source File: WindowOperatorTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testCleanupTimerWithEmptyListStateForSessionWindows() throws Exception { final int gapSize = 3; final long lateness = 10; ListStateDescriptor<Tuple2<String, Integer>> windowStateDesc = new ListStateDescriptor<>("window-contents", STRING_INT_TUPLE.createSerializer(new ExecutionConfig())); WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>( EventTimeSessionWindows.withGap(Time.seconds(gapSize)), new TimeWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), windowStateDesc, new InternalIterableWindowFunction<>(new PassThroughFunction()), EventTimeTrigger.create(), lateness, null /* late data output tag */); OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness = createTestHarness(operator); testHarness.open(); ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>(); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000)); testHarness.processWatermark(new Watermark(4998)); expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 3999)); expected.add(new Watermark(4998)); testHarness.processWatermark(new Watermark(14600)); expected.add(new Watermark(14600)); ConcurrentLinkedQueue<Object> actual = testHarness.getOutput(); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple2ResultSortComparator()); testHarness.close(); }
Example #27
Source File: AllWindowTranslationTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testAggregateWithEvictor() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime); DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); DataStream<Tuple2<String, Integer>> window1 = source .windowAll(SlidingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS))) .evictor(CountEvictor.of(100)) .aggregate(new DummyAggregationFunction()); OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation(); OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator(); Assert.assertTrue(operator instanceof WindowOperator); WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator; Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger); Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingEventTimeWindows); Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor); processElementAndEnsureOutput( winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); }
Example #28
Source File: WindowTranslationTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test @SuppressWarnings("rawtypes") public void testReduceWithEvictor() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime); DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); DummyReducer reducer = new DummyReducer(); DataStream<Tuple2<String, Integer>> window1 = source .keyBy(0) .window(SlidingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS))) .evictor(CountEvictor.of(100)) .reduce(reducer); OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation(); OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator(); Assert.assertTrue(operator instanceof EvictingWindowOperator); EvictingWindowOperator<String, Tuple2<String, Integer>, ?, ?> winOperator = (EvictingWindowOperator<String, Tuple2<String, Integer>, ?, ?>) operator; Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger); Assert.assertTrue(winOperator.getEvictor() instanceof CountEvictor); Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingEventTimeWindows); Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor); processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); }
Example #29
Source File: AllWindowTranslationTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test @SuppressWarnings("rawtypes") public void testProcessProcessingTime() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime); DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); DataStream<Tuple2<String, Integer>> window1 = source .windowAll(TumblingProcessingTimeWindows.of(Time.of(1, TimeUnit.SECONDS))) .process(new ProcessAllWindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow>() { private static final long serialVersionUID = 1L; @Override public void process( Context ctx, Iterable<Tuple2<String, Integer>> values, Collector<Tuple2<String, Integer>> out) throws Exception { for (Tuple2<String, Integer> in : values) { out.collect(in); } } }); OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation(); OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator(); Assert.assertTrue(operator instanceof WindowOperator); WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator; Assert.assertTrue(winOperator.getTrigger() instanceof ProcessingTimeTrigger); Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingProcessingTimeWindows); Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor); processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); }
Example #30
Source File: WindowTranslationTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testAggregateWithEvictor() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime); DataStream<Tuple3<String, String, Integer>> source = env.fromElements( Tuple3.of("hello", "hallo", 1), Tuple3.of("hello", "hallo", 2)); DataStream<Integer> window1 = source .keyBy(new Tuple3KeySelector()) .window(SlidingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS))) .evictor(CountEvictor.of(100)) .aggregate(new DummyAggregationFunction()); final OneInputTransformation<Tuple3<String, String, Integer>, Integer> transform = (OneInputTransformation<Tuple3<String, String, Integer>, Integer>) window1.getTransformation(); final OneInputStreamOperator<Tuple3<String, String, Integer>, Integer> operator = transform.getOperator(); Assert.assertTrue(operator instanceof WindowOperator); WindowOperator<String, Tuple3<String, String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple3<String, String, Integer>, ?, ?, ?>) operator; Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger); Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingEventTimeWindows); Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor); processElementAndEnsureOutput( winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple3<>("hello", "hallo", 1)); }