org.apache.flink.api.common.state.ListStateDescriptor Java Examples

The following examples show how to use org.apache.flink.api.common.state.ListStateDescriptor. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: alibaba-flink-connectors   Author: alibaba   File: AbstractDynamicParallelSource.java    License: Apache License 2.0 6 votes vote down vote up
@Override
	public void initializeState(FunctionInitializationContext context) throws Exception {
		LOG.info("initializeState");
		ParameterizedType p = (ParameterizedType) this.getClass().getGenericSuperclass();
		TypeInformation type0 = TypeExtractor.createTypeInfo(InputSplit.class);
		TypeInformation type1 = TypeExtractor.createTypeInfo(p.getActualTypeArguments()[1]);
//		TypeInformation<Tuple2<InputSplit, CURSOR>> stateTypeInfo = new TupleTypeInfo<>(type0, type1);
		List<PojoField> pojoFields = new ArrayList<>();
		pojoFields.add(new PojoField(InnerProgress.class.getField("inputSplit"), type0));
		pojoFields.add(new PojoField(InnerProgress.class.getField("cursor"), type1));
		TypeInformation<InnerProgress> stateTypeInfo = new PojoTypeInfo<>(InnerProgress.class, pojoFields);

//		ListStateDescriptor<Tuple2<InputSplit, CURSOR>> descriptor = new ListStateDescriptor<>(SOURCE_STATE_NAME, stateTypeInfo);
		ListStateDescriptor<InnerProgress<CURSOR>> descriptor = new ListStateDescriptor(SOURCE_STATE_NAME, stateTypeInfo);
		unionInitialProgress = context.getOperatorStateStore().getUnionListState(descriptor);
		LOG.info("Restoring state: {}", unionInitialProgress);
		allSplitsInCP = new ArrayList<>();
		if (context.isRestored()) {
			recoryFromState = true;
			for (InnerProgress progress: unionInitialProgress.get()){
				allSplitsInCP.add(new InnerProgress(progress.inputSplit, progress.cursor));
			}
		}
	}
 
Example #2
Source Project: flink   Author: flink-tpc-ds   File: WindowOperatorTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testTumblingEventTimeWindowsApply() throws Exception {
	closeCalled.set(0);

	final int windowSize = 3;

	ListStateDescriptor<Tuple2<String, Integer>> stateDesc = new ListStateDescriptor<>("window-contents",
			STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
			TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
			new TimeWindow.Serializer(),
			new TupleKeySelector(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			stateDesc,
			new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>()),
			EventTimeTrigger.create(),
			0,
			null /* late data output tag */);

	testTumblingEventTimeWindows(operator);

	// we close once in the rest...
	Assert.assertEquals("Close was not called.", 2, closeCalled.get());
}
 
Example #3
Source Project: flink   Author: flink-tpc-ds   File: KeyedStateInputFormat.java    License: Apache License 2.0 6 votes vote down vote up
private Context(AbstractKeyedStateBackend<K> keyedStateBackend, InternalTimerService<VoidNamespace> timerService) throws Exception {
	eventTimers = keyedStateBackend.getPartitionedState(
		USER_TIMERS_NAME,
		StringSerializer.INSTANCE,
		new ListStateDescriptor<>(EVENT_TIMER_STATE, Types.LONG)
	);

	timerService.forEachEventTimeTimer((namespace, timer) -> {
		if (namespace.equals(VoidNamespace.INSTANCE)) {
			eventTimers.add(timer);
		}
	});

	procTimers = keyedStateBackend.getPartitionedState(
		USER_TIMERS_NAME,
		StringSerializer.INSTANCE,
		new ListStateDescriptor<>(PROC_TIMER_STATE, Types.LONG)
	);

	timerService.forEachProcessingTimeTimer((namespace, timer) -> {
		if (namespace.equals(VoidNamespace.INSTANCE)) {
			procTimers.add(timer);
		}
	});
}
 
Example #4
Source Project: flink   Author: flink-tpc-ds   File: StateBackendTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testApplyToAllKeysLambdaFunction() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {
		ListStateDescriptor<String> listStateDescriptor =
			new ListStateDescriptor<>("foo", StringSerializer.INSTANCE);

		ListState<String> listState =
			backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, listStateDescriptor);

		for (int i = 0; i < 100; ++i) {
			backend.setCurrentKey(i);
			listState.add("Hello" + i);
		}

		// valid state value via applyToAllKeys().
		backend.applyToAllKeys(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, listStateDescriptor,
			(Integer key, ListState<String> state) -> assertEquals("Hello" + key, state.get().iterator().next())
		);
	}
	finally {
		IOUtils.closeQuietly(backend);
		backend.dispose();
	}
}
 
Example #5
Source Project: flink   Author: apache   File: StateBackendMigrationTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testKeyedListStateRegistrationFailsIfNewStateSerializerIsIncompatible() {
	final String stateName = "test-name";

	try {
		testKeyedListStateUpgrade(
			new ListStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ListStateDescriptor<>(
				stateName,
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed");
	} catch (Exception expected) {
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}
 
Example #6
Source Project: flink   Author: flink-tpc-ds   File: AllWindowTranslationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
@SuppressWarnings("rawtypes")
public void testMergingWindowsWithEvictor() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DataStream<Tuple3<String, String, Integer>> window1 = source
			.windowAll(EventTimeSessionWindows.withGap(Time.seconds(5)))
			.evictor(CountEvictor.of(5))
			.process(new TestProcessAllWindowFunction());

	OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple3<String, String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof WindowOperator);
	WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof EventTimeSessionWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);

	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}
 
Example #7
Source Project: flink   Author: flink-tpc-ds   File: SavepointWriterITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	state = context.getOperatorStateStore().getUnionListState(
		new ListStateDescriptor<>("numbers", Types.INT)
	);

	if (context.isRestored()) {
		Set<Integer> expected = new HashSet<>();
		expected.add(1);
		expected.add(2);
		expected.add(3);

		for (Integer number : state.get()) {
			Assert.assertTrue("Duplicate state", expected.contains(number));
			expected.remove(number);
		}

		Assert.assertTrue("Failed to bootstrap all state elements: " + Arrays.toString(expected.toArray()), expected.isEmpty());
	}
}
 
Example #8
Source Project: flink   Author: apache   File: HeavyDeploymentStressTestProgram.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {

	readyToFail = false;

	if (context.isRestored()) {
		isRunning = false;
	} else {
		isRunning = true;

		OperatorStateStore operatorStateStore = context.getOperatorStateStore();
		for (int i = 0; i < numListStates; ++i) {

			ListStateDescriptor<String> listStateDescriptor =
				new ListStateDescriptor<>("test-list-state-" + i, String.class);

			ListState<String> unionListState =
				operatorStateStore.getUnionListState(listStateDescriptor);

			for (int j = 0; j < numPartitionsPerListState; ++j) {
				unionListState.add(String.valueOf(j));
			}
		}
	}
}
 
Example #9
Source Project: Flink-CEPplus   Author: ljygz   File: StateBackendTestBase.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This test verifies that all ListState implementations are consistent in not allowing
 * {@link ListState#addAll(List)} to be called with {@code null}.
 */
@Test
public void testListStateAddAllNull() throws Exception {
	AbstractKeyedStateBackend<String> keyedBackend = createKeyedBackend(StringSerializer.INSTANCE);

	final ListStateDescriptor<Long> stateDescr = new ListStateDescriptor<>("my-state", Long.class);

	try {
		ListState<Long> state =
			keyedBackend.getPartitionedState(
				VoidNamespace.INSTANCE,
				VoidNamespaceSerializer.INSTANCE,
				stateDescr);

		keyedBackend.setCurrentKey("abc");
		assertNull(state.get());

		expectedException.expect(NullPointerException.class);
		state.addAll(null);
	} finally {
		keyedBackend.close();
		keyedBackend.dispose();
	}
}
 
Example #10
Source Project: Flink-CEPplus   Author: ljygz   File: StateBackendTestBase.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Verify that an empty {@code ListState} yields {@code null}.
 */
@Test
public void testListStateDefaultValue() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	ListStateDescriptor<String> kvId = new ListStateDescriptor<>("id", String.class);

	ListState<String> state = backend.getPartitionedState(
			VoidNamespace.INSTANCE,
			VoidNamespaceSerializer.INSTANCE, kvId);

	backend.setCurrentKey(1);
	assertNull(state.get());

	state.update(Arrays.asList("Ciao", "Bello"));
	assertThat(state.get(), containsInAnyOrder("Ciao", "Bello"));

	state.clear();
	assertNull(state.get());

	backend.dispose();
}
 
Example #11
Source Project: flink   Author: apache   File: StateBackendTestBase.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This test verifies that all ListState implementations are consistent in not allowing
 * adding {@code null}.
 */
@Test
public void testListStateAddNull() throws Exception {
	AbstractKeyedStateBackend<String> keyedBackend = createKeyedBackend(StringSerializer.INSTANCE);

	final ListStateDescriptor<Long> stateDescr = new ListStateDescriptor<>("my-state", Long.class);

	try {
		ListState<Long> state =
			keyedBackend.getPartitionedState(
				VoidNamespace.INSTANCE,
				VoidNamespaceSerializer.INSTANCE,
				stateDescr);

		keyedBackend.setCurrentKey("abc");
		assertNull(state.get());

		expectedException.expect(NullPointerException.class);
		state.add(null);
	} finally {
		keyedBackend.close();
		keyedBackend.dispose();
	}
}
 
Example #12
Source Project: Flink-CEPplus   Author: ljygz   File: StateBackendTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testApplyToAllKeysLambdaFunction() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {
		ListStateDescriptor<String> listStateDescriptor =
			new ListStateDescriptor<>("foo", StringSerializer.INSTANCE);

		ListState<String> listState =
			backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, listStateDescriptor);

		for (int i = 0; i < 100; ++i) {
			backend.setCurrentKey(i);
			listState.add("Hello" + i);
		}

		// valid state value via applyToAllKeys().
		backend.applyToAllKeys(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, listStateDescriptor,
			(Integer key, ListState<String> state) -> assertEquals("Hello" + key, state.get().iterator().next())
		);
	}
	finally {
		IOUtils.closeQuietly(backend);
		backend.dispose();
	}
}
 
Example #13
Source Project: Flink-CEPplus   Author: ljygz   File: StateBackendMigrationTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testKeyedListStateRegistrationFailsIfNewStateSerializerIsIncompatible() throws Exception {
	final String stateName = "test-name";

	try {
		testKeyedListStateUpgrade(
			new ListStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ListStateDescriptor<>(
				stateName,
				new TestType.IncompatibleTestTypeSerializer()));

		Assert.fail("should have failed");
	} catch (Exception expected) {
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}
 
Example #14
Source Project: aliyun-log-flink-connector   Author: aliyun   File: FlinkLogConsumer.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
    LOG.debug("Initializing state from Flink state");

    TypeInformation<Tuple2<LogstoreShardMeta, String>> shardsStateTypeInfo = new TupleTypeInfo<Tuple2<LogstoreShardMeta, String>>(
            TypeInformation.of(LogstoreShardMeta.class),
            TypeInformation.of(String.class));
    cursorStateForCheckpoint = context.getOperatorStateStore().getUnionListState(
            new ListStateDescriptor<>(CURSOR_STATE_STORE_NAME, shardsStateTypeInfo));
    if (!context.isRestored()) {
        LOG.info("No state restored for FlinkLogConsumer.");
        return;
    }
    if (cursorsToRestore != null) {
        LOG.info("Flink state has been restored already.");
        return;
    }
    createClientIfNeeded();
    cursorsToRestore = new HashMap<>();
    for (Tuple2<LogstoreShardMeta, String> cursor : cursorStateForCheckpoint.get()) {
        final LogstoreShardMeta shardMeta = cursor.f0;
        final String checkpoint = cursor.f1;
        cursorsToRestore.put(shardMeta, checkpoint);
    }
    LOG.info("The following offsets restored from Flink state: {}", cursorsToRestore);
}
 
Example #15
Source Project: Flink-CEPplus   Author: ljygz   File: DataStreamAllroundTestJobFactory.java    License: Apache License 2.0 6 votes vote down vote up
private static <IN, STATE> ArtificialStateBuilder<IN> createListStateBuilder(
	JoinFunction<IN, STATE, STATE> inputAndOldStateToNewState,
	ListStateDescriptor<STATE> listStateDescriptor) {

	JoinFunction<IN, Iterable<STATE>, List<STATE>> listStateGenerator = (first, second) -> {
		List<STATE> newState = new ArrayList<>();
		for (STATE s : second) {
			newState.add(inputAndOldStateToNewState.join(first, s));
		}
		return newState;
	};

	return new ArtificialListStateBuilder<>(
		listStateDescriptor.getName(),
		listStateGenerator,
		listStateGenerator,
		listStateDescriptor);
}
 
Example #16
Source Project: flink   Author: apache   File: SavepointWriterITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	state = context.getOperatorStateStore().getUnionListState(
		new ListStateDescriptor<>("numbers", Types.INT)
	);

	if (context.isRestored()) {
		Set<Integer> expected = new HashSet<>();
		expected.add(1);
		expected.add(2);
		expected.add(3);

		for (Integer number : state.get()) {
			Assert.assertTrue("Duplicate state", expected.contains(number));
			expected.remove(number);
		}

		Assert.assertTrue("Failed to bootstrap all state elements: " + Arrays.toString(expected.toArray()), expected.isEmpty());
	}
}
 
Example #17
Source Project: flink   Author: apache   File: WindowTranslationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("rawtypes")
public void testProcessWithCustomTrigger() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DataStream<Tuple2<String, Integer>> window1 = source
			.keyBy(new TupleKeySelector())
			.window(TumblingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS)))
			.trigger(CountTrigger.of(1))
			.process(new ProcessWindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, String, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void process(String key,
						Context ctx,
						Iterable<Tuple2<String, Integer>> values,
						Collector<Tuple2<String, Integer>> out) throws Exception {
					for (Tuple2<String, Integer> in : values) {
						out.collect(in);
					}
				}
			});

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof WindowOperator);
	WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof CountTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingEventTimeWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);

	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}
 
Example #18
Source Project: flink   Author: apache   File: TtlStateFactory.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
private Map<Class<? extends StateDescriptor>, SupplierWithException<IS, Exception>> createStateFactories() {
	return Stream.of(
		Tuple2.of(ValueStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createValueState),
		Tuple2.of(ListStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createListState),
		Tuple2.of(MapStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createMapState),
		Tuple2.of(ReducingStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createReducingState),
		Tuple2.of(AggregatingStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createAggregatingState),
		Tuple2.of(FoldingStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createFoldingState)
	).collect(Collectors.toMap(t -> t.f0, t -> t.f1));
}
 
Example #19
Source Project: Flink-CEPplus   Author: ljygz   File: WindowTranslationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("rawtypes")
public void testProcessEventTime() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DataStream<Tuple2<String, Integer>> window1 = source
			.keyBy(new TupleKeySelector())
			.window(TumblingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS)))
			.process(new ProcessWindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, String, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void process(String key,
						Context ctx,
						Iterable<Tuple2<String, Integer>> values,
						Collector<Tuple2<String, Integer>> out) throws Exception {
					for (Tuple2<String, Integer> in : values) {
						out.collect(in);
					}
				}
			});

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof WindowOperator);
	WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingEventTimeWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);

	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}
 
Example #20
Source Project: Flink-CEPplus   Author: ljygz   File: AllWindowTranslationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("rawtypes")
public void testProcessEventTime() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DataStream<Tuple2<String, Integer>> window1 = source
			.windowAll(TumblingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS)))
			.process(new ProcessAllWindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void process(
						Context ctx,
						Iterable<Tuple2<String, Integer>> values,
						Collector<Tuple2<String, Integer>> out) throws Exception {
					for (Tuple2<String, Integer> in : values) {
						out.collect(in);
					}
				}
			});

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof WindowOperator);
	WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingEventTimeWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);

	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}
 
Example #21
Source Project: flink   Author: apache   File: WindowTranslationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings({"rawtypes", "unchecked"})
public void testFoldWithEvictor() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DataStream<Tuple3<String, String, Integer>> window1 = source
			.keyBy(0)
			.window(SlidingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS)))
			.evictor(CountEvictor.of(100))
			.fold(new Tuple3<>("", "", 1), new DummyFolder());

	OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>> transform =
			(OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple3<String, String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof EvictingWindowOperator);
	EvictingWindowOperator<String, Tuple2<String, Integer>, ?, ?> winOperator = (EvictingWindowOperator<String, Tuple2<String, Integer>, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator.getEvictor() instanceof CountEvictor);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingEventTimeWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);

	winOperator.setOutputType((TypeInformation) window1.getType(), new ExecutionConfig());
	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}
 
Example #22
Source Project: Flink-CEPplus   Author: ljygz   File: RocksDBSnapshotTransformFactoryAdaptor.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <SV, SEV> StateSnapshotTransformFactory<SV> wrapStateSnapshotTransformFactory(
	StateDescriptor<?, SV> stateDesc,
	StateSnapshotTransformFactory<SEV> snapshotTransformFactory,
	TypeSerializer<SV> stateSerializer) {
	if (stateDesc instanceof ListStateDescriptor) {
		TypeSerializer<SEV> elementSerializer = ((ListSerializer<SEV>) stateSerializer).getElementSerializer();
		return new RocksDBListStateSnapshotTransformFactory<>(snapshotTransformFactory, elementSerializer);
	} else if (stateDesc instanceof MapStateDescriptor) {
		return new RocksDBMapStateSnapshotTransformFactory<>(snapshotTransformFactory);
	} else {
		return new RocksDBValueStateSnapshotTransformFactory<>(snapshotTransformFactory);
	}
}
 
Example #23
Source Project: Flink-CEPplus   Author: ljygz   File: RocksDbTtlCompactFiltersManager.java    License: Apache License 2.0 5 votes vote down vote up
public void configCompactFilter(
		@Nonnull StateDescriptor<?, ?> stateDesc,
		TypeSerializer<?> stateSerializer) {
	StateTtlConfig ttlConfig = stateDesc.getTtlConfig();
	if (ttlConfig.isEnabled() && ttlConfig.getCleanupStrategies().inRocksdbCompactFilter()) {
		if (!enableTtlCompactionFilter) {
			LOG.warn("Cannot configure RocksDB TTL compaction filter for state <{}>: " +
				"feature is disabled for the state backend.", stateDesc.getName());
			return;
		}
		FlinkCompactionFilterFactory compactionFilterFactory = compactionFilterFactories.get(stateDesc.getName());
		Preconditions.checkNotNull(compactionFilterFactory);
		long ttl = ttlConfig.getTtl().toMilliseconds();

		StateTtlConfig.RocksdbCompactFilterCleanupStrategy rocksdbCompactFilterCleanupStrategy =
			ttlConfig.getCleanupStrategies().getRocksdbCompactFilterCleanupStrategy();
		Preconditions.checkNotNull(rocksdbCompactFilterCleanupStrategy);
		long queryTimeAfterNumEntries =
			rocksdbCompactFilterCleanupStrategy.getQueryTimeAfterNumEntries();

		FlinkCompactionFilter.Config config;
		if (stateDesc instanceof ListStateDescriptor) {
			TypeSerializer<?> elemSerializer = ((ListSerializer<?>) stateSerializer).getElementSerializer();
			int len = elemSerializer.getLength();
			if (len > 0) {
				config = FlinkCompactionFilter.Config.createForFixedElementList(
					ttl, queryTimeAfterNumEntries, len + 1); // plus one byte for list element delimiter
			} else {
				config = FlinkCompactionFilter.Config.createForList(
					ttl, queryTimeAfterNumEntries,
					new ListElementFilterFactory<>(elemSerializer.duplicate()));
			}
		} else if (stateDesc instanceof MapStateDescriptor) {
			config = FlinkCompactionFilter.Config.createForMap(ttl, queryTimeAfterNumEntries);
		} else {
			config = FlinkCompactionFilter.Config.createForValue(ttl, queryTimeAfterNumEntries);
		}
		compactionFilterFactory.configure(config);
	}
}
 
Example #24
Source Project: Flink-CEPplus   Author: ljygz   File: HeapKeyedStateBackend.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private <SV, SEV> StateSnapshotTransformFactory<SV> getStateSnapshotTransformFactory(
	StateDescriptor<?, SV> stateDesc,
	StateSnapshotTransformFactory<SEV> snapshotTransformFactory) {
	if (stateDesc instanceof ListStateDescriptor) {
		return (StateSnapshotTransformFactory<SV>) new StateSnapshotTransformers.ListStateSnapshotTransformFactory<>(snapshotTransformFactory);
	} else if (stateDesc instanceof MapStateDescriptor) {
		return (StateSnapshotTransformFactory<SV>) new StateSnapshotTransformers.MapStateSnapshotTransformFactory<>(snapshotTransformFactory);
	} else {
		return (StateSnapshotTransformFactory<SV>) snapshotTransformFactory;
	}
}
 
Example #25
Source Project: flink   Author: apache   File: ReinterpretDataStreamAsKeyedStreamITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	canFail = false;
	position = 0L;
	isRestored = context.isRestored();
	positionState = context.getOperatorStateStore().getListState(
		new ListStateDescriptor<>("posState", Long.class));

	if (isRestored) {
		for (long value : positionState.get()) {
			position += value;
		}
	}
}
 
Example #26
Source Project: flink   Author: flink-tpc-ds   File: WindowTranslationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("rawtypes")
public void testProcessWithCustomTrigger() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DataStream<Tuple2<String, Integer>> window1 = source
			.keyBy(new TupleKeySelector())
			.window(TumblingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS)))
			.trigger(CountTrigger.of(1))
			.process(new ProcessWindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, String, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void process(String key,
						Context ctx,
						Iterable<Tuple2<String, Integer>> values,
						Collector<Tuple2<String, Integer>> out) throws Exception {
					for (Tuple2<String, Integer> in : values) {
						out.collect(in);
					}
				}
			});

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof WindowOperator);
	WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof CountTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingEventTimeWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);

	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}
 
Example #27
Source Project: flink   Author: flink-tpc-ds   File: WindowTranslationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("rawtypes")
public void testProcessProcessingTime() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DataStream<Tuple2<String, Integer>> window1 = source
			.keyBy(new TupleKeySelector())
			.window(TumblingProcessingTimeWindows.of(Time.of(1, TimeUnit.SECONDS)))
			.process(new ProcessWindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, String, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void process(String key,
						Context ctx,
						Iterable<Tuple2<String, Integer>> values,
						Collector<Tuple2<String, Integer>> out) throws Exception {
					for (Tuple2<String, Integer> in : values) {
						out.collect(in);
					}
				}
			});

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof WindowOperator);
	WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof ProcessingTimeTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingProcessingTimeWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);

	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));

}
 
Example #28
Source Project: Flink-CEPplus   Author: ljygz   File: TtlStateFactory.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
private Map<Class<? extends StateDescriptor>, SupplierWithException<IS, Exception>> createStateFactories() {
	return Stream.of(
		Tuple2.of(ValueStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createValueState),
		Tuple2.of(ListStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createListState),
		Tuple2.of(MapStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createMapState),
		Tuple2.of(ReducingStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createReducingState),
		Tuple2.of(AggregatingStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createAggregatingState),
		Tuple2.of(FoldingStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createFoldingState)
	).collect(Collectors.toMap(t -> t.f0, t -> t.f1));
}
 
Example #29
Source Project: Flink-CEPplus   Author: ljygz   File: TtlStateFactory.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private <T> IS createListState() throws Exception {
	ListStateDescriptor<T> listStateDesc = (ListStateDescriptor<T>) stateDesc;
	ListStateDescriptor<TtlValue<T>> ttlDescriptor = new ListStateDescriptor<>(
		stateDesc.getName(), new TtlSerializer<>(LongSerializer.INSTANCE, listStateDesc.getElementSerializer()));
	return (IS) new TtlListState<>(createTtlStateContext(ttlDescriptor));
}
 
Example #30
Source Project: flink   Author: apache   File: WindowTranslationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("rawtypes")
public void testReduceWithEvictorAndProcessFunction() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DummyReducer reducer = new DummyReducer();

	DataStream<Tuple2<String, Integer>> window1 = source
			.keyBy(0)
			.window(SlidingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS)))
			.evictor(CountEvictor.of(100))
			.reduce(
					reducer,
					new ProcessWindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple, TimeWindow>() {
						@Override
						public void process(
								Tuple tuple,
								Context context,
								Iterable<Tuple2<String, Integer>> elements,
								Collector<Tuple2<String, Integer>> out) throws Exception {
							for (Tuple2<String, Integer> in : elements) {
								out.collect(in);
							}
						}
					});

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof EvictingWindowOperator);
	EvictingWindowOperator<String, Tuple2<String, Integer>, ?, ?> winOperator = (EvictingWindowOperator<String, Tuple2<String, Integer>, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator.getEvictor() instanceof CountEvictor);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingEventTimeWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);

	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}