org.apache.flink.api.common.state.ListStateDescriptor Java Examples

@Override
	public void initializeState(FunctionInitializationContext context) throws Exception {
		LOG.info("initializeState");
		ParameterizedType p = (ParameterizedType) this.getClass().getGenericSuperclass();
		TypeInformation type0 = TypeExtractor.createTypeInfo(InputSplit.class);
		TypeInformation type1 = TypeExtractor.createTypeInfo(p.getActualTypeArguments()[1]);
//		TypeInformation<Tuple2<InputSplit, CURSOR>> stateTypeInfo = new TupleTypeInfo<>(type0, type1);
		List<PojoField> pojoFields = new ArrayList<>();
		pojoFields.add(new PojoField(InnerProgress.class.getField("inputSplit"), type0));
		pojoFields.add(new PojoField(InnerProgress.class.getField("cursor"), type1));
		TypeInformation<InnerProgress> stateTypeInfo = new PojoTypeInfo<>(InnerProgress.class, pojoFields);

//		ListStateDescriptor<Tuple2<InputSplit, CURSOR>> descriptor = new ListStateDescriptor<>(SOURCE_STATE_NAME, stateTypeInfo);
		ListStateDescriptor<InnerProgress<CURSOR>> descriptor = new ListStateDescriptor(SOURCE_STATE_NAME, stateTypeInfo);
		unionInitialProgress = context.getOperatorStateStore().getUnionListState(descriptor);
		LOG.info("Restoring state: {}", unionInitialProgress);
		allSplitsInCP = new ArrayList<>();
		if (context.isRestored()) {
			recoryFromState = true;
			for (InnerProgress progress: unionInitialProgress.get()){
				allSplitsInCP.add(new InnerProgress(progress.inputSplit, progress.cursor));
			}
		}
	}
 

@Test
@SuppressWarnings("unchecked")
public void testTumblingEventTimeWindowsApply() throws Exception {
	closeCalled.set(0);

	final int windowSize = 3;

	ListStateDescriptor<Tuple2<String, Integer>> stateDesc = new ListStateDescriptor<>("window-contents",
			STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
			TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
			new TimeWindow.Serializer(),
			new TupleKeySelector(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			stateDesc,
			new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>()),
			EventTimeTrigger.create(),
			0,
			null /* late data output tag */);

	testTumblingEventTimeWindows(operator);

	// we close once in the rest...
	Assert.assertEquals("Close was not called.", 2, closeCalled.get());
}
 

private Context(AbstractKeyedStateBackend<K> keyedStateBackend, InternalTimerService<VoidNamespace> timerService) throws Exception {
	eventTimers = keyedStateBackend.getPartitionedState(
		USER_TIMERS_NAME,
		StringSerializer.INSTANCE,
		new ListStateDescriptor<>(EVENT_TIMER_STATE, Types.LONG)
	);

	timerService.forEachEventTimeTimer((namespace, timer) -> {
		if (namespace.equals(VoidNamespace.INSTANCE)) {
			eventTimers.add(timer);
		}
	});

	procTimers = keyedStateBackend.getPartitionedState(
		USER_TIMERS_NAME,
		StringSerializer.INSTANCE,
		new ListStateDescriptor<>(PROC_TIMER_STATE, Types.LONG)
	);

	timerService.forEachProcessingTimeTimer((namespace, timer) -> {
		if (namespace.equals(VoidNamespace.INSTANCE)) {
			procTimers.add(timer);
		}
	});
}
 

@Test
public void testApplyToAllKeysLambdaFunction() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {
		ListStateDescriptor<String> listStateDescriptor =
			new ListStateDescriptor<>("foo", StringSerializer.INSTANCE);

		ListState<String> listState =
			backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, listStateDescriptor);

		for (int i = 0; i < 100; ++i) {
			backend.setCurrentKey(i);
			listState.add("Hello" + i);
		}

		// valid state value via applyToAllKeys().
		backend.applyToAllKeys(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, listStateDescriptor,
			(Integer key, ListState<String> state) -> assertEquals("Hello" + key, state.get().iterator().next())
		);
	}
	finally {
		IOUtils.closeQuietly(backend);
		backend.dispose();
	}
}
 

@Test
public void testKeyedListStateRegistrationFailsIfNewStateSerializerIsIncompatible() {
	final String stateName = "test-name";

	try {
		testKeyedListStateUpgrade(
			new ListStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ListStateDescriptor<>(
				stateName,
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed");
	} catch (Exception expected) {
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}
 

@Test
@SuppressWarnings("rawtypes")
public void testMergingWindowsWithEvictor() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DataStream<Tuple3<String, String, Integer>> window1 = source
			.windowAll(EventTimeSessionWindows.withGap(Time.seconds(5)))
			.evictor(CountEvictor.of(5))
			.process(new TestProcessAllWindowFunction());

	OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple3<String, String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof WindowOperator);
	WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof EventTimeSessionWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);

	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}
 

@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	state = context.getOperatorStateStore().getUnionListState(
		new ListStateDescriptor<>("numbers", Types.INT)
	);

	if (context.isRestored()) {
		Set<Integer> expected = new HashSet<>();
		expected.add(1);
		expected.add(2);
		expected.add(3);

		for (Integer number : state.get()) {
			Assert.assertTrue("Duplicate state", expected.contains(number));
			expected.remove(number);
		}

		Assert.assertTrue("Failed to bootstrap all state elements: " + Arrays.toString(expected.toArray()), expected.isEmpty());
	}
}
 

@Override
public void initializeState(FunctionInitializationContext context) throws Exception {

	readyToFail = false;

	if (context.isRestored()) {
		isRunning = false;
	} else {
		isRunning = true;

		OperatorStateStore operatorStateStore = context.getOperatorStateStore();
		for (int i = 0; i < numListStates; ++i) {

			ListStateDescriptor<String> listStateDescriptor =
				new ListStateDescriptor<>("test-list-state-" + i, String.class);

			ListState<String> unionListState =
				operatorStateStore.getUnionListState(listStateDescriptor);

			for (int j = 0; j < numPartitionsPerListState; ++j) {
				unionListState.add(String.valueOf(j));
			}
		}
	}
}
 

/**
 * This test verifies that all ListState implementations are consistent in not allowing
 * {@link ListState#addAll(List)} to be called with {@code null}.
 */
@Test
public void testListStateAddAllNull() throws Exception {
	AbstractKeyedStateBackend<String> keyedBackend = createKeyedBackend(StringSerializer.INSTANCE);

	final ListStateDescriptor<Long> stateDescr = new ListStateDescriptor<>("my-state", Long.class);

	try {
		ListState<Long> state =
			keyedBackend.getPartitionedState(
				VoidNamespace.INSTANCE,
				VoidNamespaceSerializer.INSTANCE,
				stateDescr);

		keyedBackend.setCurrentKey("abc");
		assertNull(state.get());

		expectedException.expect(NullPointerException.class);
		state.addAll(null);
	} finally {
		keyedBackend.close();
		keyedBackend.dispose();
	}
}
 

/**
 * Verify that an empty {@code ListState} yields {@code null}.
 */
@Test
public void testListStateDefaultValue() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	ListStateDescriptor<String> kvId = new ListStateDescriptor<>("id", String.class);

	ListState<String> state = backend.getPartitionedState(
			VoidNamespace.INSTANCE,
			VoidNamespaceSerializer.INSTANCE, kvId);

	backend.setCurrentKey(1);
	assertNull(state.get());

	state.update(Arrays.asList("Ciao", "Bello"));
	assertThat(state.get(), containsInAnyOrder("Ciao", "Bello"));

	state.clear();
	assertNull(state.get());

	backend.dispose();
}
 

/**
 * This test verifies that all ListState implementations are consistent in not allowing
 * adding {@code null}.
 */
@Test
public void testListStateAddNull() throws Exception {
	AbstractKeyedStateBackend<String> keyedBackend = createKeyedBackend(StringSerializer.INSTANCE);

	final ListStateDescriptor<Long> stateDescr = new ListStateDescriptor<>("my-state", Long.class);

	try {
		ListState<Long> state =
			keyedBackend.getPartitionedState(
				VoidNamespace.INSTANCE,
				VoidNamespaceSerializer.INSTANCE,
				stateDescr);

		keyedBackend.setCurrentKey("abc");
		assertNull(state.get());

		expectedException.expect(NullPointerException.class);
		state.add(null);
	} finally {
		keyedBackend.close();
		keyedBackend.dispose();
	}
}
 

@Test
public void testApplyToAllKeysLambdaFunction() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {
		ListStateDescriptor<String> listStateDescriptor =
			new ListStateDescriptor<>("foo", StringSerializer.INSTANCE);

		ListState<String> listState =
			backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, listStateDescriptor);

		for (int i = 0; i < 100; ++i) {
			backend.setCurrentKey(i);
			listState.add("Hello" + i);
		}

		// valid state value via applyToAllKeys().
		backend.applyToAllKeys(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, listStateDescriptor,
			(Integer key, ListState<String> state) -> assertEquals("Hello" + key, state.get().iterator().next())
		);
	}
	finally {
		IOUtils.closeQuietly(backend);
		backend.dispose();
	}
}
 

@Test
public void testKeyedListStateRegistrationFailsIfNewStateSerializerIsIncompatible() throws Exception {
	final String stateName = "test-name";

	try {
		testKeyedListStateUpgrade(
			new ListStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ListStateDescriptor<>(
				stateName,
				new TestType.IncompatibleTestTypeSerializer()));

		Assert.fail("should have failed");
	} catch (Exception expected) {
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}
 

@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
    LOG.debug("Initializing state from Flink state");

    TypeInformation<Tuple2<LogstoreShardMeta, String>> shardsStateTypeInfo = new TupleTypeInfo<Tuple2<LogstoreShardMeta, String>>(
            TypeInformation.of(LogstoreShardMeta.class),
            TypeInformation.of(String.class));
    cursorStateForCheckpoint = context.getOperatorStateStore().getUnionListState(
            new ListStateDescriptor<>(CURSOR_STATE_STORE_NAME, shardsStateTypeInfo));
    if (!context.isRestored()) {
        LOG.info("No state restored for FlinkLogConsumer.");
        return;
    }
    if (cursorsToRestore != null) {
        LOG.info("Flink state has been restored already.");
        return;
    }
    createClientIfNeeded();
    cursorsToRestore = new HashMap<>();
    for (Tuple2<LogstoreShardMeta, String> cursor : cursorStateForCheckpoint.get()) {
        final LogstoreShardMeta shardMeta = cursor.f0;
        final String checkpoint = cursor.f1;
        cursorsToRestore.put(shardMeta, checkpoint);
    }
    LOG.info("The following offsets restored from Flink state: {}", cursorsToRestore);
}
 

private static <IN, STATE> ArtificialStateBuilder<IN> createListStateBuilder(
	JoinFunction<IN, STATE, STATE> inputAndOldStateToNewState,
	ListStateDescriptor<STATE> listStateDescriptor) {

	JoinFunction<IN, Iterable<STATE>, List<STATE>> listStateGenerator = (first, second) -> {
		List<STATE> newState = new ArrayList<>();
		for (STATE s : second) {
			newState.add(inputAndOldStateToNewState.join(first, s));
		}
		return newState;
	};

	return new ArtificialListStateBuilder<>(
		listStateDescriptor.getName(),
		listStateGenerator,
		listStateGenerator,
		listStateDescriptor);
}
 

@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	state = context.getOperatorStateStore().getUnionListState(
		new ListStateDescriptor<>("numbers", Types.INT)
	);

	if (context.isRestored()) {
		Set<Integer> expected = new HashSet<>();
		expected.add(1);
		expected.add(2);
		expected.add(3);

		for (Integer number : state.get()) {
			Assert.assertTrue("Duplicate state", expected.contains(number));
			expected.remove(number);
		}

		Assert.assertTrue("Failed to bootstrap all state elements: " + Arrays.toString(expected.toArray()), expected.isEmpty());
	}
}
 

@Test
@SuppressWarnings("rawtypes")
public void testProcessWithCustomTrigger() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DataStream<Tuple2<String, Integer>> window1 = source
			.keyBy(new TupleKeySelector())
			.window(TumblingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS)))
			.trigger(CountTrigger.of(1))
			.process(new ProcessWindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, String, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void process(String key,
						Context ctx,
						Iterable<Tuple2<String, Integer>> values,
						Collector<Tuple2<String, Integer>> out) throws Exception {
					for (Tuple2<String, Integer> in : values) {
						out.collect(in);
					}
				}
			});

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof WindowOperator);
	WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof CountTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingEventTimeWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);

	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}
 

@SuppressWarnings("deprecation")
private Map<Class<? extends StateDescriptor>, SupplierWithException<IS, Exception>> createStateFactories() {
	return Stream.of(
		Tuple2.of(ValueStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createValueState),
		Tuple2.of(ListStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createListState),
		Tuple2.of(MapStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createMapState),
		Tuple2.of(ReducingStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createReducingState),
		Tuple2.of(AggregatingStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createAggregatingState),
		Tuple2.of(FoldingStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createFoldingState)
	).collect(Collectors.toMap(t -> t.f0, t -> t.f1));
}
 

@Test
@SuppressWarnings("rawtypes")
public void testProcessEventTime() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DataStream<Tuple2<String, Integer>> window1 = source
			.keyBy(new TupleKeySelector())
			.window(TumblingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS)))
			.process(new ProcessWindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, String, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void process(String key,
						Context ctx,
						Iterable<Tuple2<String, Integer>> values,
						Collector<Tuple2<String, Integer>> out) throws Exception {
					for (Tuple2<String, Integer> in : values) {
						out.collect(in);
					}
				}
			});

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof WindowOperator);
	WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingEventTimeWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);

	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}
 

@Test
@SuppressWarnings("rawtypes")
public void testProcessEventTime() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DataStream<Tuple2<String, Integer>> window1 = source
			.windowAll(TumblingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS)))
			.process(new ProcessAllWindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void process(
						Context ctx,
						Iterable<Tuple2<String, Integer>> values,
						Collector<Tuple2<String, Integer>> out) throws Exception {
					for (Tuple2<String, Integer> in : values) {
						out.collect(in);
					}
				}
			});

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof WindowOperator);
	WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingEventTimeWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);

	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}
 

@Test
@SuppressWarnings({"rawtypes", "unchecked"})
public void testFoldWithEvictor() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DataStream<Tuple3<String, String, Integer>> window1 = source
			.keyBy(0)
			.window(SlidingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS)))
			.evictor(CountEvictor.of(100))
			.fold(new Tuple3<>("", "", 1), new DummyFolder());

	OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>> transform =
			(OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple3<String, String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof EvictingWindowOperator);
	EvictingWindowOperator<String, Tuple2<String, Integer>, ?, ?> winOperator = (EvictingWindowOperator<String, Tuple2<String, Integer>, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator.getEvictor() instanceof CountEvictor);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingEventTimeWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);

	winOperator.setOutputType((TypeInformation) window1.getType(), new ExecutionConfig());
	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}
 

@SuppressWarnings("unchecked")
static <SV, SEV> StateSnapshotTransformFactory<SV> wrapStateSnapshotTransformFactory(
	StateDescriptor<?, SV> stateDesc,
	StateSnapshotTransformFactory<SEV> snapshotTransformFactory,
	TypeSerializer<SV> stateSerializer) {
	if (stateDesc instanceof ListStateDescriptor) {
		TypeSerializer<SEV> elementSerializer = ((ListSerializer<SEV>) stateSerializer).getElementSerializer();
		return new RocksDBListStateSnapshotTransformFactory<>(snapshotTransformFactory, elementSerializer);
	} else if (stateDesc instanceof MapStateDescriptor) {
		return new RocksDBMapStateSnapshotTransformFactory<>(snapshotTransformFactory);
	} else {
		return new RocksDBValueStateSnapshotTransformFactory<>(snapshotTransformFactory);
	}
}
 

public void configCompactFilter(
		@Nonnull StateDescriptor<?, ?> stateDesc,
		TypeSerializer<?> stateSerializer) {
	StateTtlConfig ttlConfig = stateDesc.getTtlConfig();
	if (ttlConfig.isEnabled() && ttlConfig.getCleanupStrategies().inRocksdbCompactFilter()) {
		if (!enableTtlCompactionFilter) {
			LOG.warn("Cannot configure RocksDB TTL compaction filter for state <{}>: " +
				"feature is disabled for the state backend.", stateDesc.getName());
			return;
		}
		FlinkCompactionFilterFactory compactionFilterFactory = compactionFilterFactories.get(stateDesc.getName());
		Preconditions.checkNotNull(compactionFilterFactory);
		long ttl = ttlConfig.getTtl().toMilliseconds();

		StateTtlConfig.RocksdbCompactFilterCleanupStrategy rocksdbCompactFilterCleanupStrategy =
			ttlConfig.getCleanupStrategies().getRocksdbCompactFilterCleanupStrategy();
		Preconditions.checkNotNull(rocksdbCompactFilterCleanupStrategy);
		long queryTimeAfterNumEntries =
			rocksdbCompactFilterCleanupStrategy.getQueryTimeAfterNumEntries();

		FlinkCompactionFilter.Config config;
		if (stateDesc instanceof ListStateDescriptor) {
			TypeSerializer<?> elemSerializer = ((ListSerializer<?>) stateSerializer).getElementSerializer();
			int len = elemSerializer.getLength();
			if (len > 0) {
				config = FlinkCompactionFilter.Config.createForFixedElementList(
					ttl, queryTimeAfterNumEntries, len + 1); // plus one byte for list element delimiter
			} else {
				config = FlinkCompactionFilter.Config.createForList(
					ttl, queryTimeAfterNumEntries,
					new ListElementFilterFactory<>(elemSerializer.duplicate()));
			}
		} else if (stateDesc instanceof MapStateDescriptor) {
			config = FlinkCompactionFilter.Config.createForMap(ttl, queryTimeAfterNumEntries);
		} else {
			config = FlinkCompactionFilter.Config.createForValue(ttl, queryTimeAfterNumEntries);
		}
		compactionFilterFactory.configure(config);
	}
}
 

@SuppressWarnings("unchecked")
private <SV, SEV> StateSnapshotTransformFactory<SV> getStateSnapshotTransformFactory(
	StateDescriptor<?, SV> stateDesc,
	StateSnapshotTransformFactory<SEV> snapshotTransformFactory) {
	if (stateDesc instanceof ListStateDescriptor) {
		return (StateSnapshotTransformFactory<SV>) new StateSnapshotTransformers.ListStateSnapshotTransformFactory<>(snapshotTransformFactory);
	} else if (stateDesc instanceof MapStateDescriptor) {
		return (StateSnapshotTransformFactory<SV>) new StateSnapshotTransformers.MapStateSnapshotTransformFactory<>(snapshotTransformFactory);
	} else {
		return (StateSnapshotTransformFactory<SV>) snapshotTransformFactory;
	}
}
 

@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	canFail = false;
	position = 0L;
	isRestored = context.isRestored();
	positionState = context.getOperatorStateStore().getListState(
		new ListStateDescriptor<>("posState", Long.class));

	if (isRestored) {
		for (long value : positionState.get()) {
			position += value;
		}
	}
}
 

@Test
@SuppressWarnings("rawtypes")
public void testProcessWithCustomTrigger() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DataStream<Tuple2<String, Integer>> window1 = source
			.keyBy(new TupleKeySelector())
			.window(TumblingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS)))
			.trigger(CountTrigger.of(1))
			.process(new ProcessWindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, String, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void process(String key,
						Context ctx,
						Iterable<Tuple2<String, Integer>> values,
						Collector<Tuple2<String, Integer>> out) throws Exception {
					for (Tuple2<String, Integer> in : values) {
						out.collect(in);
					}
				}
			});

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof WindowOperator);
	WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof CountTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingEventTimeWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);

	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}
 

@Test
@SuppressWarnings("rawtypes")
public void testProcessProcessingTime() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DataStream<Tuple2<String, Integer>> window1 = source
			.keyBy(new TupleKeySelector())
			.window(TumblingProcessingTimeWindows.of(Time.of(1, TimeUnit.SECONDS)))
			.process(new ProcessWindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, String, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void process(String key,
						Context ctx,
						Iterable<Tuple2<String, Integer>> values,
						Collector<Tuple2<String, Integer>> out) throws Exception {
					for (Tuple2<String, Integer> in : values) {
						out.collect(in);
					}
				}
			});

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof WindowOperator);
	WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof ProcessingTimeTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingProcessingTimeWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);

	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));

}
 

@SuppressWarnings("deprecation")
private Map<Class<? extends StateDescriptor>, SupplierWithException<IS, Exception>> createStateFactories() {
	return Stream.of(
		Tuple2.of(ValueStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createValueState),
		Tuple2.of(ListStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createListState),
		Tuple2.of(MapStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createMapState),
		Tuple2.of(ReducingStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createReducingState),
		Tuple2.of(AggregatingStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createAggregatingState),
		Tuple2.of(FoldingStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createFoldingState)
	).collect(Collectors.toMap(t -> t.f0, t -> t.f1));
}
 

@SuppressWarnings("unchecked")
private <T> IS createListState() throws Exception {
	ListStateDescriptor<T> listStateDesc = (ListStateDescriptor<T>) stateDesc;
	ListStateDescriptor<TtlValue<T>> ttlDescriptor = new ListStateDescriptor<>(
		stateDesc.getName(), new TtlSerializer<>(LongSerializer.INSTANCE, listStateDesc.getElementSerializer()));
	return (IS) new TtlListState<>(createTtlStateContext(ttlDescriptor));
}
 

@Test
@SuppressWarnings("rawtypes")
public void testReduceWithEvictorAndProcessFunction() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DummyReducer reducer = new DummyReducer();

	DataStream<Tuple2<String, Integer>> window1 = source
			.keyBy(0)
			.window(SlidingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS)))
			.evictor(CountEvictor.of(100))
			.reduce(
					reducer,
					new ProcessWindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple, TimeWindow>() {
						@Override
						public void process(
								Tuple tuple,
								Context context,
								Iterable<Tuple2<String, Integer>> elements,
								Collector<Tuple2<String, Integer>> out) throws Exception {
							for (Tuple2<String, Integer> in : elements) {
								out.collect(in);
							}
						}
					});

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof EvictingWindowOperator);
	EvictingWindowOperator<String, Tuple2<String, Integer>, ?, ?> winOperator = (EvictingWindowOperator<String, Tuple2<String, Integer>, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator.getEvictor() instanceof CountEvictor);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingEventTimeWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);

	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}