org.apache.flink.api.common.state.ValueStateDescriptor Java Examples

/**
 * Verify that an empty {@code ValueState} will yield the default value.
 */
@Test
public void testValueStateDefaultValue() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	ValueStateDescriptor<String> kvId = new ValueStateDescriptor<>("id", String.class, "Hello");

	ValueState<String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

	backend.setCurrentKey(1);
	assertEquals("Hello", state.value());

	state.update("Ciao");
	assertEquals("Ciao", state.value());

	state.clear();
	assertEquals("Hello", state.value());

	backend.dispose();
}
 

@Override
public void initializeState(StateInitializationContext context) throws Exception {
	super.initializeState(context);

	counterState = context
		.getKeyedStateStore()
		.getState(new ValueStateDescriptor<>(prefix + "counter-state", LongSerializer.INSTANCE));

	// set key manually to make RocksDBListState get the serialized key.
	setCurrentKey("10");

	if (context.isRestored()) {
		counter =  counterState.value();
		assertEquals(snapshotOutData, counter);
		counterState.clear();
	}
}
 

@Override
public void initializeState(StateInitializationContext context) throws Exception {
	super.initializeState(context);

	// initializeState through the provided context
	computationStates = context.getKeyedStateStore().getState(
		new ValueStateDescriptor<>(
			NFA_STATE_NAME,
			new NFAStateSerializer()));

	partialMatches = new SharedBuffer<>(context.getKeyedStateStore(), inputSerializer);

	elementQueueState = context.getKeyedStateStore().getMapState(
			new MapStateDescriptor<>(
					EVENT_QUEUE_STATE_NAME,
					LongSerializer.INSTANCE,
					new ListSerializer<>(inputSerializer)));

	migrateOldState();
}
 

@Override
public <N, UK, UV> StateMapView<N, UK, UV> getStateMapView(String stateName, MapViewTypeInfo<UK, UV> mapViewTypeInfo) throws Exception {
	MapStateDescriptor<UK, UV> mapStateDescriptor = new MapStateDescriptor<>(
		stateName,
		mapViewTypeInfo.getKeyType(),
		mapViewTypeInfo.getValueType());

	MapState<UK, UV> mapState = keyedStateBackend.getOrCreateKeyedState(windowSerializer, mapStateDescriptor);
	// explict cast to internal state
	InternalMapState<?, N, UK, UV> internalMapState = (InternalMapState<?, N, UK, UV>) mapState;

	if (mapViewTypeInfo.isNullAware()) {
		ValueStateDescriptor<UV> nullStateDescriptor = new ValueStateDescriptor<>(
			stateName + NULL_STATE_POSTFIX,
			mapViewTypeInfo.getValueType());
		ValueState<UV> nullState = keyedStateBackend.getOrCreateKeyedState(windowSerializer, nullStateDescriptor);
		// explict cast to internal state
		InternalValueState<?, N, UV> internalNullState = (InternalValueState<?, N, UV>) nullState;
		return new StateMapView.NamespacedStateMapViewWithKeysNullable<>(internalMapState, internalNullState);
	} else {
		return new StateMapView.NamespacedStateMapViewWithKeysNotNull<>(internalMapState);
	}
}
 

@Test
public void testValueStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = new StreamingRuntimeContext(
			createDescriptorCapturingMockOp(descriptorCapture, config),
			createMockEnvironment(),
			Collections.<String, Accumulator<?, ?>>emptyMap());

	ValueStateDescriptor<TaskInfo> descr = new ValueStateDescriptor<>("name", TaskInfo.class);
	context.getState(descr);

	StateDescriptor<?, ?> descrIntercepted = (StateDescriptor<?, ?>) descriptorCapture.get();
	TypeSerializer<?> serializer = descrIntercepted.getSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(serializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 

@Override
public void open(Configuration parameters) throws Exception {
    super.open(parameters);
    ValueStateDescriptor<Boolean> keyedStateDuplicated =
            new ValueStateDescriptor<>("KeyedStateDeduplication",
                    TypeInformation.of(new TypeHint<Boolean>() {}));
    // 状态 TTL 相关配置，过期时间设定为 36 小时
    StateTtlConfig ttlConfig = StateTtlConfig
            .newBuilder(Time.hours(36))
            .setUpdateType(StateTtlConfig.UpdateType.OnCreateAndWrite)
            .setStateVisibility(
                    StateTtlConfig.StateVisibility.NeverReturnExpired)
            .cleanupInRocksdbCompactFilter(50000000L)
            .build();
    // 开启 TTL
    keyedStateDuplicated.enableTimeToLive(ttlConfig);
    // 从状态后端恢复状态
    isExist = getRuntimeContext().getState(keyedStateDuplicated);
}
 

@Test
public void testValueStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = createRuntimeContext(descriptorCapture, config);
	ValueStateDescriptor<TaskInfo> descr = new ValueStateDescriptor<>("name", TaskInfo.class);
	context.getState(descr);

	StateDescriptor<?, ?> descrIntercepted = (StateDescriptor<?, ?>) descriptorCapture.get();
	TypeSerializer<?> serializer = descrIntercepted.getSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(serializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 

@Test
public void testValueStateWorkWithTtl() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
	try {
		ValueStateDescriptor<MutableLong> kvId = new ValueStateDescriptor<>("id", MutableLong.class);
		kvId.enableTimeToLive(StateTtlConfig.newBuilder(Time.seconds(1)).build());

		ValueState<MutableLong> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
		backend.setCurrentKey(1);
		state.update(new MutableLong());
		state.value();
	} finally {
		backend.close();
		backend.dispose();
	}
}
 

@Override
public void open(Configuration parameters) throws Exception {
	function = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
	function.open(new PerKeyStateDataViewStore(getRuntimeContext()));

	output = new JoinedRow();

	// input element are all binary row as they are came from network
	BaseRowTypeInfo inputType = new BaseRowTypeInfo(inputFieldTypes);
	// we keep the elements received in a map state indexed based on their ingestion time
	ListTypeInfo<BaseRow> rowListTypeInfo = new ListTypeInfo<BaseRow>(inputType);
	MapStateDescriptor<Long, List<BaseRow>> mapStateDescriptor = new MapStateDescriptor<Long, List<BaseRow>>(
		"inputState", BasicTypeInfo.LONG_TYPE_INFO, rowListTypeInfo);
	inputState = getRuntimeContext().getMapState(mapStateDescriptor);

	BaseRowTypeInfo accTypeInfo = new BaseRowTypeInfo(accTypes);
	ValueStateDescriptor<BaseRow> stateDescriptor =
		new ValueStateDescriptor<BaseRow>("accState", accTypeInfo);
	accState = getRuntimeContext().getState(stateDescriptor);

	initCleanupTimeState("ProcTimeBoundedRangeOverCleanupTime");
}
 

@Test
public void testValueStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = new StreamingRuntimeContext(
			createDescriptorCapturingMockOp(descriptorCapture, config),
			createMockEnvironment(),
			Collections.<String, Accumulator<?, ?>>emptyMap());

	ValueStateDescriptor<TaskInfo> descr = new ValueStateDescriptor<>("name", TaskInfo.class);
	context.getState(descr);

	StateDescriptor<?, ?> descrIntercepted = (StateDescriptor<?, ?>) descriptorCapture.get();
	TypeSerializer<?> serializer = descrIntercepted.getSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(serializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 

@Override
public void open() throws Exception {
	super.open();

	LOG.info("Opening RowTimeSortOperator");
	if (gComparator != null) {
		comparator = gComparator.newInstance(getContainingTask().getUserCodeClassLoader());
		gComparator = null;
	}

	BasicTypeInfo<Long> keyTypeInfo = BasicTypeInfo.LONG_TYPE_INFO;
	ListTypeInfo<RowData> valueTypeInfo = new ListTypeInfo<>(inputRowType);
	MapStateDescriptor<Long, List<RowData>> mapStateDescriptor = new MapStateDescriptor<>(
			"dataState", keyTypeInfo, valueTypeInfo);
	dataState = getRuntimeContext().getMapState(mapStateDescriptor);

	ValueStateDescriptor<Long> lastTriggeringTsDescriptor = new ValueStateDescriptor<>("lastTriggeringTsState",
			Long.class);
	lastTriggeringTsState = getRuntimeContext().getState(lastTriggeringTsDescriptor);
}
 

@Override
public void initializeState(StateInitializationContext context) throws Exception {
	super.initializeState(context);

	counterState = context
		.getKeyedStateStore()
		.getState(new ValueStateDescriptor<>(prefix + "counter-state", LongSerializer.INSTANCE));

	// set key manually to make RocksDBListState get the serialized key.
	setCurrentKey("10");

	if (context.isRestored()) {
		counter =  counterState.value();
		assertEquals(snapshotOutData, counter);
		counterState.clear();
	}
}
 

@Override
public Tuple2<Long, Long> map(Tuple2<Long, Long> value) throws Exception {
	ValueState<Long> state = getRuntimeContext().getState(
		new ValueStateDescriptor<>("testState", new TestSerializer()));

	state.update(value.f1);
	return value;
}
 

@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);
	// instantiate function
	function = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
	function.open(new PerKeyStateDataViewStore(getRuntimeContext()));

	BaseRowTypeInfo accTypeInfo = new BaseRowTypeInfo(accTypes);
	ValueStateDescriptor<BaseRow> accDesc = new ValueStateDescriptor<>("accState", accTypeInfo);
	accState = getRuntimeContext().getState(accDesc);

	initCleanupTimeState("GroupTableAggregateCleanupTime");
}
 

@Override
public <S extends Serializable> ValueState<S> getKeyValueState(String name,
	TypeInformation<S> stateType,
	S defaultState) {

	checkNotNull(name, "The name of the state must not be null");
	checkNotNull(stateType, "The state type information must not be null");

	ValueStateDescriptor<S> stateDesc = new ValueStateDescriptor<>(name, stateType.createSerializer(getExecutionConfig()), defaultState);
	return getPartitionedState(stateDesc);
}
 

/**
 * Verify that the user-specified state backend is used even if checkpointing is disabled.
 */
@Test
public void testStateBackendWithoutCheckpointing() throws Exception {

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(1);

	see.getConfig().setRestartStrategy(RestartStrategies.noRestart());
	see.setStateBackend(new FailingStateBackend());

	see.fromElements(new Tuple2<>("Hello", 1))
		.keyBy(0)
		.map(new RichMapFunction<Tuple2<String, Integer>, String>() {
			private static final long serialVersionUID = 1L;

			@Override
			public void open(Configuration parameters) throws Exception {
				super.open(parameters);
				getRuntimeContext().getState(new ValueStateDescriptor<>("Test", Integer.class));
			}

			@Override
			public String map(Tuple2<String, Integer> value) throws Exception {
				return value.f0;
			}
		})
		.print();

	try {
		see.execute();
		fail();
	}
	catch (JobExecutionException e) {
		assertTrue(ExceptionUtils.findThrowable(e, SuccessException.class).isPresent());
	}
}
 

private static void setKey(
	AbstractKeyedStateBackend<Integer> backend,
	ValueStateDescriptor<Integer> descriptor,
	int key) throws Exception {
	backend.setCurrentKey(key);
	backend
		.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, descriptor)
		.update(0);
}
 

@Override
public void clear(TimeWindow w, TriggerContext ctx) throws Exception {
    // clear trigger state
    ValueState<Boolean> firstSeen = ctx.getPartitionedState(
        new ValueStateDescriptor<>("firstSeen", Types.BOOLEAN));
    firstSeen.clear();
}
 

@Override
public void open(InternalWindowProcessFunction.Context<?, CountWindow> ctx) throws Exception {
	String descriptorName = "tumble-count-assigner";
	ValueStateDescriptor<Long> countDescriptor = new ValueStateDescriptor<>(
		descriptorName,
		Types.LONG);
	this.count = ctx.getPartitionedState(countDescriptor);
}
 

@Override
public void open(InternalWindowProcessFunction.Context<?, CountWindow> ctx) throws Exception {
	String descriptorName = "slide-count-assigner";
	ValueStateDescriptor<Long> countDescriptor = new ValueStateDescriptor<>(
		descriptorName,
		Types.LONG);
	this.count = ctx.getPartitionedState(countDescriptor);
}
 

/**
 * Publishes the keyed stream as queryable ValueState instance.
 *
 * @param queryableStateName Name under which to the publish the queryable state instance
 * @return Queryable state instance
 */
@PublicEvolving
public QueryableStateStream<KEY, T> asQueryableState(String queryableStateName) {
	ValueStateDescriptor<T> valueStateDescriptor = new ValueStateDescriptor<T>(
			UUID.randomUUID().toString(),
			getType());

	return asQueryableState(queryableStateName, valueStateDescriptor);
}
 

@Override
public <T> ValueState<T> bindValue(String id, StateSpec<ValueState<T>> spec, Coder<T> coder) {
  try {
    keyedStateBackend.getOrCreateKeyedState(
        StringSerializer.INSTANCE,
        new ValueStateDescriptor<>(id, new CoderTypeSerializer<>(coder)));
  } catch (Exception e) {
    throw new RuntimeException(e);
  }

  return null;
}
 

@Override
public void processElement(StreamRecord<String> element) throws Exception {
	// we also don't care

	ValueState<String> state = getPartitionedState(
			VoidNamespace.INSTANCE,
			VoidNamespaceSerializer.INSTANCE,
			new ValueStateDescriptor<>("count", StringSerializer.INSTANCE));

	state.update(element.getValue());
}
 

@Override
public void open(Configuration parameters) {
	ValueStateDescriptor<List<Tuple2<Event, Integer>>> previousWindowDescriptor =
		new ValueStateDescriptor<>("eventsSeenSoFar",
			new ListTypeInfo<>(new TupleTypeInfo<>(TypeInformation.of(Event.class), BasicTypeInfo.INT_TYPE_INFO)));

	eventsSeenSoFar = getRuntimeContext().getState(previousWindowDescriptor);

	ValueStateDescriptor<Long> lastSequenceNumberDescriptor =
		new ValueStateDescriptor<>("lastSequenceNumber", BasicTypeInfo.LONG_TYPE_INFO);

	lastSequenceNumber = getRuntimeContext().getState(lastSequenceNumberDescriptor);
}
 

private TestValueState() throws Exception {
	this.state = backend.createInternalState(
		VoidNamespaceSerializer.INSTANCE,
		new ValueStateDescriptor<>("TestValueState", StringSerializer.INSTANCE),
		snapshotTransformFactory);
	state.setCurrentNamespace(VoidNamespace.INSTANCE);
}
 

@Override
public void open() throws Exception {
    super.open();

    if (networkState == null) {
        networkState = getPartitionedState(new ValueStateDescriptor<Network>(
                HTM_INFERENCE_OPERATOR_STATE_NAME,
                new KryoSerializer<Network>((Class<Network>) (Class<?>) Network.class, getExecutionConfig())
        ));
    }

    initInputFunction();
}
 

@SuppressWarnings("unchecked")
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	pojoClass = getRuntimeContext().getUserCodeClassLoader().loadClass(POJO_NAME);

	fieldA = pojoClass.getDeclaredField("a");
	fieldA.setAccessible(true);

	if (hasBField) {
		fieldB = pojoClass.getDeclaredField("b");
		fieldB.setAccessible(true);
	}

	if (keyed) {
		keyedValueState = context.getKeyedStateStore().getState(
			new ValueStateDescriptor<>("keyedValueState", (Class<Object>) pojoClass));
		keyedListState = context.getKeyedStateStore().getListState(
			new ListStateDescriptor<>("keyedListState", (Class<Object>) pojoClass));

		ReduceFunction<Object> reduceFunction = new FirstValueReducer<>();
		keyedReducingState = context.getKeyedStateStore().getReducingState(
			new ReducingStateDescriptor<>("keyedReducingState", reduceFunction, (Class<Object>) pojoClass));
	} else {
		partitionableListState = context.getOperatorStateStore().getListState(
			new ListStateDescriptor<>("partitionableListState", (Class<Object>) pojoClass));
		unionListState = context.getOperatorStateStore().getUnionListState(
			new ListStateDescriptor<>("unionListState", (Class<Object>) pojoClass));
	}
}
 

@Test
public void testKeyedValueStateSerializerReconfiguration() throws Exception {
	final String stateName = "test-name";

	testKeyedValueStateUpgrade(
		new ValueStateDescriptor<>(
			stateName,
			new TestType.V1TestTypeSerializer()),
		new ValueStateDescriptor<>(
			stateName,
			// the test fails if this serializer is used instead of a reconfigured new serializer
			new TestType.ReconfigurationRequiringTestTypeSerializer()));
}
 

@Override
public void open(Configuration parameters) throws Exception {
	function = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
	function.open(new PerKeyStateDataViewStore(getRuntimeContext()));

	output = new JoinedRow();

	ValueStateDescriptor<Long> lastTriggeringTsDescriptor = new ValueStateDescriptor<Long>(
		"lastTriggeringTsState",
		Types.LONG);
	lastTriggeringTsState = getRuntimeContext().getState(lastTriggeringTsDescriptor);

	BaseRowTypeInfo accTypeInfo = new BaseRowTypeInfo(accTypes);
	ValueStateDescriptor<BaseRow> accStateDesc = new ValueStateDescriptor<BaseRow>("accState", accTypeInfo);
	accState = getRuntimeContext().getState(accStateDesc);

	// input element are all binary row as they are came from network
	BaseRowTypeInfo inputType = new BaseRowTypeInfo(inputFieldTypes);
	ListTypeInfo<BaseRow> rowListTypeInfo = new ListTypeInfo<BaseRow>(inputType);
	MapStateDescriptor<Long, List<BaseRow>> inputStateDesc = new MapStateDescriptor<Long, List<BaseRow>>(
		"inputState",
		Types.LONG,
		rowListTypeInfo);
	inputState = getRuntimeContext().getMapState(inputStateDesc);

	initCleanupTimeState("RowTimeBoundedRangeOverCleanupTime");
}
 

/**
 * Tests simple value state queryable state instance. Each source emits
 * (subtaskIndex, 0)..(subtaskIndex, numElements) tuples, which are then
 * queried. The tests succeeds after each subtask index is queried with
 * value numElements (the latest element updated the state).
 */
@Test
public void testValueState() throws Exception {
	final Deadline deadline = Deadline.now().plus(TEST_TIMEOUT);
	final long numElements = 1024L;

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStateBackend(stateBackend);
	env.setParallelism(maxParallelism);
	// Very important, because cluster is shared between tests and we
	// don't explicitly check that all slots are available before
	// submitting.
	env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 1000L));

	DataStream<Tuple2<Integer, Long>> source = env.addSource(new TestAscendingValueSource(numElements));

	// Value state
	ValueStateDescriptor<Tuple2<Integer, Long>> valueState = new ValueStateDescriptor<>("any", source.getType());

	source.keyBy(new KeySelector<Tuple2<Integer, Long>, Integer>() {
		private static final long serialVersionUID = 7662520075515707428L;

		@Override
		public Integer getKey(Tuple2<Integer, Long> value) {
			return value.f0;
		}
	}).asQueryableState("hakuna", valueState);

	try (AutoCancellableJob autoCancellableJob = new AutoCancellableJob(deadline, clusterClient, env)) {

		final JobID jobId = autoCancellableJob.getJobId();
		final JobGraph jobGraph = autoCancellableJob.getJobGraph();

		ClientUtils.submitJob(clusterClient, jobGraph);
		executeValueQuery(deadline, client, jobId, "hakuna", valueState, numElements);
	}
}