Java Code Examples for org.apache.flink.api.common.state.ValueState#value()

The following examples show how to use org.apache.flink.api.common.state.ValueState#value() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: StateBackendTestBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testValueStateWorkWithTtl() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
	try {
		ValueStateDescriptor<MutableLong> kvId = new ValueStateDescriptor<>("id", MutableLong.class);
		kvId.enableTimeToLive(StateTtlConfig.newBuilder(Time.seconds(1)).build());

		ValueState<MutableLong> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
		backend.setCurrentKey(1);
		state.update(new MutableLong());
		state.value();
	} finally {
		backend.close();
		backend.dispose();
	}
}
 
Example 2
Source File: CleanupState.java    From flink with Apache License 2.0 6 votes vote down vote up
default void registerProcessingCleanupTimer(
		ValueState<Long> cleanupTimeState,
		long currentTime,
		long minRetentionTime,
		long maxRetentionTime,
		TimerService timerService) throws Exception {

	// last registered timer
	Long curCleanupTime = cleanupTimeState.value();

	// check if a cleanup timer is registered and
	// that the current cleanup timer won't delete state we need to keep
	if (curCleanupTime == null || (currentTime + minRetentionTime) > curCleanupTime) {
		// we need to register a new (later) timer
		long cleanupTime = currentTime + maxRetentionTime;
		// register timer and remember clean-up time
		timerService.registerProcessingTimeTimer(cleanupTime);
		// delete expired timer
		if (curCleanupTime != null) {
			timerService.deleteProcessingTimeTimer(curCleanupTime);
		}
		cleanupTimeState.update(cleanupTime);
	}
}
 
Example 3
Source File: CustomWindow.java    From examples-java with Apache License 2.0 6 votes vote down vote up
@Override
public TriggerResult onElement(SensorReading r, long ts, TimeWindow w, TriggerContext ctx) throws Exception {
    // firstSeen will be false if not set yet
    ValueState<Boolean> firstSeen = ctx.getPartitionedState(
        new ValueStateDescriptor<>("firstSeen", Types.BOOLEAN));

    // register initial timer only for first element
    if (firstSeen.value() == null) {
        // compute time for next early firing by rounding watermark to second
        long t = ctx.getCurrentWatermark() + (1000 - (ctx.getCurrentWatermark() % 1000));
        ctx.registerEventTimeTimer(t);
        // register timer for the end of the window
        ctx.registerEventTimeTimer(w.getEnd());
        firstSeen.update(true);
    }
    // Continue. Do not evaluate window per element
    return TriggerResult.CONTINUE;
}
 
Example 4
Source File: DeduplicateFunctionHelper.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Processes element to deduplicate on keys, sends current element as last row, retracts previous element if
 * needed.
 *
 * @param currentRow latest row received by deduplicate function
 * @param generateRetraction whether need to send retract message to downstream
 * @param state state of function
 * @param out underlying collector
 * @throws Exception
 */
static void processLastRow(BaseRow currentRow, boolean generateRetraction, ValueState<BaseRow> state,
		Collector<BaseRow> out) throws Exception {
	// Check message should be accumulate
	Preconditions.checkArgument(BaseRowUtil.isAccumulateMsg(currentRow));
	if (generateRetraction) {
		// state stores complete row if generateRetraction is true
		BaseRow preRow = state.value();
		state.update(currentRow);
		if (preRow != null) {
			preRow.setHeader(BaseRowUtil.RETRACT_MSG);
			out.collect(preRow);
		}
	}
	out.collect(currentRow);
}
 
Example 5
Source File: StateBackendTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testValueStateWorkWithTtl() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
	try {
		ValueStateDescriptor<MutableLong> kvId = new ValueStateDescriptor<>("id", MutableLong.class);
		kvId.enableTimeToLive(StateTtlConfig.newBuilder(Time.seconds(1)).build());

		ValueState<MutableLong> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
		backend.setCurrentKey(1);
		state.update(new MutableLong());
		state.value();
	} finally {
		backend.close();
		backend.dispose();
	}
}
 
Example 6
Source File: LegacyKeyedCoProcessOperatorTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private void handleValue(
	Object value,
	Collector<String> out,
	TimerService timerService,
	int channel) throws IOException {
	final ValueState<String> state = getRuntimeContext().getState(this.state);
	if (state.value() == null) {
		out.collect("INPUT" + channel + ":" + value);
		state.update(String.valueOf(value));
		timerService.registerEventTimeTimer(timerService.currentWatermark() + 5);
	} else {
		state.clear();
		timerService.deleteEventTimeTimer(timerService.currentWatermark() + 4);
	}
}
 
Example 7
Source File: DeltaTrigger.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public TriggerResult onElement(T element, long timestamp, W window, TriggerContext ctx) throws Exception {
	ValueState<T> lastElementState = ctx.getPartitionedState(stateDesc);
	if (lastElementState.value() == null) {
		lastElementState.update(element);
		return TriggerResult.CONTINUE;
	}
	if (deltaFunction.getDelta(lastElementState.value(), element) > this.threshold) {
		lastElementState.update(element);
		return TriggerResult.FIRE;
	}
	return TriggerResult.CONTINUE;
}
 
Example 8
Source File: StatefulStreamingJob.java    From flink with Apache License 2.0 5 votes vote down vote up
private static <T> void touchState(ValueState<T> state, Supplier<T> elements) throws IOException {
	T elem = state.value();
	if (elem == null) {
		elem = elements.get();
	}
	state.update(elem);
}
 
Example 9
Source File: DeduplicateFunctionHelper.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Processes element to deduplicate on keys, sends current element as last row, retracts previous element if
 * needed.
 *
 * @param currentRow latest row received by deduplicate function
 * @param generateUpdateBefore whether need to send UPDATE_BEFORE message for updates
 * @param state state of function, null if generateUpdateBefore is false
 * @param out underlying collector
 */
static void processLastRow(
		RowData currentRow,
		boolean generateUpdateBefore,
		boolean generateInsert,
		ValueState<RowData> state,
		Collector<RowData> out) throws Exception {
	// check message should be insert only.
	Preconditions.checkArgument(currentRow.getRowKind() == RowKind.INSERT);

	if (generateUpdateBefore || generateInsert) {
		// use state to keep the previous row content if we need to generate UPDATE_BEFORE
		// or use to distinguish the first row, if we need to generate INSERT
		RowData preRow = state.value();
		state.update(currentRow);
		if (preRow == null) {
			// the first row, send INSERT message
			currentRow.setRowKind(RowKind.INSERT);
			out.collect(currentRow);
		} else {
			if (generateUpdateBefore) {
				preRow.setRowKind(RowKind.UPDATE_BEFORE);
				out.collect(preRow);
			}
			currentRow.setRowKind(RowKind.UPDATE_AFTER);
			out.collect(currentRow);
		}
	} else {
		// always send UPDATE_AFTER if INSERT is not needed
		currentRow.setRowKind(RowKind.UPDATE_AFTER);
		out.collect(currentRow);
	}
}
 
Example 10
Source File: DeduplicateFunctionHelper.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Processes element to deduplicate on keys, sends current element if it is first row.
 *
 * @param currentRow latest row received by deduplicate function
 * @param state state of function
 * @param out underlying collector
 */
static void processFirstRow(
		RowData currentRow,
		ValueState<Boolean> state,
		Collector<RowData> out) throws Exception {
	// check message should be insert only.
	Preconditions.checkArgument(currentRow.getRowKind() == RowKind.INSERT);
	// ignore record if it is not first row
	if (state.value() != null) {
		return;
	}
	state.update(true);
	// emit the first row which is INSERT message
	out.collect(currentRow);
}
 
Example 11
Source File: KeyedCoProcessOperatorTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private void handleValue(
	Object value,
	Collector<String> out,
	TimerService timerService,
	int channel) throws IOException {
	final ValueState<String> state = getRuntimeContext().getState(this.state);
	if (state.value() == null) {
		out.collect("INPUT" + channel + ":" + value);
		state.update(String.valueOf(value));
		timerService.registerEventTimeTimer(timerService.currentWatermark() + 5);
	} else {
		state.clear();
		timerService.deleteEventTimeTimer(timerService.currentWatermark() + 4);
	}
}
 
Example 12
Source File: KeyedCoProcessOperatorTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private void handleValue(
	Object value,
	Collector<String> out,
	TimerService timerService,
	int channel) throws IOException {
	final ValueState<String> state = getRuntimeContext().getState(this.state);
	if (state.value() == null) {
		out.collect("INPUT" + channel + ":" + value);
		state.update(String.valueOf(value));
		timerService.registerEventTimeTimer(timerService.currentWatermark() + 5);
	} else {
		state.clear();
		timerService.deleteEventTimeTimer(timerService.currentWatermark() + 4);
	}
}
 
Example 13
Source File: DeltaTrigger.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public TriggerResult onElement(T element, long timestamp, W window, TriggerContext ctx) throws Exception {
	ValueState<T> lastElementState = ctx.getPartitionedState(stateDesc);
	if (lastElementState.value() == null) {
		lastElementState.update(element);
		return TriggerResult.CONTINUE;
	}
	if (deltaFunction.getDelta(lastElementState.value(), element) > this.threshold) {
		lastElementState.update(element);
		return TriggerResult.FIRE;
	}
	return TriggerResult.CONTINUE;
}
 
Example 14
Source File: KeyedCoProcessOperatorTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private void handleValue(
	Object value,
	Collector<String> out,
	TimerService timerService,
	int channel) throws IOException {
	final ValueState<String> state = getRuntimeContext().getState(this.state);
	if (state.value() == null) {
		out.collect("INPUT" + channel + ":" + value);
		state.update(String.valueOf(value));
		timerService.registerProcessingTimeTimer(timerService.currentProcessingTime() + 5);
	} else {
		state.clear();
		timerService.deleteProcessingTimeTimer(timerService.currentProcessingTime() + 4);
	}
}
 
Example 15
Source File: StateBackendTestBase.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testValueStateRestoreWithWrongSerializers() throws Exception {
	CheckpointStreamFactory streamFactory = createStreamFactory();
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {
		ValueStateDescriptor<String> kvId = new ValueStateDescriptor<>("id", String.class);

		ValueState<String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

		backend.setCurrentKey(1);
		state.update("1");
		backend.setCurrentKey(2);
		state.update("2");

		// draw a snapshot
		KeyedStateHandle snapshot1 = runSnapshot(
			backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()),
			sharedStateRegistry);

		backend.dispose();
		// restore the first snapshot and validate it
		backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot1);
		snapshot1.discardState();

		@SuppressWarnings("unchecked")
		TypeSerializer<String> fakeStringSerializer =
			(TypeSerializer<String>) (TypeSerializer<?>) FloatSerializer.INSTANCE;

		try {
			kvId = new ValueStateDescriptor<>("id", fakeStringSerializer);

			state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

			state.value();

			fail("should recognize wrong serializers");
		} catch (StateMigrationException ignored) {
			// expected
		}
	} finally {
		backend.dispose();
	}
}
 
Example 16
Source File: StateBackendTestBase.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testValueStateRestoreWithWrongSerializers() throws Exception {
	CheckpointStreamFactory streamFactory = createStreamFactory();
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {
		ValueStateDescriptor<String> kvId = new ValueStateDescriptor<>("id", String.class);

		ValueState<String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

		backend.setCurrentKey(1);
		state.update("1");
		backend.setCurrentKey(2);
		state.update("2");

		// draw a snapshot
		KeyedStateHandle snapshot1 = runSnapshot(
			backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()),
			sharedStateRegistry);

		backend.dispose();
		// restore the first snapshot and validate it
		backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot1);
		snapshot1.discardState();

		@SuppressWarnings("unchecked")
		TypeSerializer<String> fakeStringSerializer =
			(TypeSerializer<String>) (TypeSerializer<?>) FloatSerializer.INSTANCE;

		try {
			kvId = new ValueStateDescriptor<>("id", fakeStringSerializer);

			state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

			state.value();

			fail("should recognize wrong serializers");
		} catch (StateMigrationException ignored) {
			// expected
		}
	} finally {
		backend.dispose();
	}
}
 
Example 17
Source File: TtlValueStateVerifier.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Override
String getInternal(@Nonnull ValueState<String> state) throws Exception {
	return state.value();
}
 
Example 18
Source File: TtlValueStateVerifier.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
String getInternal(@Nonnull ValueState<String> state) throws Exception {
	return state.value();
}
 
Example 19
Source File: StateBackendTestBase.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Verify state restore resilience when:
 *  - snapshot was taken without any Kryo registrations, specific serializers or default serializers for the state type
 *  - restored with a specific serializer for the state type
 *
 * <p> The specific serializer used on restore is {@link CustomKryoTestSerializer}, which deliberately
 * fails only on deserialization. We use the deliberate deserialization failure to acknowledge test success.
 *
 * @throws Exception expects {@link ExpectedKryoTestException} to be thrown.
 */
@Test
public void testKryoRegisteringRestoreResilienceWithRegisteredSerializer() throws Exception {
	CheckpointStreamFactory streamFactory = createStreamFactory();
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();

	AbstractKeyedStateBackend<Integer> backend = null;

	try {
		backend = createKeyedBackend(IntSerializer.INSTANCE, env);

		TypeInformation<TestPojo> pojoType = new GenericTypeInfo<>(TestPojo.class);

		// make sure that we are in fact using the KryoSerializer
		assertTrue(pojoType.createSerializer(env.getExecutionConfig()) instanceof KryoSerializer);

		ValueStateDescriptor<TestPojo> kvId = new ValueStateDescriptor<>("id", pojoType);
		ValueState<TestPojo> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

		// ============== create snapshot - no Kryo registration or specific / default serializers ==============

		// make some more modifications
		backend.setCurrentKey(1);
		state.update(new TestPojo("u1", 1));

		backend.setCurrentKey(2);
		state.update(new TestPojo("u2", 2));

		KeyedStateHandle snapshot = runSnapshot(
			backend.snapshot(
				682375462378L,
				2,
				streamFactory,
				CheckpointOptions.forCheckpointWithDefaultLocation()),
			sharedStateRegistry);

		backend.dispose();

		// ========== restore snapshot - should use specific serializer (ONLY SERIALIZATION) ==========

		env.getExecutionConfig().registerTypeWithKryoSerializer(TestPojo.class, CustomKryoTestSerializer.class);

		backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot, env);

		// re-initialize to ensure that we create the KryoSerializer from scratch, otherwise
		// initializeSerializerUnlessSet would not pick up our new config
		kvId = new ValueStateDescriptor<>("id", pojoType);
		state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

		backend.setCurrentKey(1);

		// update to test state backends that eagerly serialize, such as RocksDB
		state.update(new TestPojo("u1", 11));

		KeyedStateHandle snapshot2 = runSnapshot(
			backend.snapshot(
				682375462378L,
				2,
				streamFactory,
				CheckpointOptions.forCheckpointWithDefaultLocation()),
			sharedStateRegistry);

		snapshot.discardState();

		backend.dispose();

		// ========= restore snapshot - should use specific serializer (FAIL ON DESERIALIZATION) =========

		env.getExecutionConfig().registerTypeWithKryoSerializer(TestPojo.class, CustomKryoTestSerializer.class);

		// on the second restore, since the custom serializer will be used for
		// deserialization, we expect the deliberate failure to be thrown
		expectedException.expect(anyOf(isA(ExpectedKryoTestException.class),
			Matchers.<Throwable>hasProperty("cause", isA(ExpectedKryoTestException.class))));

		// state backends that eagerly deserializes (such as the memory state backend) will fail here
		backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot2, env);

		state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

		backend.setCurrentKey(1);
		// state backends that lazily deserializes (such as RocksDB) will fail here
		state.value();

		backend.dispose();
	} finally {
		// ensure that native resources are also released in case of exception
		if (backend != null) {
			backend.dispose();
		}
	}
}
 
Example 20
Source File: StateBackendTestBase.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testValueStateRestoreWithWrongSerializers() throws Exception {
	CheckpointStreamFactory streamFactory = createStreamFactory();
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {
		ValueStateDescriptor<String> kvId = new ValueStateDescriptor<>("id", String.class);

		ValueState<String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

		backend.setCurrentKey(1);
		state.update("1");
		backend.setCurrentKey(2);
		state.update("2");

		// draw a snapshot
		KeyedStateHandle snapshot1 = runSnapshot(
			backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()),
			sharedStateRegistry);

		backend.dispose();
		// restore the first snapshot and validate it
		backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot1);
		snapshot1.discardState();

		@SuppressWarnings("unchecked")
		TypeSerializer<String> fakeStringSerializer =
			(TypeSerializer<String>) (TypeSerializer<?>) FloatSerializer.INSTANCE;

		try {
			kvId = new ValueStateDescriptor<>("id", fakeStringSerializer);

			state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

			state.value();

			fail("should recognize wrong serializers");
		} catch (StateMigrationException ignored) {
			// expected
		}
	} finally {
		backend.dispose();
	}
}