org.apache.flink.runtime.state.internal.InternalMapState Java Examples

The following examples show how to use org.apache.flink.runtime.state.internal.InternalMapState. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: KVStateRequestSerializerRocksDBTest.java From flink with Apache License 2.0

6 votes

/**
 * Tests map serialization and deserialization match.
 *
 * @see KvStateRequestSerializerTest#testMapSerialization()
 * KvStateRequestSerializerTest#testMapSerialization() using the heap state back-end
 * test
 */
@Test
public void testMapSerialization() throws Exception {
	final long key = 0L;

	// objects for RocksDB state list serialisation
	final RocksDBKeyedStateBackend<Long> longHeapKeyedStateBackend = RocksDBTestUtils
		.builderForTestDefaults(temporaryFolder.getRoot(), LongSerializer.INSTANCE)
		.build();

	longHeapKeyedStateBackend.setCurrentKey(key);

	final InternalMapState<Long, VoidNamespace, Long, String> mapState =
			(InternalMapState<Long, VoidNamespace, Long, String>)
					longHeapKeyedStateBackend.getPartitionedState(
							VoidNamespace.INSTANCE,
							VoidNamespaceSerializer.INSTANCE,
							new MapStateDescriptor<>("test", LongSerializer.INSTANCE, StringSerializer.INSTANCE));

	KvStateRequestSerializerTest.testMapSerialization(key, mapState);
	longHeapKeyedStateBackend.dispose();
}

Example #2

Source File: PerWindowStateDataViewStore.java From flink with Apache License 2.0

6 votes

@Override
public <N, UK, UV> StateMapView<N, UK, UV> getStateMapView(String stateName, MapViewTypeInfo<UK, UV> mapViewTypeInfo) throws Exception {
	MapStateDescriptor<UK, UV> mapStateDescriptor = new MapStateDescriptor<>(
		stateName,
		mapViewTypeInfo.getKeyType(),
		mapViewTypeInfo.getValueType());

	MapState<UK, UV> mapState = keyedStateBackend.getOrCreateKeyedState(windowSerializer, mapStateDescriptor);
	// explict cast to internal state
	InternalMapState<?, N, UK, UV> internalMapState = (InternalMapState<?, N, UK, UV>) mapState;

	if (mapViewTypeInfo.isNullAware()) {
		ValueStateDescriptor<UV> nullStateDescriptor = new ValueStateDescriptor<>(
			stateName + NULL_STATE_POSTFIX,
			mapViewTypeInfo.getValueType());
		ValueState<UV> nullState = keyedStateBackend.getOrCreateKeyedState(windowSerializer, nullStateDescriptor);
		// explict cast to internal state
		InternalValueState<?, N, UV> internalNullState = (InternalValueState<?, N, UV>) nullState;
		return new StateMapView.NamespacedStateMapViewWithKeysNullable<>(internalMapState, internalNullState);
	} else {
		return new StateMapView.NamespacedStateMapViewWithKeysNotNull<>(internalMapState);
	}
}

Example #3

Source File: PerWindowStateDataViewStore.java From flink with Apache License 2.0

6 votes

@Override
public <N, UK, UV> StateMapView<N, UK, UV> getStateMapView(String stateName, MapViewTypeInfo<UK, UV> mapViewTypeInfo) throws Exception {
	MapStateDescriptor<UK, UV> mapStateDescriptor = new MapStateDescriptor<>(
		stateName,
		mapViewTypeInfo.getKeyType(),
		mapViewTypeInfo.getValueType());

	MapState<UK, UV> mapState = keyedStateBackend.getOrCreateKeyedState(windowSerializer, mapStateDescriptor);
	// explict cast to internal state
	InternalMapState<?, N, UK, UV> internalMapState = (InternalMapState<?, N, UK, UV>) mapState;

	if (mapViewTypeInfo.isNullAware()) {
		ValueStateDescriptor<UV> nullStateDescriptor = new ValueStateDescriptor<>(
			stateName + NULL_STATE_POSTFIX,
			mapViewTypeInfo.getValueType());
		ValueState<UV> nullState = keyedStateBackend.getOrCreateKeyedState(windowSerializer, nullStateDescriptor);
		// explict cast to internal state
		InternalValueState<?, N, UV> internalNullState = (InternalValueState<?, N, UV>) nullState;
		return new StateMapView.NamespacedStateMapViewWithKeysNullable<>(internalMapState, internalNullState);
	} else {
		return new StateMapView.NamespacedStateMapViewWithKeysNotNull<>(internalMapState);
	}
}

Example #4

Source File: KvStateRequestSerializerTest.java From flink with Apache License 2.0

5 votes

/**
 * Tests map serialization utils.
 */
@Test
public void testMapSerialization() throws Exception {
	final long key = 0L;
	final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend = getLongHeapKeyedStateBackend(key);

	final InternalMapState<Long, VoidNamespace, Long, String> mapState =
			(InternalMapState<Long, VoidNamespace, Long, String>)
					longHeapKeyedStateBackend.getPartitionedState(
							VoidNamespace.INSTANCE,
							VoidNamespaceSerializer.INSTANCE,
							new MapStateDescriptor<>("test", LongSerializer.INSTANCE, StringSerializer.INSTANCE));

	testMapSerialization(key, mapState);
}

Example #5

Source File: KvStateRequestSerializerTest.java From flink with Apache License 2.0

5 votes

/**
 * Tests map serialization utils.
 */
@Test
public void testMapSerialization() throws Exception {
	final long key = 0L;
	final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend = getLongHeapKeyedStateBackend(key);

	final InternalMapState<Long, VoidNamespace, Long, String> mapState =
			(InternalMapState<Long, VoidNamespace, Long, String>)
					longHeapKeyedStateBackend.getPartitionedState(
							VoidNamespace.INSTANCE,
							VoidNamespaceSerializer.INSTANCE,
							new MapStateDescriptor<>("test", LongSerializer.INSTANCE, StringSerializer.INSTANCE));

	testMapSerialization(key, mapState);
}

Example #6

Source File: KvStateRequestSerializerTest.java From Flink-CEPplus with Apache License 2.0

5 votes

/**
 * Tests map serialization utils.
 */
@Test
public void testMapSerialization() throws Exception {
	final long key = 0L;
	final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend = getLongHeapKeyedStateBackend(key);

	final InternalMapState<Long, VoidNamespace, Long, String> mapState =
			(InternalMapState<Long, VoidNamespace, Long, String>)
					longHeapKeyedStateBackend.getPartitionedState(
							VoidNamespace.INSTANCE,
							VoidNamespaceSerializer.INSTANCE,
							new MapStateDescriptor<>("test", LongSerializer.INSTANCE, StringSerializer.INSTANCE));

	testMapSerialization(key, mapState);
}

Example #7

Source File: KVStateRequestSerializerRocksDBTest.java From flink with Apache License 2.0

4 votes

/**
 * Tests map serialization and deserialization match.
 *
 * @see KvStateRequestSerializerTest#testMapSerialization()
 * KvStateRequestSerializerTest#testMapSerialization() using the heap state back-end
 * test
 */
@Test
public void testMapSerialization() throws Exception {
	final long key = 0L;

	// objects for RocksDB state list serialisation
	DBOptions dbOptions = PredefinedOptions.DEFAULT.createDBOptions();
	dbOptions.setCreateIfMissing(true);
	ExecutionConfig executionConfig = new ExecutionConfig();
	final RocksDBKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new RocksDBKeyedStateBackendBuilder<>(
			"no-op",
			ClassLoader.getSystemClassLoader(),
			temporaryFolder.getRoot(),
			dbOptions,
			stateName -> PredefinedOptions.DEFAULT.createColumnOptions(),
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			1,
			new KeyGroupRange(0, 0),
			executionConfig,
			TestLocalRecoveryConfig.disabled(),
			RocksDBStateBackend.PriorityQueueStateType.HEAP,
			TtlTimeProvider.DEFAULT,
			new UnregisteredMetricsGroup(),
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			new CloseableRegistry()
		).build();
	longHeapKeyedStateBackend.setCurrentKey(key);

	final InternalMapState<Long, VoidNamespace, Long, String> mapState =
			(InternalMapState<Long, VoidNamespace, Long, String>)
					longHeapKeyedStateBackend.getPartitionedState(
							VoidNamespace.INSTANCE,
							VoidNamespaceSerializer.INSTANCE,
							new MapStateDescriptor<>("test", LongSerializer.INSTANCE, StringSerializer.INSTANCE));

	KvStateRequestSerializerTest.testMapSerialization(key, mapState);
	longHeapKeyedStateBackend.dispose();
}

Example #8

Source File: HeapKeyedStateBackendSnapshotMigrationTest.java From Flink-CEPplus with Apache License 2.0

4 votes

@Test
public void testMapStateMigrationAfterHashMapSerRemoval() throws Exception {
	ClassLoader cl = getClass().getClassLoader();
	URL resource = cl.getResource("heap_keyed_statebackend_1_5_map.snapshot");

	Preconditions.checkNotNull(resource, "Binary snapshot resource not found!");

	final SnapshotResult<KeyedStateHandle> stateHandles;
	try (BufferedInputStream bis = new BufferedInputStream((new FileInputStream(resource.getFile())))) {
		stateHandles = InstantiationUtil.deserializeObject(bis, Thread.currentThread().getContextClassLoader());
	}
	final KeyedStateHandle stateHandle = stateHandles.getJobManagerOwnedSnapshot();
	try (final HeapKeyedStateBackend<String> keyedBackend = createKeyedBackend(StateObjectCollection.singleton(stateHandle))) {
		final Integer namespace1 = 1;
		final Integer namespace2 = 2;
		final Integer namespace3 = 3;

		final MapStateDescriptor<Long, Long> stateDescr = new MapStateDescriptor<>("my-map-state", Long.class, Long.class);
		stateDescr.initializeSerializerUnlessSet(new ExecutionConfig());

		InternalMapState<String, Integer, Long, Long> state = keyedBackend.createInternalState(IntSerializer.INSTANCE, stateDescr);

		keyedBackend.setCurrentKey("abc");
		state.setCurrentNamespace(namespace1);
		assertEquals(33L, (long) state.get(33L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(2, getStateSize(state));

		state.setCurrentNamespace(namespace2);
		assertEquals(22L, (long) state.get(22L));
		assertEquals(11L, (long) state.get(11L));
		assertEquals(2, getStateSize(state));

		state.setCurrentNamespace(namespace3);
		assertEquals(44L, (long) state.get(44L));
		assertEquals(1, getStateSize(state));

		keyedBackend.setCurrentKey("def");
		state.setCurrentNamespace(namespace1);
		assertEquals(11L, (long) state.get(11L));
		assertEquals(44L, (long) state.get(44L));
		assertEquals(2, getStateSize(state));

		state.setCurrentNamespace(namespace3);
		assertEquals(22L, (long) state.get(22L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(33L, (long) state.get(33L));
		assertEquals(3, getStateSize(state));

		keyedBackend.setCurrentKey("jkl");
		state.setCurrentNamespace(namespace1);
		assertEquals(11L, (long) state.get(11L));
		assertEquals(22L, (long) state.get(22L));
		assertEquals(33L, (long) state.get(33L));
		assertEquals(44L, (long) state.get(44L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(5, getStateSize(state));

		keyedBackend.setCurrentKey("mno");
		state.setCurrentNamespace(namespace3);
		assertEquals(11L, (long) state.get(11L));
		assertEquals(22L, (long) state.get(22L));
		assertEquals(33L, (long) state.get(33L));
		assertEquals(44L, (long) state.get(44L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(5, getStateSize(state));

		RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = keyedBackend.snapshot(
				1L,
				1L,
				new MemCheckpointStreamFactory(4 * 1024 * 1024),
				CheckpointOptions.forCheckpointWithDefaultLocation());

		snapshot.run();
	}
}

Example #9

Source File: KvStateRequestSerializerTest.java From flink with Apache License 2.0

4 votes

/**
 * Verifies that the serialization of a map using the given map state
 * matches the deserialization with {@link KvStateSerializer#deserializeList}.
 *
 * @param key
 * 		key of the map state
 * @param mapState
 * 		map state using the {@link VoidNamespace}, must also be a {@link InternalKvState} instance
 *
 * @throws Exception
 */
public static void testMapSerialization(
		final long key,
		final InternalMapState<Long, VoidNamespace, Long, String> mapState) throws Exception {

	TypeSerializer<Long> userKeySerializer = LongSerializer.INSTANCE;
	TypeSerializer<String> userValueSerializer = StringSerializer.INSTANCE;
	mapState.setCurrentNamespace(VoidNamespace.INSTANCE);

	// Map
	final int numElements = 10;

	final Map<Long, String> expectedValues = new HashMap<>();
	for (int i = 1; i <= numElements; i++) {
		final long value = ThreadLocalRandom.current().nextLong();
		expectedValues.put(value, Long.toString(value));
		mapState.put(value, Long.toString(value));
	}

	expectedValues.put(0L, null);
	mapState.put(0L, null);

	final byte[] serializedKey =
		KvStateSerializer.serializeKeyAndNamespace(
			key, LongSerializer.INSTANCE,
			VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE);

	final byte[] serializedValues = mapState.getSerializedValue(
			serializedKey,
			mapState.getKeySerializer(),
			mapState.getNamespaceSerializer(),
			mapState.getValueSerializer());

	Map<Long, String> actualValues = KvStateSerializer.deserializeMap(serializedValues, userKeySerializer, userValueSerializer);
	assertEquals(expectedValues.size(), actualValues.size());
	for (Map.Entry<Long, String> actualEntry : actualValues.entrySet()) {
		assertEquals(expectedValues.get(actualEntry.getKey()), actualEntry.getValue());
	}

	// Single value
	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	long expectedKey = ThreadLocalRandom.current().nextLong();
	String expectedValue = Long.toString(expectedKey);
	byte[] isNull = {0};

	baos.write(KvStateSerializer.serializeValue(expectedKey, userKeySerializer));
	baos.write(isNull);
	baos.write(KvStateSerializer.serializeValue(expectedValue, userValueSerializer));
	byte[] serializedValue = baos.toByteArray();

	Map<Long, String> actualValue = KvStateSerializer.deserializeMap(serializedValue, userKeySerializer, userValueSerializer);
	assertEquals(1, actualValue.size());
	assertEquals(expectedValue, actualValue.get(expectedKey));
}

Example #10

Source File: HeapKeyedStateBackendSnapshotMigrationTest.java From Flink-CEPplus with Apache License 2.0

4 votes

private <K, N, UK, UV> int getStateSize(InternalMapState<K, N, UK, UV> mapState) throws Exception {
	int i = 0;
	for (Iterator<Map.Entry<UK, UV>> itt = mapState.iterator(); itt.hasNext(); i++, itt.next()) {}
	return i;
}

Example #11

Source File: HeapKeyedStateBackendSnapshotMigrationTest.java From flink with Apache License 2.0

4 votes

private <K, N, UK, UV> int getStateSize(InternalMapState<K, N, UK, UV> mapState) throws Exception {
	int i = 0;
	for (Iterator<Map.Entry<UK, UV>> itt = mapState.iterator(); itt.hasNext(); i++, itt.next()) {}
	return i;
}

Example #12

Source File: HeapKeyedStateBackendSnapshotMigrationTest.java From flink with Apache License 2.0

4 votes

@Test
public void testMapStateMigrationAfterHashMapSerRemoval() throws Exception {
	ClassLoader cl = getClass().getClassLoader();
	URL resource = cl.getResource("heap_keyed_statebackend_1_5_map.snapshot");

	Preconditions.checkNotNull(resource, "Binary snapshot resource not found!");

	final SnapshotResult<KeyedStateHandle> stateHandles;
	try (BufferedInputStream bis = new BufferedInputStream((new FileInputStream(resource.getFile())))) {
		stateHandles = InstantiationUtil.deserializeObject(bis, Thread.currentThread().getContextClassLoader());
	}
	final KeyedStateHandle stateHandle = stateHandles.getJobManagerOwnedSnapshot();
	try (final HeapKeyedStateBackend<String> keyedBackend = createKeyedBackend(StateObjectCollection.singleton(stateHandle))) {
		final Integer namespace1 = 1;
		final Integer namespace2 = 2;
		final Integer namespace3 = 3;

		final MapStateDescriptor<Long, Long> stateDescr = new MapStateDescriptor<>("my-map-state", Long.class, Long.class);
		stateDescr.initializeSerializerUnlessSet(new ExecutionConfig());

		InternalMapState<String, Integer, Long, Long> state = keyedBackend.createInternalState(IntSerializer.INSTANCE, stateDescr);

		keyedBackend.setCurrentKey("abc");
		state.setCurrentNamespace(namespace1);
		assertEquals(33L, (long) state.get(33L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(2, getStateSize(state));

		state.setCurrentNamespace(namespace2);
		assertEquals(22L, (long) state.get(22L));
		assertEquals(11L, (long) state.get(11L));
		assertEquals(2, getStateSize(state));

		state.setCurrentNamespace(namespace3);
		assertEquals(44L, (long) state.get(44L));
		assertEquals(1, getStateSize(state));

		keyedBackend.setCurrentKey("def");
		state.setCurrentNamespace(namespace1);
		assertEquals(11L, (long) state.get(11L));
		assertEquals(44L, (long) state.get(44L));
		assertEquals(2, getStateSize(state));

		state.setCurrentNamespace(namespace3);
		assertEquals(22L, (long) state.get(22L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(33L, (long) state.get(33L));
		assertEquals(3, getStateSize(state));

		keyedBackend.setCurrentKey("jkl");
		state.setCurrentNamespace(namespace1);
		assertEquals(11L, (long) state.get(11L));
		assertEquals(22L, (long) state.get(22L));
		assertEquals(33L, (long) state.get(33L));
		assertEquals(44L, (long) state.get(44L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(5, getStateSize(state));

		keyedBackend.setCurrentKey("mno");
		state.setCurrentNamespace(namespace3);
		assertEquals(11L, (long) state.get(11L));
		assertEquals(22L, (long) state.get(22L));
		assertEquals(33L, (long) state.get(33L));
		assertEquals(44L, (long) state.get(44L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(5, getStateSize(state));

		RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = keyedBackend.snapshot(
				1L,
				1L,
				new MemCheckpointStreamFactory(4 * 1024 * 1024),
				CheckpointOptions.forCheckpointWithDefaultLocation());

		snapshot.run();
	}
}

Example #13

Source File: TtlMapState.java From flink with Apache License 2.0

4 votes

TtlMapState(TtlStateContext<InternalMapState<K, N, UK, TtlValue<UV>>, Map<UK, UV>> ttlStateContext) {
	super(ttlStateContext);
}

Example #14

Source File: StateMapView.java From flink with Apache License 2.0

4 votes

public NamespacedStateMapViewWithKeysNullable(InternalMapState<?, N, MK, MV> internalMapState, InternalValueState<?, N, MV> internalNullState) {
	this.internalMapState = internalMapState;
	this.internalNullState = internalNullState;
}

Example #15

Source File: StateMapView.java From flink with Apache License 2.0

4 votes

public NamespacedStateMapViewWithKeysNotNull(InternalMapState<?, N, MK, MV> internalMapState) {
	this.internalMapState = internalMapState;
}

Example #16

Source File: KVStateRequestSerializerRocksDBTest.java From Flink-CEPplus with Apache License 2.0

4 votes

/**
 * Tests map serialization and deserialization match.
 *
 * @see KvStateRequestSerializerTest#testMapSerialization()
 * KvStateRequestSerializerTest#testMapSerialization() using the heap state back-end
 * test
 */
@Test
public void testMapSerialization() throws Exception {
	final long key = 0L;

	// objects for RocksDB state list serialisation
	DBOptions dbOptions = PredefinedOptions.DEFAULT.createDBOptions();
	dbOptions.setCreateIfMissing(true);
	ExecutionConfig executionConfig = new ExecutionConfig();
	final RocksDBKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new RocksDBKeyedStateBackendBuilder<>(
			"no-op",
			ClassLoader.getSystemClassLoader(),
			temporaryFolder.getRoot(),
			dbOptions,
			stateName -> PredefinedOptions.DEFAULT.createColumnOptions(),
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			1,
			new KeyGroupRange(0, 0),
			executionConfig,
			TestLocalRecoveryConfig.disabled(),
			RocksDBStateBackend.PriorityQueueStateType.HEAP,
			TtlTimeProvider.DEFAULT,
			new UnregisteredMetricsGroup(),
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			new CloseableRegistry()
		).build();
	longHeapKeyedStateBackend.setCurrentKey(key);

	final InternalMapState<Long, VoidNamespace, Long, String> mapState =
			(InternalMapState<Long, VoidNamespace, Long, String>)
					longHeapKeyedStateBackend.getPartitionedState(
							VoidNamespace.INSTANCE,
							VoidNamespaceSerializer.INSTANCE,
							new MapStateDescriptor<>("test", LongSerializer.INSTANCE, StringSerializer.INSTANCE));

	KvStateRequestSerializerTest.testMapSerialization(key, mapState);
	longHeapKeyedStateBackend.dispose();
}

Example #17

Source File: KvStateRequestSerializerTest.java From flink with Apache License 2.0

4 votes

/**
 * Verifies that the serialization of a map using the given map state
 * matches the deserialization with {@link KvStateSerializer#deserializeList}.
 *
 * @param key
 * 		key of the map state
 * @param mapState
 * 		map state using the {@link VoidNamespace}, must also be a {@link InternalKvState} instance
 *
 * @throws Exception
 */
public static void testMapSerialization(
		final long key,
		final InternalMapState<Long, VoidNamespace, Long, String> mapState) throws Exception {

	TypeSerializer<Long> userKeySerializer = LongSerializer.INSTANCE;
	TypeSerializer<String> userValueSerializer = StringSerializer.INSTANCE;
	mapState.setCurrentNamespace(VoidNamespace.INSTANCE);

	// Map
	final int numElements = 10;

	final Map<Long, String> expectedValues = new HashMap<>();
	for (int i = 1; i <= numElements; i++) {
		final long value = ThreadLocalRandom.current().nextLong();
		expectedValues.put(value, Long.toString(value));
		mapState.put(value, Long.toString(value));
	}

	expectedValues.put(0L, null);
	mapState.put(0L, null);

	final byte[] serializedKey =
		KvStateSerializer.serializeKeyAndNamespace(
			key, LongSerializer.INSTANCE,
			VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE);

	final byte[] serializedValues = mapState.getSerializedValue(
			serializedKey,
			mapState.getKeySerializer(),
			mapState.getNamespaceSerializer(),
			mapState.getValueSerializer());

	Map<Long, String> actualValues = KvStateSerializer.deserializeMap(serializedValues, userKeySerializer, userValueSerializer);
	assertEquals(expectedValues.size(), actualValues.size());
	for (Map.Entry<Long, String> actualEntry : actualValues.entrySet()) {
		assertEquals(expectedValues.get(actualEntry.getKey()), actualEntry.getValue());
	}

	// Single value
	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	long expectedKey = ThreadLocalRandom.current().nextLong();
	String expectedValue = Long.toString(expectedKey);
	byte[] isNull = {0};

	baos.write(KvStateSerializer.serializeValue(expectedKey, userKeySerializer));
	baos.write(isNull);
	baos.write(KvStateSerializer.serializeValue(expectedValue, userValueSerializer));
	byte[] serializedValue = baos.toByteArray();

	Map<Long, String> actualValue = KvStateSerializer.deserializeMap(serializedValue, userKeySerializer, userValueSerializer);
	assertEquals(1, actualValue.size());
	assertEquals(expectedValue, actualValue.get(expectedKey));
}

Example #18

Source File: TtlMapState.java From Flink-CEPplus with Apache License 2.0

4 votes

TtlMapState(TtlStateContext<InternalMapState<K, N, UK, TtlValue<UV>>, Map<UK, UV>> ttlStateContext) {
	super(ttlStateContext);
}

Example #19

Source File: HeapKeyedStateBackendSnapshotMigrationTest.java From flink with Apache License 2.0

4 votes

private <K, N, UK, UV> int getStateSize(InternalMapState<K, N, UK, UV> mapState) throws Exception {
	int i = 0;
	for (Iterator<Map.Entry<UK, UV>> itt = mapState.iterator(); itt.hasNext(); i++, itt.next()) {}
	return i;
}

Example #20

Source File: HeapKeyedStateBackendSnapshotMigrationTest.java From flink with Apache License 2.0

4 votes

@Test
public void testMapStateMigrationAfterHashMapSerRemoval() throws Exception {
	ClassLoader cl = getClass().getClassLoader();
	URL resource = cl.getResource("heap_keyed_statebackend_1_5_map.snapshot");

	Preconditions.checkNotNull(resource, "Binary snapshot resource not found!");

	final SnapshotResult<KeyedStateHandle> stateHandles;
	try (BufferedInputStream bis = new BufferedInputStream((new FileInputStream(resource.getFile())))) {
		stateHandles = InstantiationUtil.deserializeObject(bis, Thread.currentThread().getContextClassLoader());
	}
	final KeyedStateHandle stateHandle = stateHandles.getJobManagerOwnedSnapshot();
	try (final HeapKeyedStateBackend<String> keyedBackend = createKeyedBackend(StateObjectCollection.singleton(stateHandle))) {
		final Integer namespace1 = 1;
		final Integer namespace2 = 2;
		final Integer namespace3 = 3;

		final MapStateDescriptor<Long, Long> stateDescr = new MapStateDescriptor<>("my-map-state", Long.class, Long.class);
		stateDescr.initializeSerializerUnlessSet(new ExecutionConfig());

		InternalMapState<String, Integer, Long, Long> state = keyedBackend.createInternalState(IntSerializer.INSTANCE, stateDescr);

		keyedBackend.setCurrentKey("abc");
		state.setCurrentNamespace(namespace1);
		assertEquals(33L, (long) state.get(33L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(2, getStateSize(state));

		state.setCurrentNamespace(namespace2);
		assertEquals(22L, (long) state.get(22L));
		assertEquals(11L, (long) state.get(11L));
		assertEquals(2, getStateSize(state));

		state.setCurrentNamespace(namespace3);
		assertEquals(44L, (long) state.get(44L));
		assertEquals(1, getStateSize(state));

		keyedBackend.setCurrentKey("def");
		state.setCurrentNamespace(namespace1);
		assertEquals(11L, (long) state.get(11L));
		assertEquals(44L, (long) state.get(44L));
		assertEquals(2, getStateSize(state));

		state.setCurrentNamespace(namespace3);
		assertEquals(22L, (long) state.get(22L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(33L, (long) state.get(33L));
		assertEquals(3, getStateSize(state));

		keyedBackend.setCurrentKey("jkl");
		state.setCurrentNamespace(namespace1);
		assertEquals(11L, (long) state.get(11L));
		assertEquals(22L, (long) state.get(22L));
		assertEquals(33L, (long) state.get(33L));
		assertEquals(44L, (long) state.get(44L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(5, getStateSize(state));

		keyedBackend.setCurrentKey("mno");
		state.setCurrentNamespace(namespace3);
		assertEquals(11L, (long) state.get(11L));
		assertEquals(22L, (long) state.get(22L));
		assertEquals(33L, (long) state.get(33L));
		assertEquals(44L, (long) state.get(44L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(5, getStateSize(state));

		RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = keyedBackend.snapshot(
				1L,
				1L,
				new MemCheckpointStreamFactory(4 * 1024 * 1024),
				CheckpointOptions.forCheckpointWithDefaultLocation());

		snapshot.run();
	}
}

Example #21

Source File: TtlMapState.java From flink with Apache License 2.0

4 votes

TtlMapState(TtlStateContext<InternalMapState<K, N, UK, TtlValue<UV>>, Map<UK, UV>> ttlStateContext) {
	super(ttlStateContext);
}

Example #22

Source File: StateMapView.java From flink with Apache License 2.0

4 votes

public NamespacedStateMapViewWithKeysNullable(InternalMapState<?, N, MK, MV> internalMapState, InternalValueState<?, N, MV> internalNullState) {
	this.internalMapState = internalMapState;
	this.internalNullState = internalNullState;
}

Example #23

Source File: StateMapView.java From flink with Apache License 2.0

4 votes

public NamespacedStateMapViewWithKeysNotNull(InternalMapState<?, N, MK, MV> internalMapState) {
	this.internalMapState = internalMapState;
}

Example #24

Source File: KvStateRequestSerializerTest.java From Flink-CEPplus with Apache License 2.0

4 votes

/**
 * Verifies that the serialization of a map using the given map state
 * matches the deserialization with {@link KvStateSerializer#deserializeList}.
 *
 * @param key
 * 		key of the map state
 * @param mapState
 * 		map state using the {@link VoidNamespace}, must also be a {@link InternalKvState} instance
 *
 * @throws Exception
 */
public static void testMapSerialization(
		final long key,
		final InternalMapState<Long, VoidNamespace, Long, String> mapState) throws Exception {

	TypeSerializer<Long> userKeySerializer = LongSerializer.INSTANCE;
	TypeSerializer<String> userValueSerializer = StringSerializer.INSTANCE;
	mapState.setCurrentNamespace(VoidNamespace.INSTANCE);

	// Map
	final int numElements = 10;

	final Map<Long, String> expectedValues = new HashMap<>();
	for (int i = 1; i <= numElements; i++) {
		final long value = ThreadLocalRandom.current().nextLong();
		expectedValues.put(value, Long.toString(value));
		mapState.put(value, Long.toString(value));
	}

	expectedValues.put(0L, null);
	mapState.put(0L, null);

	final byte[] serializedKey =
		KvStateSerializer.serializeKeyAndNamespace(
			key, LongSerializer.INSTANCE,
			VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE);

	final byte[] serializedValues = mapState.getSerializedValue(
			serializedKey,
			mapState.getKeySerializer(),
			mapState.getNamespaceSerializer(),
			mapState.getValueSerializer());

	Map<Long, String> actualValues = KvStateSerializer.deserializeMap(serializedValues, userKeySerializer, userValueSerializer);
	assertEquals(expectedValues.size(), actualValues.size());
	for (Map.Entry<Long, String> actualEntry : actualValues.entrySet()) {
		assertEquals(expectedValues.get(actualEntry.getKey()), actualEntry.getValue());
	}

	// Single value
	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	long expectedKey = ThreadLocalRandom.current().nextLong();
	String expectedValue = Long.toString(expectedKey);
	byte[] isNull = {0};

	baos.write(KvStateSerializer.serializeValue(expectedKey, userKeySerializer));
	baos.write(isNull);
	baos.write(KvStateSerializer.serializeValue(expectedValue, userValueSerializer));
	byte[] serializedValue = baos.toByteArray();

	Map<Long, String> actualValue = KvStateSerializer.deserializeMap(serializedValue, userKeySerializer, userValueSerializer);
	assertEquals(1, actualValue.size());
	assertEquals(expectedValue, actualValue.get(expectedKey));
}