org.apache.flink.queryablestate.client.state.serialization.KvStateSerializer Java Examples

@Before
public void setUp() throws Exception {
	if (!mapStateDesc.isSerializerInitialized()) {
		mapStateDesc.initializeSerializerUnlessSet(new ExecutionConfig());
	}

	Map<Long, Long> initMap = new HashMap<>();
	initMap.put(1L, 5L);
	initMap.put(2L, 5L);

	byte[] initSer = KvStateSerializer.serializeMap(
			initMap.entrySet(),
			BasicTypeInfo.LONG_TYPE_INFO.createSerializer(new ExecutionConfig()),
			BasicTypeInfo.LONG_TYPE_INFO.createSerializer(new ExecutionConfig()));

	mapState = ImmutableMapState.createState(mapStateDesc, initSer);
}
 

/**
 * Tests key and namespace serialization utils.
 */
@Test
public void testKeyAndNamespaceSerialization() throws Exception {
	TypeSerializer<Long> keySerializer = LongSerializer.INSTANCE;
	TypeSerializer<String> namespaceSerializer = StringSerializer.INSTANCE;

	long expectedKey = Integer.MAX_VALUE + 12323L;
	String expectedNamespace = "knilf";

	byte[] serializedKeyAndNamespace = KvStateSerializer.serializeKeyAndNamespace(
			expectedKey, keySerializer, expectedNamespace, namespaceSerializer);

	Tuple2<Long, String> actual = KvStateSerializer.deserializeKeyAndNamespace(
			serializedKeyAndNamespace, keySerializer, namespaceSerializer);

	assertEquals(expectedKey, actual.f0.longValue());
	assertEquals(expectedNamespace, actual.f1);
}
 

/**
 * Returns the value by getting the serialized value and deserializing it
 * if it is not null.
 */
private static <V, K, N> List<V> getSerializedList(
		InternalKvState<K, N, V> kvState,
		K key,
		TypeSerializer<K> keySerializer,
		N namespace,
		TypeSerializer<N> namespaceSerializer,
		TypeSerializer<V> valueSerializer) throws Exception {

	byte[] serializedKeyAndNamespace = KvStateSerializer.serializeKeyAndNamespace(
			key, keySerializer, namespace, namespaceSerializer);

	byte[] serializedValue = kvState.getSerializedValue(
			serializedKeyAndNamespace,
			kvState.getKeySerializer(),
			kvState.getNamespaceSerializer(),
			kvState.getValueSerializer()
	);

	if (serializedValue == null) {
		return null;
	} else {
		return KvStateSerializer.deserializeList(serializedValue, valueSerializer);
	}
}
 

/**
 * Returns the value by getting the serialized value and deserializing it
 * if it is not null.
 */
protected static <V, K, N> V getSerializedValue(
		InternalKvState<K, N, V> kvState,
		K key,
		TypeSerializer<K> keySerializer,
		N namespace,
		TypeSerializer<N> namespaceSerializer,
		TypeSerializer<V> valueSerializer) throws Exception {

	byte[] serializedKeyAndNamespace = KvStateSerializer.serializeKeyAndNamespace(
			key, keySerializer, namespace, namespaceSerializer);

	byte[] serializedValue = kvState.getSerializedValue(
			serializedKeyAndNamespace,
			kvState.getKeySerializer(),
			kvState.getNamespaceSerializer(),
			kvState.getValueSerializer()
	);

	if (serializedValue == null) {
		return null;
	} else {
		return KvStateSerializer.deserializeValue(serializedValue, valueSerializer);
	}
}
 

@Override
public byte[] getSerializedValue(
		final byte[] serializedKeyAndNamespace,
		final TypeSerializer<K> safeKeySerializer,
		final TypeSerializer<N> safeNamespaceSerializer,
		final TypeSerializer<V> safeValueSerializer) throws Exception {

	//TODO make KvStateSerializer key-group aware to save this round trip and key-group computation
	Tuple2<K, N> keyAndNamespace = KvStateSerializer.deserializeKeyAndNamespace(
			serializedKeyAndNamespace, safeKeySerializer, safeNamespaceSerializer);

	int keyGroup = KeyGroupRangeAssignment.assignToKeyGroup(keyAndNamespace.f0, backend.getNumberOfKeyGroups());

	RocksDBSerializedCompositeKeyBuilder<K> keyBuilder =
					new RocksDBSerializedCompositeKeyBuilder<>(
						safeKeySerializer,
						backend.getKeyGroupPrefixBytes(),
						32
					);
	keyBuilder.setKeyAndKeyGroup(keyAndNamespace.f0, keyGroup);
	byte[] key = keyBuilder.buildCompositeKeyNamespace(keyAndNamespace.f1, namespaceSerializer);
	return backend.db.get(columnFamily, key);
}
 

@Override
public byte[] getSerializedValue(
		final byte[] serializedKeyAndNamespace,
		final TypeSerializer<K> safeKeySerializer,
		final TypeSerializer<N> safeNamespaceSerializer,
		final TypeSerializer<SV> safeValueSerializer) throws Exception {

	Preconditions.checkNotNull(serializedKeyAndNamespace);
	Preconditions.checkNotNull(safeKeySerializer);
	Preconditions.checkNotNull(safeNamespaceSerializer);
	Preconditions.checkNotNull(safeValueSerializer);

	Tuple2<K, N> keyAndNamespace = KvStateSerializer.deserializeKeyAndNamespace(
			serializedKeyAndNamespace, safeKeySerializer, safeNamespaceSerializer);

	SV result = stateTable.get(keyAndNamespace.f0, keyAndNamespace.f1);

	if (result == null) {
		return null;
	}
	return KvStateSerializer.serializeValue(result, safeValueSerializer);
}
 

/**
 * Tests key and namespace serialization utils.
 */
@Test
public void testKeyAndNamespaceSerialization() throws Exception {
	TypeSerializer<Long> keySerializer = LongSerializer.INSTANCE;
	TypeSerializer<String> namespaceSerializer = StringSerializer.INSTANCE;

	long expectedKey = Integer.MAX_VALUE + 12323L;
	String expectedNamespace = "knilf";

	byte[] serializedKeyAndNamespace = KvStateSerializer.serializeKeyAndNamespace(
			expectedKey, keySerializer, expectedNamespace, namespaceSerializer);

	Tuple2<Long, String> actual = KvStateSerializer.deserializeKeyAndNamespace(
			serializedKeyAndNamespace, keySerializer, namespaceSerializer);

	assertEquals(expectedKey, actual.f0.longValue());
	assertEquals(expectedNamespace, actual.f1);
}
 

/**
 * Returns the value by getting the serialized value and deserializing it
 * if it is not null.
 */
private static <V, K, N> List<V> getSerializedList(
		InternalKvState<K, N, V> kvState,
		K key,
		TypeSerializer<K> keySerializer,
		N namespace,
		TypeSerializer<N> namespaceSerializer,
		TypeSerializer<V> valueSerializer) throws Exception {

	byte[] serializedKeyAndNamespace = KvStateSerializer.serializeKeyAndNamespace(
			key, keySerializer, namespace, namespaceSerializer);

	byte[] serializedValue = kvState.getSerializedValue(
			serializedKeyAndNamespace,
			kvState.getKeySerializer(),
			kvState.getNamespaceSerializer(),
			kvState.getValueSerializer()
	);

	if (serializedValue == null) {
		return null;
	} else {
		return KvStateSerializer.deserializeList(serializedValue, valueSerializer);
	}
}
 

/**
 * Tests key and namespace serialization utils.
 */
@Test
public void testKeyAndNamespaceSerialization() throws Exception {
	TypeSerializer<Long> keySerializer = LongSerializer.INSTANCE;
	TypeSerializer<String> namespaceSerializer = StringSerializer.INSTANCE;

	long expectedKey = Integer.MAX_VALUE + 12323L;
	String expectedNamespace = "knilf";

	byte[] serializedKeyAndNamespace = KvStateSerializer.serializeKeyAndNamespace(
			expectedKey, keySerializer, expectedNamespace, namespaceSerializer);

	Tuple2<Long, String> actual = KvStateSerializer.deserializeKeyAndNamespace(
			serializedKeyAndNamespace, keySerializer, namespaceSerializer);

	assertEquals(expectedKey, actual.f0.longValue());
	assertEquals(expectedNamespace, actual.f1);
}
 

/**
 * Returns the value by getting the serialized value and deserializing it
 * if it is not null.
 */
protected static <V, K, N> V getSerializedValue(
		InternalKvState<K, N, V> kvState,
		K key,
		TypeSerializer<K> keySerializer,
		N namespace,
		TypeSerializer<N> namespaceSerializer,
		TypeSerializer<V> valueSerializer) throws Exception {

	byte[] serializedKeyAndNamespace = KvStateSerializer.serializeKeyAndNamespace(
			key, keySerializer, namespace, namespaceSerializer);

	byte[] serializedValue = kvState.getSerializedValue(
			serializedKeyAndNamespace,
			kvState.getKeySerializer(),
			kvState.getNamespaceSerializer(),
			kvState.getValueSerializer()
	);

	if (serializedValue == null) {
		return null;
	} else {
		return KvStateSerializer.deserializeValue(serializedValue, valueSerializer);
	}
}
 

/**
 * Returns the value by getting the serialized value and deserializing it
 * if it is not null.
 */
private static <V, K, N> List<V> getSerializedList(
		InternalKvState<K, N, V> kvState,
		K key,
		TypeSerializer<K> keySerializer,
		N namespace,
		TypeSerializer<N> namespaceSerializer,
		TypeSerializer<V> valueSerializer) throws Exception {

	byte[] serializedKeyAndNamespace = KvStateSerializer.serializeKeyAndNamespace(
			key, keySerializer, namespace, namespaceSerializer);

	byte[] serializedValue = kvState.getSerializedValue(
			serializedKeyAndNamespace,
			kvState.getKeySerializer(),
			kvState.getNamespaceSerializer(),
			kvState.getValueSerializer()
	);

	if (serializedValue == null) {
		return null;
	} else {
		return KvStateSerializer.deserializeList(serializedValue, valueSerializer);
	}
}
 

/**
 * Returns the value by getting the serialized value and deserializing it
 * if it is not null.
 */
protected static <V, K, N> V getSerializedValue(
		InternalKvState<K, N, V> kvState,
		K key,
		TypeSerializer<K> keySerializer,
		N namespace,
		TypeSerializer<N> namespaceSerializer,
		TypeSerializer<V> valueSerializer) throws Exception {

	byte[] serializedKeyAndNamespace = KvStateSerializer.serializeKeyAndNamespace(
			key, keySerializer, namespace, namespaceSerializer);

	byte[] serializedValue = kvState.getSerializedValue(
			serializedKeyAndNamespace,
			kvState.getKeySerializer(),
			kvState.getNamespaceSerializer(),
			kvState.getValueSerializer()
	);

	if (serializedValue == null) {
		return null;
	} else {
		return KvStateSerializer.deserializeValue(serializedValue, valueSerializer);
	}
}
 

@Override
public byte[] getSerializedValue(
		final byte[] serializedKeyAndNamespace,
		final TypeSerializer<K> safeKeySerializer,
		final TypeSerializer<N> safeNamespaceSerializer,
		final TypeSerializer<SV> safeValueSerializer) throws Exception {

	Preconditions.checkNotNull(serializedKeyAndNamespace);
	Preconditions.checkNotNull(safeKeySerializer);
	Preconditions.checkNotNull(safeNamespaceSerializer);
	Preconditions.checkNotNull(safeValueSerializer);

	Tuple2<K, N> keyAndNamespace = KvStateSerializer.deserializeKeyAndNamespace(
			serializedKeyAndNamespace, safeKeySerializer, safeNamespaceSerializer);

	SV result = stateTable.get(keyAndNamespace.f0, keyAndNamespace.f1);

	if (result == null) {
		return null;
	}
	return KvStateSerializer.serializeValue(result, safeValueSerializer);
}
 

/**
 * Tests map deserialization with too few bytes.
 */
@Test(expected = IOException.class)
public void testDeserializeMapTooShort3() throws Exception {
	// Long (Key1) + Boolean (false) + Long (Value1) + 1 byte (incomplete Key2)
	KvStateSerializer.deserializeMap(new byte[] {1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 3},
		LongSerializer.INSTANCE, LongSerializer.INSTANCE);
}
 

@SuppressWarnings("unchecked")
public static <V, S extends State> S createState(
	StateDescriptor<S, V> stateDescriptor,
	byte[] serializedState) throws IOException {
	final V state = KvStateSerializer.deserializeValue(
		serializedState,
		stateDescriptor.getSerializer());
	return (S) new ImmutableValueState<>(state);
}
 

@SuppressWarnings("unchecked")
public static <V, S extends State> S createState(
	StateDescriptor<S, V> stateDescriptor,
	byte[] serializedState) throws IOException {
	final V state = KvStateSerializer.deserializeValue(
		serializedState,
		stateDescriptor.getSerializer());
	return (S) new ImmutableReducingState<>(state);
}
 

@SuppressWarnings("unchecked")
public static <V, S extends State> S createState(
	StateDescriptor<S, V> stateDescriptor,
	byte[] serializedState) throws IOException {
	final V state = KvStateSerializer.deserializeValue(
		serializedState,
		stateDescriptor.getSerializer());
	return (S) new ImmutableValueState<>(state);
}
 

@SuppressWarnings("unchecked")
public static <ACC, S extends State> S createState(
	StateDescriptor<S, ACC> stateDescriptor,
	byte[] serializedState) throws IOException {
	final ACC state = KvStateSerializer.deserializeValue(
		serializedState,
		stateDescriptor.getSerializer());
	return (S) new ImmutableFoldingState<>(state);
}
 

@SuppressWarnings("unchecked")
public static <OUT, ACC, S extends State> S createState(
	StateDescriptor<S, ACC> stateDescriptor,
	byte[] serializedState) throws IOException {
	final ACC accumulator = KvStateSerializer.deserializeValue(
		serializedState,
		stateDescriptor.getSerializer());
	final OUT state = ((AggregatingStateDescriptor<?, ACC, OUT>) stateDescriptor).
		getAggregateFunction().getResult(accumulator);
	return (S) new ImmutableAggregatingState<>(state);
}
 

/**
 * Tests key and namespace deserialization utils with too many bytes.
 */
@Test(expected = IOException.class)
public void testKeyAndNamespaceDeserializationTooMany1() throws Exception {
	// Long + null String + 1 byte
	KvStateSerializer.deserializeKeyAndNamespace(
		new byte[] {1, 1, 1, 1, 1, 1, 1, 1, 42, 0, 2}, LongSerializer.INSTANCE,
		StringSerializer.INSTANCE);
}
 

@SuppressWarnings("unchecked")
public static <V, S extends State> S createState(
	StateDescriptor<S, V> stateDescriptor,
	byte[] serializedState) throws IOException {
	final V state = KvStateSerializer.deserializeValue(
		serializedState,
		stateDescriptor.getSerializer());
	return (S) new ImmutableReducingState<>(state);
}
 

/**
 * Tests key and namespace deserialization utils with too many bytes.
 */
@Test(expected = IOException.class)
public void testKeyAndNamespaceDeserializationTooMany2() throws Exception {
	// Long + null String + 2 bytes
	KvStateSerializer.deserializeKeyAndNamespace(
		new byte[] {1, 1, 1, 1, 1, 1, 1, 1, 42, 0, 2, 2}, LongSerializer.INSTANCE,
		StringSerializer.INSTANCE);
}
 

/**
 * Tests key and namespace deserialization utils with too many bytes.
 */
@Test(expected = IOException.class)
public void testKeyAndNamespaceDeserializationTooMany1() throws Exception {
	// Long + null String + 1 byte
	KvStateSerializer.deserializeKeyAndNamespace(
		new byte[] {1, 1, 1, 1, 1, 1, 1, 1, 42, 0, 2}, LongSerializer.INSTANCE,
		StringSerializer.INSTANCE);
}
 

@SuppressWarnings("unchecked")
public static <V, T, S extends State> S createState(
	StateDescriptor<S, T> stateDescriptor,
	byte[] serializedState) throws IOException {
	final List<V> state = KvStateSerializer.deserializeList(
		serializedState,
		((ListStateDescriptor<V>) stateDescriptor).getElementSerializer());
	return (S) new ImmutableListState<>(state);
}
 

/**
 * Returns the value by getting the serialized value and deserializing it
 * if it is not null.
 */
private static <UK, UV, K, N> Map<UK, UV> getSerializedMap(
		InternalKvState<K, N, Map<UK, UV>> kvState,
		K key,
		TypeSerializer<K> keySerializer,
		N namespace,
		TypeSerializer<N> namespaceSerializer,
		TypeSerializer<UK> userKeySerializer,
		TypeSerializer<UV> userValueSerializer
) throws Exception {

	byte[] serializedKeyAndNamespace = KvStateSerializer.serializeKeyAndNamespace(
			key, keySerializer, namespace, namespaceSerializer);

	byte[] serializedValue = kvState.getSerializedValue(
			serializedKeyAndNamespace,
			kvState.getKeySerializer(),
			kvState.getNamespaceSerializer(),
			kvState.getValueSerializer()
	);

	if (serializedValue == null) {
		return null;
	} else {
		return KvStateSerializer.deserializeMap(serializedValue, userKeySerializer, userValueSerializer);
	}
}
 

/**
 * Tests list deserialization with too few bytes.
 */
@Test
public void testDeserializeListEmpty() throws Exception {
	List<Long> actualValue = KvStateSerializer
		.deserializeList(new byte[] {}, LongSerializer.INSTANCE);
	assertEquals(0, actualValue.size());
}
 

/**
 * Tests value deserialization with too many bytes.
 */
@Test(expected = IOException.class)
public void testDeserializeValueTooMany2() throws Exception {
	// Long + 2 bytes
	KvStateSerializer.deserializeValue(new byte[] {1, 1, 1, 1, 1, 1, 1, 1, 2, 2},
		LongSerializer.INSTANCE);
}
 

@Override
public byte[] getSerializedValue(
		final byte[] serializedKeyAndNamespace,
		final TypeSerializer<K> safeKeySerializer,
		final TypeSerializer<N> safeNamespaceSerializer,
		final TypeSerializer<List<V>> safeValueSerializer) throws Exception {

	Preconditions.checkNotNull(serializedKeyAndNamespace);
	Preconditions.checkNotNull(safeKeySerializer);
	Preconditions.checkNotNull(safeNamespaceSerializer);
	Preconditions.checkNotNull(safeValueSerializer);

	Tuple2<K, N> keyAndNamespace = KvStateSerializer.deserializeKeyAndNamespace(
			serializedKeyAndNamespace, safeKeySerializer, safeNamespaceSerializer);

	List<V> result = stateTable.get(keyAndNamespace.f0, keyAndNamespace.f1);

	if (result == null) {
		return null;
	}

	final TypeSerializer<V> dupSerializer = ((ListSerializer<V>) safeValueSerializer).getElementSerializer();

	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	DataOutputViewStreamWrapper view = new DataOutputViewStreamWrapper(baos);

	// write the same as RocksDB writes lists, with one ',' separator
	for (int i = 0; i < result.size(); i++) {
		dupSerializer.serialize(result.get(i), view);
		if (i < result.size() -1) {
			view.writeByte(',');
		}
	}
	view.flush();

	return baos.toByteArray();
}
 

@Override
public byte[] getSerializedValue(
		final byte[] serializedKeyAndNamespace,
		final TypeSerializer<K> safeKeySerializer,
		final TypeSerializer<N> safeNamespaceSerializer,
		final TypeSerializer<Map<UK, UV>> safeValueSerializer) throws Exception {

	Preconditions.checkNotNull(serializedKeyAndNamespace);
	Preconditions.checkNotNull(safeKeySerializer);
	Preconditions.checkNotNull(safeNamespaceSerializer);
	Preconditions.checkNotNull(safeValueSerializer);

	Tuple2<K, N> keyAndNamespace = KvStateSerializer.deserializeKeyAndNamespace(
			serializedKeyAndNamespace, safeKeySerializer, safeNamespaceSerializer);

	Map<UK, UV> result = stateTable.get(keyAndNamespace.f0, keyAndNamespace.f1);

	if (result == null) {
		return null;
	}

	final MapSerializer<UK, UV> serializer = (MapSerializer<UK, UV>) safeValueSerializer;

	final TypeSerializer<UK> dupUserKeySerializer = serializer.getKeySerializer();
	final TypeSerializer<UV> dupUserValueSerializer = serializer.getValueSerializer();

	return KvStateSerializer.serializeMap(result.entrySet(), dupUserKeySerializer, dupUserValueSerializer);
}
 

/**
 * Tests value deserialization with too many bytes.
 */
@Test(expected = IOException.class)
public void testDeserializeValueTooMany1() throws Exception {
	// Long + 1 byte
	KvStateSerializer.deserializeValue(new byte[] {1, 1, 1, 1, 1, 1, 1, 1, 2},
		LongSerializer.INSTANCE);
}