Java Code Examples for org.apache.flink.util.StateMigrationException

The following examples show how to use org.apache.flink.util.StateMigrationException. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
KeyedBackendSerializationProxy<K> readMetaData(DataInputView dataInputView)
	throws IOException, StateMigrationException {
	// isSerializerPresenceRequired flag is set to false, since for the RocksDB state backend,
	// deserialization of state happens lazily during runtime; we depend on the fact
	// that the new serializer for states could be compatible, and therefore the restore can continue
	// without old serializers required to be present.
	KeyedBackendSerializationProxy<K> serializationProxy =
		new KeyedBackendSerializationProxy<>(userCodeClassLoader);
	serializationProxy.read(dataInputView);
	if (!isKeySerializerCompatibilityChecked) {
		// check for key serializer compatibility; this also reconfigures the
		// key serializer to be compatible, if it is required and is possible
		TypeSerializerSchemaCompatibility<K> keySerializerSchemaCompat =
			keySerializerProvider.setPreviousSerializerSnapshotForRestoredState(serializationProxy.getKeySerializerSnapshot());
		if (keySerializerSchemaCompat.isCompatibleAfterMigration() || keySerializerSchemaCompat.isIncompatible()) {
			throw new StateMigrationException("The new key serializer must be compatible.");
		}

		isKeySerializerCompatibilityChecked = true;
	}

	return serializationProxy;
}
 
Example 2
Source Project: Flink-CEPplus   Source File: RocksDBFullRestoreOperation.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Restores all key-groups data that is referenced by the passed state handles.
 *
 */
@Override
public RocksDBRestoreResult restore()
	throws IOException, StateMigrationException, RocksDBException {
	openDB();
	for (KeyedStateHandle keyedStateHandle : restoreStateHandles) {
		if (keyedStateHandle != null) {

			if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) {
				throw new IllegalStateException("Unexpected state handle type, " +
					"expected: " + KeyGroupsStateHandle.class +
					", but found: " + keyedStateHandle.getClass());
			}
			this.currentKeyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle;
			restoreKeyGroupsInStateHandle();
		}
	}
	return new RocksDBRestoreResult(this.db, defaultColumnFamilyHandle, nativeMetricMonitor,
		-1, null, null);
}
 
Example 3
Source Project: flink   Source File: StateBackendMigrationTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testBroadcastStateRegistrationFailsIfNewValueSerializerIsIncompatible() {
	final String stateName = "broadcast-state";

	try {
		testBroadcastStateValueUpgrade(
			new MapStateDescriptor<>(
				stateName,
				IntSerializer.INSTANCE,
				new TestType.V1TestTypeSerializer()),
			new MapStateDescriptor<>(
				stateName,
				IntSerializer.INSTANCE,
				// new value serializer is incompatible
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed.");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	}
}
 
Example 4
Source Project: Flink-CEPplus   Source File: RocksDBKeyedStateBackend.java    License: Apache License 2.0 6 votes vote down vote up
private <N, S extends State, SV> RegisteredKeyValueStateBackendMetaInfo<N, SV> updateRestoredStateMetaInfo(
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> oldStateInfo,
	StateDescriptor<S, SV> stateDesc,
	TypeSerializer<N> namespaceSerializer,
	TypeSerializer<SV> stateSerializer) throws Exception {

	@SuppressWarnings("unchecked")
	RegisteredKeyValueStateBackendMetaInfo<N, SV> restoredKvStateMetaInfo = oldStateInfo.f1;

	TypeSerializerSchemaCompatibility<N> s = restoredKvStateMetaInfo.updateNamespaceSerializer(namespaceSerializer);
	if (s.isCompatibleAfterMigration() || s.isIncompatible()) {
		throw new StateMigrationException("The new namespace serializer must be compatible.");
	}

	restoredKvStateMetaInfo.checkStateMetaInfo(stateDesc);

	TypeSerializerSchemaCompatibility<SV> newStateSerializerCompatibility =
		restoredKvStateMetaInfo.updateStateSerializer(stateSerializer);
	if (newStateSerializerCompatibility.isCompatibleAfterMigration()) {
		migrateStateValues(stateDesc, oldStateInfo);
	} else if (newStateSerializerCompatibility.isIncompatible()) {
		throw new StateMigrationException("The new state serializer cannot be incompatible.");
	}

	return restoredKvStateMetaInfo;
}
 
Example 5
Source Project: flink   Source File: StateBackendMigrationTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testKeyedListStateRegistrationFailsIfNewStateSerializerIsIncompatible() {
	final String stateName = "test-name";

	try {
		testKeyedListStateUpgrade(
			new ListStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ListStateDescriptor<>(
				stateName,
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed");
	} catch (Exception expected) {
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}
 
Example 6
@Test
public void testKeyedValueStateRegistrationFailsIfNewStateSerializerIsIncompatible() throws Exception {
	final String stateName = "test-name";

	try {
		testKeyedValueStateUpgrade(
			new ValueStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ValueStateDescriptor<>(
				stateName,
				new TestType.IncompatibleTestTypeSerializer()));

		Assert.fail("should have failed");
	} catch (Exception expected) {
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}
 
Example 7
@Test
public void testKeyedListStateRegistrationFailsIfNewStateSerializerIsIncompatible() throws Exception {
	final String stateName = "test-name";

	try {
		testKeyedListStateUpgrade(
			new ListStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ListStateDescriptor<>(
				stateName,
				new TestType.IncompatibleTestTypeSerializer()));

		Assert.fail("should have failed");
	} catch (Exception expected) {
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}
 
Example 8
Source Project: flink   Source File: StateBackendMigrationTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testKeyedValueStateRegistrationFailsIfNewStateSerializerIsIncompatible() {
	final String stateName = "test-name";

	try {
		testKeyedValueStateUpgrade(
			new ValueStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ValueStateDescriptor<>(
				stateName,
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed");
	} catch (Exception expected) {
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}
 
Example 9
@Test
public void testOperatorParitionableListStateRegistrationFailsIfNewSerializerIsIncompatible() throws Exception {
	final String stateName = "partitionable-list-state";

	try {
		testOperatorPartitionableListStateUpgrade(
			new ListStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ListStateDescriptor<>(
				stateName,
				// restore with a new incompatible serializer
				new TestType.IncompatibleTestTypeSerializer()));

		Assert.fail("should have failed.");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	}
}
 
Example 10
@Test
public void testOperatorUnionListStateRegistrationFailsIfNewSerializerIsIncompatible() throws Exception {
	final String stateName = "union-list-state";

	try {
		testOperatorUnionListStateUpgrade(
			new ListStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ListStateDescriptor<>(
				stateName,
				// restore with a new incompatible serializer
				new TestType.IncompatibleTestTypeSerializer()));

		Assert.fail("should have failed.");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	}
}
 
Example 11
@Test
public void testBroadcastStateRegistrationFailsIfNewValueSerializerIsIncompatible() throws Exception {
	final String stateName = "broadcast-state";

	try {
		testBroadcastStateValueUpgrade(
			new MapStateDescriptor<>(
				stateName,
				IntSerializer.INSTANCE,
				new TestType.V1TestTypeSerializer()),
			new MapStateDescriptor<>(
				stateName,
				IntSerializer.INSTANCE,
				// new value serializer is incompatible
				new TestType.IncompatibleTestTypeSerializer()));

		Assert.fail("should have failed.");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	}
}
 
Example 12
Source Project: flink   Source File: StateBackendMigrationTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testOperatorUnionListStateRegistrationFailsIfNewSerializerIsIncompatible() {
	final String stateName = "union-list-state";

	try {
		testOperatorUnionListStateUpgrade(
			new ListStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ListStateDescriptor<>(
				stateName,
				// restore with a new incompatible serializer
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed.");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	}
}
 
Example 13
Source Project: flink   Source File: RocksDBMapState.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void migrateSerializedValue(
	DataInputDeserializer serializedOldValueInput,
	DataOutputSerializer serializedMigratedValueOutput,
	TypeSerializer<Map<UK, UV>> priorSerializer,
	TypeSerializer<Map<UK, UV>> newSerializer) throws StateMigrationException {

	checkArgument(priorSerializer instanceof MapSerializer);
	checkArgument(newSerializer instanceof MapSerializer);

	TypeSerializer<UV> priorMapValueSerializer = ((MapSerializer<UK, UV>) priorSerializer).getValueSerializer();
	TypeSerializer<UV> newMapValueSerializer = ((MapSerializer<UK, UV>) newSerializer).getValueSerializer();

	try {
		boolean isNull = serializedOldValueInput.readBoolean();
		UV mapUserValue = null;
		if (!isNull) {
			mapUserValue = priorMapValueSerializer.deserialize(serializedOldValueInput);
		}
		serializedMigratedValueOutput.writeBoolean(mapUserValue == null);
		newMapValueSerializer.serialize(mapUserValue, serializedMigratedValueOutput);
	} catch (Exception e) {
		throw new StateMigrationException("Error while trying to migrate RocksDB map state.", e);
	}
}
 
Example 14
Source Project: flink   Source File: RocksDBFullRestoreOperation.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Restores all key-groups data that is referenced by the passed state handles.
 *
 */
@Override
public RocksDBRestoreResult restore()
	throws IOException, StateMigrationException, RocksDBException {
	openDB();
	for (KeyedStateHandle keyedStateHandle : restoreStateHandles) {
		if (keyedStateHandle != null) {

			if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) {
				throw new IllegalStateException("Unexpected state handle type, " +
					"expected: " + KeyGroupsStateHandle.class +
					", but found: " + keyedStateHandle.getClass());
			}
			this.currentKeyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle;
			restoreKeyGroupsInStateHandle();
		}
	}
	return new RocksDBRestoreResult(this.db, defaultColumnFamilyHandle, nativeMetricMonitor,
		-1, null, null);
}
 
Example 15
Source Project: flink   Source File: RocksDBFullRestoreOperation.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Restore the KV-state / ColumnFamily meta data for all key-groups referenced by the current state handle.
 */
private void restoreKVStateMetaData() throws IOException, StateMigrationException {
	KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(currentStateHandleInView);

	this.keygroupStreamCompressionDecorator = serializationProxy.isUsingKeyGroupCompression() ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	List<StateMetaInfoSnapshot> restoredMetaInfos =
		serializationProxy.getStateMetaInfoSnapshots();
	currentStateHandleKVStateColumnFamilies = new ArrayList<>(restoredMetaInfos.size());

	for (StateMetaInfoSnapshot restoredMetaInfo : restoredMetaInfos) {
		RocksDbKvStateInfo registeredStateCFHandle =
			getOrRegisterStateColumnFamilyHandle(null, restoredMetaInfo);
		currentStateHandleKVStateColumnFamilies.add(registeredStateCFHandle.columnFamilyHandle);
	}
}
 
Example 16
Source Project: flink   Source File: RocksDBKeyedStateBackend.java    License: Apache License 2.0 6 votes vote down vote up
private <N, S extends State, SV> RegisteredKeyValueStateBackendMetaInfo<N, SV> updateRestoredStateMetaInfo(
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> oldStateInfo,
	StateDescriptor<S, SV> stateDesc,
	TypeSerializer<N> namespaceSerializer,
	TypeSerializer<SV> stateSerializer) throws Exception {

	@SuppressWarnings("unchecked")
	RegisteredKeyValueStateBackendMetaInfo<N, SV> restoredKvStateMetaInfo = oldStateInfo.f1;

	TypeSerializerSchemaCompatibility<N> s = restoredKvStateMetaInfo.updateNamespaceSerializer(namespaceSerializer);
	if (s.isCompatibleAfterMigration() || s.isIncompatible()) {
		throw new StateMigrationException("The new namespace serializer must be compatible.");
	}

	restoredKvStateMetaInfo.checkStateMetaInfo(stateDesc);

	TypeSerializerSchemaCompatibility<SV> newStateSerializerCompatibility =
		restoredKvStateMetaInfo.updateStateSerializer(stateSerializer);
	if (newStateSerializerCompatibility.isCompatibleAfterMigration()) {
		migrateStateValues(stateDesc, oldStateInfo);
	} else if (newStateSerializerCompatibility.isIncompatible()) {
		throw new StateMigrationException("The new state serializer cannot be incompatible.");
	}

	return restoredKvStateMetaInfo;
}
 
Example 17
Source Project: flink   Source File: StateBackendMigrationTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testKeyedValueStateRegistrationFailsIfNewStateSerializerIsIncompatible() {
	final String stateName = "test-name";

	try {
		testKeyedValueStateUpgrade(
			new ValueStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ValueStateDescriptor<>(
				stateName,
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed");
	} catch (Exception expected) {
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}
 
Example 18
Source Project: flink   Source File: StateBackendMigrationTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testKeyedListStateRegistrationFailsIfNewStateSerializerIsIncompatible() {
	final String stateName = "test-name";

	try {
		testKeyedListStateUpgrade(
			new ListStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ListStateDescriptor<>(
				stateName,
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed");
	} catch (Exception expected) {
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}
 
Example 19
Source Project: flink   Source File: StateBackendMigrationTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testKeyedMapStateRegistrationFailsIfNewStateSerializerIsIncompatible() {
	final String stateName = "test-name";

	try {
		testKeyedMapStateUpgrade(
			new MapStateDescriptor<>(
			stateName,
			IntSerializer.INSTANCE,
			new TestType.V1TestTypeSerializer()),
			new MapStateDescriptor<>(
			stateName,
			IntSerializer.INSTANCE,
			// restore with a V2 serializer that has a different schema
			new TestType.IncompatibleTestTypeSerializer()));
		fail("should have failed");
	} catch (Exception expected) {
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}
 
Example 20
Source Project: flink   Source File: StateBackendMigrationTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testOperatorParitionableListStateRegistrationFailsIfNewSerializerIsIncompatible() throws Exception {
	final String stateName = "partitionable-list-state";

	try {
		testOperatorPartitionableListStateUpgrade(
			new ListStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ListStateDescriptor<>(
				stateName,
				// restore with a new incompatible serializer
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed.");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	}
}
 
Example 21
Source Project: flink   Source File: StateBackendMigrationTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testBroadcastStateRegistrationFailsIfNewValueSerializerIsIncompatible() {
	final String stateName = "broadcast-state";

	try {
		testBroadcastStateValueUpgrade(
			new MapStateDescriptor<>(
				stateName,
				IntSerializer.INSTANCE,
				new TestType.V1TestTypeSerializer()),
			new MapStateDescriptor<>(
				stateName,
				IntSerializer.INSTANCE,
				// new value serializer is incompatible
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed.");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	}
}
 
Example 22
Source Project: flink   Source File: StateBackendMigrationTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testBroadcastStateRegistrationFailsIfNewKeySerializerIsIncompatible() {
	final String stateName = "broadcast-state";

	try {
		testBroadcastStateKeyUpgrade(
			new MapStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer(),
				IntSerializer.INSTANCE),
			new MapStateDescriptor<>(
				stateName,
				// new key serializer is incompatible
				new TestType.IncompatibleTestTypeSerializer(),
				IntSerializer.INSTANCE));

		fail("should have failed.");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	}
}
 
Example 23
Source Project: flink   Source File: RocksDBMapState.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void migrateSerializedValue(
	DataInputDeserializer serializedOldValueInput,
	DataOutputSerializer serializedMigratedValueOutput,
	TypeSerializer<Map<UK, UV>> priorSerializer,
	TypeSerializer<Map<UK, UV>> newSerializer) throws StateMigrationException {

	checkArgument(priorSerializer instanceof MapSerializer);
	checkArgument(newSerializer instanceof MapSerializer);

	TypeSerializer<UV> priorMapValueSerializer = ((MapSerializer<UK, UV>) priorSerializer).getValueSerializer();
	TypeSerializer<UV> newMapValueSerializer = ((MapSerializer<UK, UV>) newSerializer).getValueSerializer();

	try {
		boolean isNull = serializedOldValueInput.readBoolean();
		UV mapUserValue = null;
		if (!isNull) {
			mapUserValue = priorMapValueSerializer.deserialize(serializedOldValueInput);
		}
		serializedMigratedValueOutput.writeBoolean(mapUserValue == null);
		newMapValueSerializer.serialize(mapUserValue, serializedMigratedValueOutput);
	} catch (Exception e) {
		throw new StateMigrationException("Error while trying to migrate RocksDB map state.", e);
	}
}
 
Example 24
Source Project: flink   Source File: AbstractRocksDBRestoreOperation.java    License: Apache License 2.0 6 votes vote down vote up
KeyedBackendSerializationProxy<K> readMetaData(DataInputView dataInputView)
	throws IOException, StateMigrationException {
	// isSerializerPresenceRequired flag is set to false, since for the RocksDB state backend,
	// deserialization of state happens lazily during runtime; we depend on the fact
	// that the new serializer for states could be compatible, and therefore the restore can continue
	// without old serializers required to be present.
	KeyedBackendSerializationProxy<K> serializationProxy =
		new KeyedBackendSerializationProxy<>(userCodeClassLoader);
	serializationProxy.read(dataInputView);
	if (!isKeySerializerCompatibilityChecked) {
		// check for key serializer compatibility; this also reconfigures the
		// key serializer to be compatible, if it is required and is possible
		TypeSerializerSchemaCompatibility<K> keySerializerSchemaCompat =
			keySerializerProvider.setPreviousSerializerSnapshotForRestoredState(serializationProxy.getKeySerializerSnapshot());
		if (keySerializerSchemaCompat.isCompatibleAfterMigration() || keySerializerSchemaCompat.isIncompatible()) {
			throw new StateMigrationException("The new key serializer must be compatible.");
		}

		isKeySerializerCompatibilityChecked = true;
	}

	return serializationProxy;
}
 
Example 25
Source Project: flink   Source File: RocksDBFullRestoreOperation.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Restores all key-groups data that is referenced by the passed state handles.
 *
 */
@Override
public RocksDBRestoreResult restore()
	throws IOException, StateMigrationException, RocksDBException {
	openDB();
	for (KeyedStateHandle keyedStateHandle : restoreStateHandles) {
		if (keyedStateHandle != null) {

			if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) {
				throw new IllegalStateException("Unexpected state handle type, " +
					"expected: " + KeyGroupsStateHandle.class +
					", but found: " + keyedStateHandle.getClass());
			}
			this.currentKeyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle;
			restoreKeyGroupsInStateHandle();
		}
	}
	return new RocksDBRestoreResult(this.db, defaultColumnFamilyHandle, nativeMetricMonitor,
		-1, null, null);
}
 
Example 26
Source Project: flink   Source File: RocksDBFullRestoreOperation.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Restore the KV-state / ColumnFamily meta data for all key-groups referenced by the current state handle.
 */
private void restoreKVStateMetaData() throws IOException, StateMigrationException {
	KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(currentStateHandleInView);

	this.keygroupStreamCompressionDecorator = serializationProxy.isUsingKeyGroupCompression() ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	List<StateMetaInfoSnapshot> restoredMetaInfos =
		serializationProxy.getStateMetaInfoSnapshots();
	currentStateHandleKVStateColumnFamilies = new ArrayList<>(restoredMetaInfos.size());

	for (StateMetaInfoSnapshot restoredMetaInfo : restoredMetaInfos) {
		RocksDbKvStateInfo registeredStateCFHandle =
			getOrRegisterStateColumnFamilyHandle(null, restoredMetaInfo);
		currentStateHandleKVStateColumnFamilies.add(registeredStateCFHandle.columnFamilyHandle);
	}
}
 
Example 27
Source Project: flink   Source File: RocksDBKeyedStateBackend.java    License: Apache License 2.0 6 votes vote down vote up
private <N, S extends State, SV> RegisteredKeyValueStateBackendMetaInfo<N, SV> updateRestoredStateMetaInfo(
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> oldStateInfo,
	StateDescriptor<S, SV> stateDesc,
	TypeSerializer<N> namespaceSerializer,
	TypeSerializer<SV> stateSerializer) throws Exception {

	@SuppressWarnings("unchecked")
	RegisteredKeyValueStateBackendMetaInfo<N, SV> restoredKvStateMetaInfo = oldStateInfo.f1;

	TypeSerializerSchemaCompatibility<N> s = restoredKvStateMetaInfo.updateNamespaceSerializer(namespaceSerializer);
	if (s.isCompatibleAfterMigration() || s.isIncompatible()) {
		throw new StateMigrationException("The new namespace serializer must be compatible.");
	}

	restoredKvStateMetaInfo.checkStateMetaInfo(stateDesc);

	TypeSerializerSchemaCompatibility<SV> newStateSerializerCompatibility =
		restoredKvStateMetaInfo.updateStateSerializer(stateSerializer);
	if (newStateSerializerCompatibility.isCompatibleAfterMigration()) {
		migrateStateValues(stateDesc, oldStateInfo);
	} else if (newStateSerializerCompatibility.isIncompatible()) {
		throw new StateMigrationException("The new state serializer cannot be incompatible.");
	}

	return restoredKvStateMetaInfo;
}
 
Example 28
Source Project: Flink-CEPplus   Source File: PojoSerializerUpgradeTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Changing field types of a POJO as keyed state should require a state migration.
 */
@Test
public void testChangedFieldTypesWithKeyedState() throws Exception {
	try {
		testPojoSerializerUpgrade(SOURCE_A, SOURCE_C, true, true);
		fail("Expected a state migration exception.");
	} catch (Exception e) {
		if (CommonTestUtils.containsCause(e, StateMigrationException.class)) {
			// StateMigrationException expected
		} else {
			throw e;
		}
	}
}
 
Example 29
Source Project: Flink-CEPplus   Source File: RocksDBListState.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void migrateSerializedValue(
		DataInputDeserializer serializedOldValueInput,
		DataOutputSerializer serializedMigratedValueOutput,
		TypeSerializer<List<V>> priorSerializer,
		TypeSerializer<List<V>> newSerializer) throws StateMigrationException {

	Preconditions.checkArgument(priorSerializer instanceof ListSerializer);
	Preconditions.checkArgument(newSerializer instanceof ListSerializer);

	TypeSerializer<V> priorElementSerializer =
		((ListSerializer<V>) priorSerializer).getElementSerializer();

	TypeSerializer<V> newElementSerializer =
		((ListSerializer<V>) newSerializer).getElementSerializer();

	try {
		while (serializedOldValueInput.available() > 0) {
			V element = deserializeNextElement(serializedOldValueInput, priorElementSerializer);
			newElementSerializer.serialize(element, serializedMigratedValueOutput);
			if (serializedOldValueInput.available() > 0) {
				serializedMigratedValueOutput.write(DELIMITER);
			}
		}
	} catch (Exception e) {
		throw new StateMigrationException("Error while trying to migrate RocksDB list state.", e);
	}
}
 
Example 30
Source Project: Flink-CEPplus   Source File: RocksDBFullRestoreOperation.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Restore one key groups state handle.
 */
private void restoreKeyGroupsInStateHandle()
	throws IOException, StateMigrationException, RocksDBException {
	try {
		currentStateHandleInStream = currentKeyGroupsStateHandle.openInputStream();
		cancelStreamRegistry.registerCloseable(currentStateHandleInStream);
		currentStateHandleInView = new DataInputViewStreamWrapper(currentStateHandleInStream);
		restoreKVStateMetaData();
		restoreKVStateData();
	} finally {
		if (cancelStreamRegistry.unregisterCloseable(currentStateHandleInStream)) {
			IOUtils.closeQuietly(currentStateHandleInStream);
		}
	}
}