org.apache.flink.util.StateMigrationException Java Examples

The following examples show how to use org.apache.flink.util.StateMigrationException. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RocksDBFullRestoreOperation.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Restores all key-groups data that is referenced by the passed state handles.
 *
 */
@Override
public RocksDBRestoreResult restore()
	throws IOException, StateMigrationException, RocksDBException {
	openDB();
	for (KeyedStateHandle keyedStateHandle : restoreStateHandles) {
		if (keyedStateHandle != null) {

			if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) {
				throw new IllegalStateException("Unexpected state handle type, " +
					"expected: " + KeyGroupsStateHandle.class +
					", but found: " + keyedStateHandle.getClass());
			}
			this.currentKeyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle;
			restoreKeyGroupsInStateHandle();
		}
	}
	return new RocksDBRestoreResult(this.db, defaultColumnFamilyHandle, nativeMetricMonitor,
		-1, null, null);
}
 
Example #2
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testKeyedValueStateRegistrationFailsIfNewStateSerializerIsIncompatible() {
	final String stateName = "test-name";

	try {
		testKeyedValueStateUpgrade(
			new ValueStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ValueStateDescriptor<>(
				stateName,
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed");
	} catch (Exception expected) {
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}
 
Example #3
Source File: RocksDBKeyedStateBackend.java    From flink with Apache License 2.0 6 votes vote down vote up
private <N, S extends State, SV> RegisteredKeyValueStateBackendMetaInfo<N, SV> updateRestoredStateMetaInfo(
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> oldStateInfo,
	StateDescriptor<S, SV> stateDesc,
	TypeSerializer<N> namespaceSerializer,
	TypeSerializer<SV> stateSerializer) throws Exception {

	@SuppressWarnings("unchecked")
	RegisteredKeyValueStateBackendMetaInfo<N, SV> restoredKvStateMetaInfo = oldStateInfo.f1;

	TypeSerializerSchemaCompatibility<N> s = restoredKvStateMetaInfo.updateNamespaceSerializer(namespaceSerializer);
	if (s.isCompatibleAfterMigration() || s.isIncompatible()) {
		throw new StateMigrationException("The new namespace serializer must be compatible.");
	}

	restoredKvStateMetaInfo.checkStateMetaInfo(stateDesc);

	TypeSerializerSchemaCompatibility<SV> newStateSerializerCompatibility =
		restoredKvStateMetaInfo.updateStateSerializer(stateSerializer);
	if (newStateSerializerCompatibility.isCompatibleAfterMigration()) {
		migrateStateValues(stateDesc, oldStateInfo);
	} else if (newStateSerializerCompatibility.isIncompatible()) {
		throw new StateMigrationException("The new state serializer cannot be incompatible.");
	}

	return restoredKvStateMetaInfo;
}
 
Example #4
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testOperatorParitionableListStateRegistrationFailsIfNewSerializerIsIncompatible() throws Exception {
	final String stateName = "partitionable-list-state";

	try {
		testOperatorPartitionableListStateUpgrade(
			new ListStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ListStateDescriptor<>(
				stateName,
				// restore with a new incompatible serializer
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed.");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	}
}
 
Example #5
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testKeyedListStateRegistrationFailsIfNewStateSerializerIsIncompatible() {
	final String stateName = "test-name";

	try {
		testKeyedListStateUpgrade(
			new ListStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ListStateDescriptor<>(
				stateName,
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed");
	} catch (Exception expected) {
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}
 
Example #6
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testKeyedListStateRegistrationFailsIfNewStateSerializerIsIncompatible() {
	final String stateName = "test-name";

	try {
		testKeyedListStateUpgrade(
			new ListStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ListStateDescriptor<>(
				stateName,
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed");
	} catch (Exception expected) {
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}
 
Example #7
Source File: RocksDBKeyedStateBackend.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private <N, S extends State, SV> RegisteredKeyValueStateBackendMetaInfo<N, SV> updateRestoredStateMetaInfo(
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> oldStateInfo,
	StateDescriptor<S, SV> stateDesc,
	TypeSerializer<N> namespaceSerializer,
	TypeSerializer<SV> stateSerializer) throws Exception {

	@SuppressWarnings("unchecked")
	RegisteredKeyValueStateBackendMetaInfo<N, SV> restoredKvStateMetaInfo = oldStateInfo.f1;

	TypeSerializerSchemaCompatibility<N> s = restoredKvStateMetaInfo.updateNamespaceSerializer(namespaceSerializer);
	if (s.isCompatibleAfterMigration() || s.isIncompatible()) {
		throw new StateMigrationException("The new namespace serializer must be compatible.");
	}

	restoredKvStateMetaInfo.checkStateMetaInfo(stateDesc);

	TypeSerializerSchemaCompatibility<SV> newStateSerializerCompatibility =
		restoredKvStateMetaInfo.updateStateSerializer(stateSerializer);
	if (newStateSerializerCompatibility.isCompatibleAfterMigration()) {
		migrateStateValues(stateDesc, oldStateInfo);
	} else if (newStateSerializerCompatibility.isIncompatible()) {
		throw new StateMigrationException("The new state serializer cannot be incompatible.");
	}

	return restoredKvStateMetaInfo;
}
 
Example #8
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testBroadcastStateRegistrationFailsIfNewValueSerializerIsIncompatible() {
	final String stateName = "broadcast-state";

	try {
		testBroadcastStateValueUpgrade(
			new MapStateDescriptor<>(
				stateName,
				IntSerializer.INSTANCE,
				new TestType.V1TestTypeSerializer()),
			new MapStateDescriptor<>(
				stateName,
				IntSerializer.INSTANCE,
				// new value serializer is incompatible
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed.");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	}
}
 
Example #9
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testKeyedMapStateRegistrationFailsIfNewStateSerializerIsIncompatible() {
	final String stateName = "test-name";

	try {
		testKeyedMapStateUpgrade(
			new MapStateDescriptor<>(
			stateName,
			IntSerializer.INSTANCE,
			new TestType.V1TestTypeSerializer()),
			new MapStateDescriptor<>(
			stateName,
			IntSerializer.INSTANCE,
			// restore with a V2 serializer that has a different schema
			new TestType.IncompatibleTestTypeSerializer()));
		fail("should have failed");
	} catch (Exception expected) {
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}
 
Example #10
Source File: RocksDBFullRestoreOperation.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Restores all key-groups data that is referenced by the passed state handles.
 *
 */
@Override
public RocksDBRestoreResult restore()
	throws IOException, StateMigrationException, RocksDBException {
	openDB();
	for (KeyedStateHandle keyedStateHandle : restoreStateHandles) {
		if (keyedStateHandle != null) {

			if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) {
				throw new IllegalStateException("Unexpected state handle type, " +
					"expected: " + KeyGroupsStateHandle.class +
					", but found: " + keyedStateHandle.getClass());
			}
			this.currentKeyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle;
			restoreKeyGroupsInStateHandle();
		}
	}
	return new RocksDBRestoreResult(this.db, defaultColumnFamilyHandle, nativeMetricMonitor,
		-1, null, null);
}
 
Example #11
Source File: AbstractRocksDBRestoreOperation.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
KeyedBackendSerializationProxy<K> readMetaData(DataInputView dataInputView)
	throws IOException, StateMigrationException {
	// isSerializerPresenceRequired flag is set to false, since for the RocksDB state backend,
	// deserialization of state happens lazily during runtime; we depend on the fact
	// that the new serializer for states could be compatible, and therefore the restore can continue
	// without old serializers required to be present.
	KeyedBackendSerializationProxy<K> serializationProxy =
		new KeyedBackendSerializationProxy<>(userCodeClassLoader);
	serializationProxy.read(dataInputView);
	if (!isKeySerializerCompatibilityChecked) {
		// check for key serializer compatibility; this also reconfigures the
		// key serializer to be compatible, if it is required and is possible
		TypeSerializerSchemaCompatibility<K> keySerializerSchemaCompat =
			keySerializerProvider.setPreviousSerializerSnapshotForRestoredState(serializationProxy.getKeySerializerSnapshot());
		if (keySerializerSchemaCompat.isCompatibleAfterMigration() || keySerializerSchemaCompat.isIncompatible()) {
			throw new StateMigrationException("The new key serializer must be compatible.");
		}

		isKeySerializerCompatibilityChecked = true;
	}

	return serializationProxy;
}
 
Example #12
Source File: RocksDBFullRestoreOperation.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Restore the KV-state / ColumnFamily meta data for all key-groups referenced by the current state handle.
 */
private void restoreKVStateMetaData() throws IOException, StateMigrationException {
	KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(currentStateHandleInView);

	this.keygroupStreamCompressionDecorator = serializationProxy.isUsingKeyGroupCompression() ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	List<StateMetaInfoSnapshot> restoredMetaInfos =
		serializationProxy.getStateMetaInfoSnapshots();
	currentStateHandleKVStateColumnFamilies = new ArrayList<>(restoredMetaInfos.size());

	for (StateMetaInfoSnapshot restoredMetaInfo : restoredMetaInfos) {
		RocksDbKvStateInfo registeredStateCFHandle =
			getOrRegisterStateColumnFamilyHandle(null, restoredMetaInfo);
		currentStateHandleKVStateColumnFamilies.add(registeredStateCFHandle.columnFamilyHandle);
	}
}
 
Example #13
Source File: RocksDBFullRestoreOperation.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Restores all key-groups data that is referenced by the passed state handles.
 *
 */
@Override
public RocksDBRestoreResult restore()
	throws IOException, StateMigrationException, RocksDBException {
	openDB();
	for (KeyedStateHandle keyedStateHandle : restoreStateHandles) {
		if (keyedStateHandle != null) {

			if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) {
				throw new IllegalStateException("Unexpected state handle type, " +
					"expected: " + KeyGroupsStateHandle.class +
					", but found: " + keyedStateHandle.getClass());
			}
			this.currentKeyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle;
			restoreKeyGroupsInStateHandle();
		}
	}
	return new RocksDBRestoreResult(this.db, defaultColumnFamilyHandle, nativeMetricMonitor,
		-1, null, null);
}
 
Example #14
Source File: AbstractRocksDBRestoreOperation.java    From flink with Apache License 2.0 6 votes vote down vote up
KeyedBackendSerializationProxy<K> readMetaData(DataInputView dataInputView)
	throws IOException, StateMigrationException {
	// isSerializerPresenceRequired flag is set to false, since for the RocksDB state backend,
	// deserialization of state happens lazily during runtime; we depend on the fact
	// that the new serializer for states could be compatible, and therefore the restore can continue
	// without old serializers required to be present.
	KeyedBackendSerializationProxy<K> serializationProxy =
		new KeyedBackendSerializationProxy<>(userCodeClassLoader);
	serializationProxy.read(dataInputView);
	if (!isKeySerializerCompatibilityChecked) {
		// check for key serializer compatibility; this also reconfigures the
		// key serializer to be compatible, if it is required and is possible
		TypeSerializerSchemaCompatibility<K> keySerializerSchemaCompat =
			keySerializerProvider.setPreviousSerializerSnapshotForRestoredState(serializationProxy.getKeySerializerSnapshot());
		if (keySerializerSchemaCompat.isCompatibleAfterMigration() || keySerializerSchemaCompat.isIncompatible()) {
			throw new StateMigrationException("The new key serializer must be compatible.");
		}

		isKeySerializerCompatibilityChecked = true;
	}

	return serializationProxy;
}
 
Example #15
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testBroadcastStateRegistrationFailsIfNewValueSerializerIsIncompatible() {
	final String stateName = "broadcast-state";

	try {
		testBroadcastStateValueUpgrade(
			new MapStateDescriptor<>(
				stateName,
				IntSerializer.INSTANCE,
				new TestType.V1TestTypeSerializer()),
			new MapStateDescriptor<>(
				stateName,
				IntSerializer.INSTANCE,
				// new value serializer is incompatible
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed.");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	}
}
 
Example #16
Source File: StateBackendMigrationTestBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testKeyedValueStateRegistrationFailsIfNewStateSerializerIsIncompatible() throws Exception {
	final String stateName = "test-name";

	try {
		testKeyedValueStateUpgrade(
			new ValueStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ValueStateDescriptor<>(
				stateName,
				new TestType.IncompatibleTestTypeSerializer()));

		Assert.fail("should have failed");
	} catch (Exception expected) {
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}
 
Example #17
Source File: StateBackendMigrationTestBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testKeyedListStateRegistrationFailsIfNewStateSerializerIsIncompatible() throws Exception {
	final String stateName = "test-name";

	try {
		testKeyedListStateUpgrade(
			new ListStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ListStateDescriptor<>(
				stateName,
				new TestType.IncompatibleTestTypeSerializer()));

		Assert.fail("should have failed");
	} catch (Exception expected) {
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}
 
Example #18
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testBroadcastStateRegistrationFailsIfNewKeySerializerIsIncompatible() {
	final String stateName = "broadcast-state";

	try {
		testBroadcastStateKeyUpgrade(
			new MapStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer(),
				IntSerializer.INSTANCE),
			new MapStateDescriptor<>(
				stateName,
				// new key serializer is incompatible
				new TestType.IncompatibleTestTypeSerializer(),
				IntSerializer.INSTANCE));

		fail("should have failed.");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	}
}
 
Example #19
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testKeyedValueStateRegistrationFailsIfNewStateSerializerIsIncompatible() {
	final String stateName = "test-name";

	try {
		testKeyedValueStateUpgrade(
			new ValueStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ValueStateDescriptor<>(
				stateName,
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed");
	} catch (Exception expected) {
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}
 
Example #20
Source File: StateBackendMigrationTestBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testOperatorParitionableListStateRegistrationFailsIfNewSerializerIsIncompatible() throws Exception {
	final String stateName = "partitionable-list-state";

	try {
		testOperatorPartitionableListStateUpgrade(
			new ListStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ListStateDescriptor<>(
				stateName,
				// restore with a new incompatible serializer
				new TestType.IncompatibleTestTypeSerializer()));

		Assert.fail("should have failed.");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	}
}
 
Example #21
Source File: StateBackendMigrationTestBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testOperatorUnionListStateRegistrationFailsIfNewSerializerIsIncompatible() throws Exception {
	final String stateName = "union-list-state";

	try {
		testOperatorUnionListStateUpgrade(
			new ListStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ListStateDescriptor<>(
				stateName,
				// restore with a new incompatible serializer
				new TestType.IncompatibleTestTypeSerializer()));

		Assert.fail("should have failed.");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	}
}
 
Example #22
Source File: StateBackendMigrationTestBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testBroadcastStateRegistrationFailsIfNewValueSerializerIsIncompatible() throws Exception {
	final String stateName = "broadcast-state";

	try {
		testBroadcastStateValueUpgrade(
			new MapStateDescriptor<>(
				stateName,
				IntSerializer.INSTANCE,
				new TestType.V1TestTypeSerializer()),
			new MapStateDescriptor<>(
				stateName,
				IntSerializer.INSTANCE,
				// new value serializer is incompatible
				new TestType.IncompatibleTestTypeSerializer()));

		Assert.fail("should have failed.");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	}
}
 
Example #23
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testOperatorUnionListStateRegistrationFailsIfNewSerializerIsIncompatible() {
	final String stateName = "union-list-state";

	try {
		testOperatorUnionListStateUpgrade(
			new ListStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ListStateDescriptor<>(
				stateName,
				// restore with a new incompatible serializer
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed.");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	}
}
 
Example #24
Source File: RocksDBKeyedStateBackend.java    From flink with Apache License 2.0 6 votes vote down vote up
private <N, S extends State, SV> RegisteredKeyValueStateBackendMetaInfo<N, SV> updateRestoredStateMetaInfo(
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> oldStateInfo,
	StateDescriptor<S, SV> stateDesc,
	TypeSerializer<N> namespaceSerializer,
	TypeSerializer<SV> stateSerializer) throws Exception {

	@SuppressWarnings("unchecked")
	RegisteredKeyValueStateBackendMetaInfo<N, SV> restoredKvStateMetaInfo = oldStateInfo.f1;

	TypeSerializerSchemaCompatibility<N> s = restoredKvStateMetaInfo.updateNamespaceSerializer(namespaceSerializer);
	if (s.isCompatibleAfterMigration() || s.isIncompatible()) {
		throw new StateMigrationException("The new namespace serializer must be compatible.");
	}

	restoredKvStateMetaInfo.checkStateMetaInfo(stateDesc);

	TypeSerializerSchemaCompatibility<SV> newStateSerializerCompatibility =
		restoredKvStateMetaInfo.updateStateSerializer(stateSerializer);
	if (newStateSerializerCompatibility.isCompatibleAfterMigration()) {
		migrateStateValues(stateDesc, oldStateInfo);
	} else if (newStateSerializerCompatibility.isIncompatible()) {
		throw new StateMigrationException("The new state serializer cannot be incompatible.");
	}

	return restoredKvStateMetaInfo;
}
 
Example #25
Source File: RocksDBFullRestoreOperation.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Restore the KV-state / ColumnFamily meta data for all key-groups referenced by the current state handle.
 */
private void restoreKVStateMetaData() throws IOException, StateMigrationException {
	KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(currentStateHandleInView);

	this.keygroupStreamCompressionDecorator = serializationProxy.isUsingKeyGroupCompression() ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	List<StateMetaInfoSnapshot> restoredMetaInfos =
		serializationProxy.getStateMetaInfoSnapshots();
	currentStateHandleKVStateColumnFamilies = new ArrayList<>(restoredMetaInfos.size());

	for (StateMetaInfoSnapshot restoredMetaInfo : restoredMetaInfos) {
		RocksDbKvStateInfo registeredStateCFHandle =
			getOrRegisterStateColumnFamilyHandle(null, restoredMetaInfo);
		currentStateHandleKVStateColumnFamilies.add(registeredStateCFHandle.columnFamilyHandle);
	}
}
 
Example #26
Source File: RocksDBMapState.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void migrateSerializedValue(
	DataInputDeserializer serializedOldValueInput,
	DataOutputSerializer serializedMigratedValueOutput,
	TypeSerializer<Map<UK, UV>> priorSerializer,
	TypeSerializer<Map<UK, UV>> newSerializer) throws StateMigrationException {

	checkArgument(priorSerializer instanceof MapSerializer);
	checkArgument(newSerializer instanceof MapSerializer);

	TypeSerializer<UV> priorMapValueSerializer = ((MapSerializer<UK, UV>) priorSerializer).getValueSerializer();
	TypeSerializer<UV> newMapValueSerializer = ((MapSerializer<UK, UV>) newSerializer).getValueSerializer();

	try {
		boolean isNull = serializedOldValueInput.readBoolean();
		UV mapUserValue = null;
		if (!isNull) {
			mapUserValue = priorMapValueSerializer.deserialize(serializedOldValueInput);
		}
		serializedMigratedValueOutput.writeBoolean(mapUserValue == null);
		newMapValueSerializer.serialize(mapUserValue, serializedMigratedValueOutput);
	} catch (Exception e) {
		throw new StateMigrationException("Error while trying to migrate RocksDB map state.", e);
	}
}
 
Example #27
Source File: RocksDBMapState.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void migrateSerializedValue(
	DataInputDeserializer serializedOldValueInput,
	DataOutputSerializer serializedMigratedValueOutput,
	TypeSerializer<Map<UK, UV>> priorSerializer,
	TypeSerializer<Map<UK, UV>> newSerializer) throws StateMigrationException {

	checkArgument(priorSerializer instanceof MapSerializer);
	checkArgument(newSerializer instanceof MapSerializer);

	TypeSerializer<UV> priorMapValueSerializer = ((MapSerializer<UK, UV>) priorSerializer).getValueSerializer();
	TypeSerializer<UV> newMapValueSerializer = ((MapSerializer<UK, UV>) newSerializer).getValueSerializer();

	try {
		boolean isNull = serializedOldValueInput.readBoolean();
		UV mapUserValue = null;
		if (!isNull) {
			mapUserValue = priorMapValueSerializer.deserialize(serializedOldValueInput);
		}
		serializedMigratedValueOutput.writeBoolean(mapUserValue == null);
		newMapValueSerializer.serialize(mapUserValue, serializedMigratedValueOutput);
	} catch (Exception e) {
		throw new StateMigrationException("Error while trying to migrate RocksDB map state.", e);
	}
}
 
Example #28
Source File: RocksDBFullRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Restore one key groups state handle.
 */
private void restoreKeyGroupsInStateHandle()
	throws IOException, StateMigrationException, RocksDBException {
	try {
		currentStateHandleInStream = currentKeyGroupsStateHandle.openInputStream();
		cancelStreamRegistry.registerCloseable(currentStateHandleInStream);
		currentStateHandleInView = new DataInputViewStreamWrapper(currentStateHandleInStream);
		restoreKVStateMetaData();
		restoreKVStateData();
	} finally {
		if (cancelStreamRegistry.unregisterCloseable(currentStateHandleInStream)) {
			IOUtils.closeQuietly(currentStateHandleInStream);
		}
	}
}
 
Example #29
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testPriorityQueueStateCreationFailsIfNewSerializerIsNotCompatible() throws Exception {
	CheckpointStreamFactory streamFactory = createStreamFactory();
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();

	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {
		InternalPriorityQueue<TestType> internalPriorityQueue = backend.create(
			"testPriorityQueue", new TestType.V1TestTypeSerializer());

		internalPriorityQueue.add(new TestType("key-1", 123));
		internalPriorityQueue.add(new TestType("key-2", 346));
		internalPriorityQueue.add(new TestType("key-1", 777));

		KeyedStateHandle snapshot = runSnapshot(
			backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()),
			sharedStateRegistry);
		backend.dispose();

		backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot);
		backend.create(
			"testPriorityQueue", new TestType.IncompatibleTestTypeSerializer());

		fail("should have failed");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	} finally {
		backend.dispose();
	}
}
 
Example #30
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testPriorityQueueStateCreationFailsIfNewSerializerIsNotCompatible() throws Exception {
	CheckpointStreamFactory streamFactory = createStreamFactory();
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();

	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {
		InternalPriorityQueue<TestType> internalPriorityQueue = backend.create(
			"testPriorityQueue", new TestType.V1TestTypeSerializer());

		internalPriorityQueue.add(new TestType("key-1", 123));
		internalPriorityQueue.add(new TestType("key-2", 346));
		internalPriorityQueue.add(new TestType("key-1", 777));

		KeyedStateHandle snapshot = runSnapshot(
			backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()),
			sharedStateRegistry);
		backend.dispose();

		backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot);
		backend.create(
			"testPriorityQueue", new TestType.IncompatibleTestTypeSerializer());

		fail("should have failed");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	} finally {
		backend.dispose();
	}
}