org.apache.flink.runtime.testutils.statemigration.TestType Java Examples

The following examples show how to use org.apache.flink.runtime.testutils.statemigration.TestType. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testKeyedListStateRegistrationFailsIfNewStateSerializerIsIncompatible() {
	final String stateName = "test-name";

	try {
		testKeyedListStateUpgrade(
			new ListStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ListStateDescriptor<>(
				stateName,
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed");
	} catch (Exception expected) {
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}
 
Example #2
Source File: StateSerializerProviderTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testLazilyRegisterIncompatibleSerializer() {
	TestType.V1TestTypeSerializer serializer = new TestType.V1TestTypeSerializer();
	StateSerializerProvider<TestType> testProvider = StateSerializerProvider.fromPreviousSerializerSnapshot(serializer.snapshotConfiguration());

	// register serializer that requires migration for state
	TypeSerializerSchemaCompatibility<TestType> schemaCompatibility =
		testProvider.registerNewSerializerForRestoredState(new TestType.IncompatibleTestTypeSerializer());
	assertTrue(schemaCompatibility.isIncompatible());

	try {
		// a serializer for the current schema will no longer be accessible
		testProvider.currentSchemaSerializer();

		fail();
	} catch (Exception excepted) {
		// success
	}
}
 
Example #3
Source File: StateBackendMigrationTestBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testKeyedValueStateRegistrationFailsIfNewStateSerializerIsIncompatible() throws Exception {
	final String stateName = "test-name";

	try {
		testKeyedValueStateUpgrade(
			new ValueStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ValueStateDescriptor<>(
				stateName,
				new TestType.IncompatibleTestTypeSerializer()));

		Assert.fail("should have failed");
	} catch (Exception expected) {
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}
 
Example #4
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testKeyedValueStateRegistrationFailsIfNewStateSerializerIsIncompatible() {
	final String stateName = "test-name";

	try {
		testKeyedValueStateUpgrade(
			new ValueStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ValueStateDescriptor<>(
				stateName,
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed");
	} catch (Exception expected) {
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}
 
Example #5
Source File: StateSerializerProviderTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testEagerlyRegisterIncompatibleSerializer() {
	StateSerializerProvider<TestType> testProvider = StateSerializerProvider.fromNewRegisteredSerializer(new TestType.IncompatibleTestTypeSerializer());

	// set previous serializer snapshot for state, which should let the new serializer be considered incompatible
	TypeSerializerSchemaCompatibility<TestType> schemaCompatibility =
		testProvider.setPreviousSerializerSnapshotForRestoredState(new TestType.V1TestTypeSerializer().snapshotConfiguration());
	assertTrue(schemaCompatibility.isIncompatible());

	try {
		// a serializer for the current schema will no longer be accessible
		testProvider.currentSchemaSerializer();

		fail();
	} catch (Exception excepted) {
		// success
	}
}
 
Example #6
Source File: StateSerializerProviderTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testEagerlyRegisterIncompatibleSerializer() {
	StateSerializerProvider<TestType> testProvider = StateSerializerProvider.fromNewRegisteredSerializer(new TestType.IncompatibleTestTypeSerializer());

	// set previous serializer snapshot for state, which should let the new serializer be considered incompatible
	TypeSerializerSchemaCompatibility<TestType> schemaCompatibility =
		testProvider.setPreviousSerializerSnapshotForRestoredState(new TestType.V1TestTypeSerializer().snapshotConfiguration());
	assertTrue(schemaCompatibility.isIncompatible());

	try {
		// a serializer for the current schema will no longer be accessible
		testProvider.currentSchemaSerializer();

		fail();
	} catch (Exception excepted) {
		// success
	}
}
 
Example #7
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testKeyedValueStateRegistrationFailsIfNewStateSerializerIsIncompatible() {
	final String stateName = "test-name";

	try {
		testKeyedValueStateUpgrade(
			new ValueStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ValueStateDescriptor<>(
				stateName,
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed");
	} catch (Exception expected) {
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}
 
Example #8
Source File: StateSerializerProviderTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testEagerlyRegisterIncompatibleSerializer() {
	StateSerializerProvider<TestType> testProvider = StateSerializerProvider.fromNewRegisteredSerializer(new TestType.IncompatibleTestTypeSerializer());

	// set previous serializer snapshot for state, which should let the new serializer be considered incompatible
	TypeSerializerSchemaCompatibility<TestType> schemaCompatibility =
		testProvider.setPreviousSerializerSnapshotForRestoredState(new TestType.V1TestTypeSerializer().snapshotConfiguration());
	assertTrue(schemaCompatibility.isIncompatible());

	try {
		// a serializer for the current schema will no longer be accessible
		testProvider.currentSchemaSerializer();

		fail();
	} catch (Exception excepted) {
		// success
	}
}
 
Example #9
Source File: StateBackendMigrationTestBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testOperatorParitionableListStateRegistrationFailsIfNewSerializerIsIncompatible() throws Exception {
	final String stateName = "partitionable-list-state";

	try {
		testOperatorPartitionableListStateUpgrade(
			new ListStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ListStateDescriptor<>(
				stateName,
				// restore with a new incompatible serializer
				new TestType.IncompatibleTestTypeSerializer()));

		Assert.fail("should have failed.");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	}
}
 
Example #10
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testBroadcastStateRegistrationFailsIfNewValueSerializerIsIncompatible() {
	final String stateName = "broadcast-state";

	try {
		testBroadcastStateValueUpgrade(
			new MapStateDescriptor<>(
				stateName,
				IntSerializer.INSTANCE,
				new TestType.V1TestTypeSerializer()),
			new MapStateDescriptor<>(
				stateName,
				IntSerializer.INSTANCE,
				// new value serializer is incompatible
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed.");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	}
}
 
Example #11
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testBroadcastStateRegistrationFailsIfNewKeySerializerIsIncompatible() {
	final String stateName = "broadcast-state";

	try {
		testBroadcastStateKeyUpgrade(
			new MapStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer(),
				IntSerializer.INSTANCE),
			new MapStateDescriptor<>(
				stateName,
				// new key serializer is incompatible
				new TestType.IncompatibleTestTypeSerializer(),
				IntSerializer.INSTANCE));

		fail("should have failed.");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	}
}
 
Example #12
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testOperatorParitionableListStateRegistrationFailsIfNewSerializerIsIncompatible() throws Exception {
	final String stateName = "partitionable-list-state";

	try {
		testOperatorPartitionableListStateUpgrade(
			new ListStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ListStateDescriptor<>(
				stateName,
				// restore with a new incompatible serializer
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed.");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	}
}
 
Example #13
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testOperatorUnionListStateRegistrationFailsIfNewSerializerIsIncompatible() {
	final String stateName = "union-list-state";

	try {
		testOperatorUnionListStateUpgrade(
			new ListStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ListStateDescriptor<>(
				stateName,
				// restore with a new incompatible serializer
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed.");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	}
}
 
Example #14
Source File: StateSerializerProviderTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test(expected = UnsupportedOperationException.class)
public void testSetSerializerSnapshotWithLazilyRegisteredSerializerProviderShouldFail() {
	TestType.V1TestTypeSerializer serializer = new TestType.V1TestTypeSerializer();
	StateSerializerProvider<TestType> testProvider = StateSerializerProvider.fromPreviousSerializerSnapshot(serializer.snapshotConfiguration());

	testProvider.setPreviousSerializerSnapshotForRestoredState(serializer.snapshotConfiguration());
}
 
Example #15
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testPriorityQueueStateCreationFailsIfNewSerializerIsNotCompatible() throws Exception {
	CheckpointStreamFactory streamFactory = createStreamFactory();
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();

	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {
		InternalPriorityQueue<TestType> internalPriorityQueue = backend.create(
			"testPriorityQueue", new TestType.V1TestTypeSerializer());

		internalPriorityQueue.add(new TestType("key-1", 123));
		internalPriorityQueue.add(new TestType("key-2", 346));
		internalPriorityQueue.add(new TestType("key-1", 777));

		KeyedStateHandle snapshot = runSnapshot(
			backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()),
			sharedStateRegistry);
		backend.dispose();

		backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot);
		backend.create(
			"testPriorityQueue", new TestType.IncompatibleTestTypeSerializer());

		fail("should have failed");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	} finally {
		backend.dispose();
	}
}
 
Example #16
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testKeyedMapStateAsIs() throws Exception {
	final String stateName = "test-name";

	testKeyedMapStateUpgrade(
		new MapStateDescriptor<>(
			stateName,
			IntSerializer.INSTANCE,
			new TestType.V1TestTypeSerializer()),
		new MapStateDescriptor<>(
			stateName,
			IntSerializer.INSTANCE,
			new TestType.V1TestTypeSerializer()));
}
 
Example #17
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testKeyedMapStateSerializerReconfiguration() throws Exception {
	final String stateName = "test-name";

	testKeyedMapStateUpgrade(
		new MapStateDescriptor<>(
			stateName,
			IntSerializer.INSTANCE,
			new TestType.V1TestTypeSerializer()),
		new MapStateDescriptor<>(
			stateName,
			IntSerializer.INSTANCE,
			// restore with a V2 serializer that has a different schema
			new TestType.ReconfigurationRequiringTestTypeSerializer()));
}
 
Example #18
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testKeyedStateRegistrationFailsIfNewNamespaceSerializerIsIncompatible() throws Exception {
	try {
		testNamespaceSerializerUpgrade(
			new TestType.V1TestTypeSerializer(),
			new TestType.IncompatibleTestTypeSerializer());

		fail("should have failed");
	} catch (Exception expected) {
		// the new namespace serializer is incompatible; this should fail the restore
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}
 
Example #19
Source File: StateSerializerProviderTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test(expected = UnsupportedOperationException.class)
public void testPreviousSchemaSerializerForEagerlyRegisteredStateSerializerProvider() {
	StateSerializerProvider<TestType> testProvider = StateSerializerProvider.fromNewRegisteredSerializer(new TestType.V1TestTypeSerializer());

	// this should fail with an exception
	testProvider.previousSchemaSerializer();
}
 
Example #20
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testOperatorUnionListStateSerializerReconfiguration() throws Exception {
	final String stateName = "union-list-state";

	testOperatorUnionListStateUpgrade(
		new ListStateDescriptor<>(
			stateName,
			new TestType.V1TestTypeSerializer()),
		new ListStateDescriptor<>(
			stateName,
			// restore with a new serializer that requires reconfiguration
			new TestType.ReconfigurationRequiringTestTypeSerializer()));
}
 
Example #21
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testKeyedValueStateSerializerReconfiguration() throws Exception {
	final String stateName = "test-name";

	testKeyedValueStateUpgrade(
		new ValueStateDescriptor<>(
			stateName,
			new TestType.V1TestTypeSerializer()),
		new ValueStateDescriptor<>(
			stateName,
			// the test fails if this serializer is used instead of a reconfigured new serializer
			new TestType.ReconfigurationRequiringTestTypeSerializer()));
}
 
Example #22
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testBroadcastStateValueSerializerReconfiguration() throws Exception {
	final String stateName = "broadcast-state";

	testBroadcastStateValueUpgrade(
		new MapStateDescriptor<>(
			stateName,
			IntSerializer.INSTANCE,
			new TestType.V1TestTypeSerializer()),
		new MapStateDescriptor<>(
			stateName,
			IntSerializer.INSTANCE,
			// new value serializer is a new serializer that requires reconfiguration
			new TestType.ReconfigurationRequiringTestTypeSerializer()));
}
 
Example #23
Source File: StateBackendMigrationTestBase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private void testBroadcastStateKeyUpgrade(
		MapStateDescriptor<TestType, Integer> initialAccessDescriptor,
		MapStateDescriptor<TestType, Integer> newAccessDescriptorAfterRestore) throws Exception {

	CheckpointStreamFactory streamFactory = createStreamFactory();

	OperatorStateBackend backend = createOperatorStateBackend();

	try {
		BroadcastState<TestType, Integer> state = backend.getBroadcastState(initialAccessDescriptor);

		state.put(new TestType("foo", 13), 3);
		state.put(new TestType("bar", 278), 5);

		OperatorStateHandle snapshot = runSnapshot(
			backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()));
		backend.dispose();

		backend = restoreOperatorStateBackend(snapshot);

		state = backend.getBroadcastState(newAccessDescriptorAfterRestore);

		// the state backend should have decided whether or not it needs to perform state migration;
		// make sure that reading and writing each broadcast entry works with the new serializer
		Assert.assertEquals((Integer) 3, state.get(new TestType("foo", 13)));
		Assert.assertEquals((Integer) 5, state.get(new TestType("bar", 278)));
		state.put(new TestType("new-entry", 777), 17);
	} finally {
		backend.dispose();
	}
}
 
Example #24
Source File: StateBackendMigrationTestBase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private void testBroadcastStateValueUpgrade(
		MapStateDescriptor<Integer, TestType> initialAccessDescriptor,
		MapStateDescriptor<Integer, TestType> newAccessDescriptorAfterRestore) throws Exception {
	CheckpointStreamFactory streamFactory = createStreamFactory();

	OperatorStateBackend backend = createOperatorStateBackend();

	try {
		BroadcastState<Integer, TestType> state = backend.getBroadcastState(initialAccessDescriptor);

		state.put(3, new TestType("foo", 13));
		state.put(5, new TestType("bar", 278));

		OperatorStateHandle snapshot = runSnapshot(
			backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()));
		backend.dispose();

		backend = restoreOperatorStateBackend(snapshot);

		state = backend.getBroadcastState(newAccessDescriptorAfterRestore);

		// the state backend should have decided whether or not it needs to perform state migration;
		// make sure that reading and writing each broadcast entry works with the new serializer
		Assert.assertEquals(new TestType("foo", 13), state.get(3));
		Assert.assertEquals(new TestType("bar", 278), state.get(5));
		state.put(17, new TestType("new-entry", 777));
	} finally {
		backend.dispose();
	}
}
 
Example #25
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testOperatorParitionableListStateSerializerReconfiguration() throws Exception {
	final String stateName = "partitionable-list-state";

	testOperatorPartitionableListStateUpgrade(
		new ListStateDescriptor<>(
			stateName,
			new TestType.V1TestTypeSerializer()),
		new ListStateDescriptor<>(
			stateName,
			// restore with a new serializer that requires reconfiguration
			new TestType.ReconfigurationRequiringTestTypeSerializer()));
}
 
Example #26
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testKeyedListStateSerializerReconfiguration() throws Exception {
	final String stateName = "test-name";

	testKeyedListStateUpgrade(
		new ListStateDescriptor<>(
			stateName,
			new TestType.V1TestTypeSerializer()),
		new ListStateDescriptor<>(
			stateName,
			// the test fails if this serializer is used instead of a reconfigured new serializer
			new TestType.ReconfigurationRequiringTestTypeSerializer()));
}
 
Example #27
Source File: StateBackendMigrationTestBase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testBroadcastStateKeySerializerReconfiguration() throws Exception {
	final String stateName = "broadcast-state";

	testBroadcastStateKeyUpgrade(
		new MapStateDescriptor<>(
			stateName,
			new TestType.V1TestTypeSerializer(),
			IntSerializer.INSTANCE),
		new MapStateDescriptor<>(
			stateName,
			// new key serializer is a new serializer that requires reconfiguration
			new TestType.ReconfigurationRequiringTestTypeSerializer(),
			IntSerializer.INSTANCE));
}
 
Example #28
Source File: StateBackendMigrationTestBase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testBroadcastStateValueSerializerReconfiguration() throws Exception {
	final String stateName = "broadcast-state";

	testBroadcastStateValueUpgrade(
		new MapStateDescriptor<>(
			stateName,
			IntSerializer.INSTANCE,
			new TestType.V1TestTypeSerializer()),
		new MapStateDescriptor<>(
			stateName,
			IntSerializer.INSTANCE,
			// new value serializer is a new serializer that requires reconfiguration
			new TestType.ReconfigurationRequiringTestTypeSerializer()));
}
 
Example #29
Source File: StateBackendMigrationTestBase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testBroadcastStateKeyMigration() throws Exception {
	final String stateName = "broadcast-state";

	testBroadcastStateKeyUpgrade(
		new MapStateDescriptor<>(
			stateName,
			new TestType.V1TestTypeSerializer(),
			IntSerializer.INSTANCE),
		new MapStateDescriptor<>(
			stateName,
			// new key serializer is a V2 serializer with a different schema
			new TestType.V2TestTypeSerializer(),
			IntSerializer.INSTANCE));
}
 
Example #30
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testKeyedStateRegistrationFailsIfNewNamespaceSerializerRequiresMigration() throws Exception {
	try {
		testNamespaceSerializerUpgrade(
			new TestType.V1TestTypeSerializer(),
			new TestType.V2TestTypeSerializer());

		fail("should have failed");
	} catch (Exception expected) {
		// the new namespace serializer requires migration; this should fail the restore
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}