Java Code Examples for org.apache.flink.api.common.ExecutionConfig#registerPojoType()

@Override
protected TypeSerializer<TestUserClassBase> createSerializer() {
	// only register one of the three child classes, the third child class is NO POJO
	ExecutionConfig conf = new ExecutionConfig();
	conf.registerPojoType(TestUserClass1.class);
	TypeSerializer<TestUserClassBase> serializer = type.createSerializer(conf);
	assert(serializer instanceof PojoSerializer);
	return serializer;
}
 

@Override
protected TypeSerializer<TestUserInterface> createSerializer() {
	// only register one of the two child classes
	ExecutionConfig conf = new ExecutionConfig();
	conf.registerPojoType(TestUserClass2.class);
	TypeSerializer<TestUserInterface> serializer = type.createSerializer(conf);
	assert(serializer instanceof KryoSerializer);
	return serializer;
}
 

@Override
protected TypeSerializer<TestUserClassBase> createSerializer() {
	// only register one of the three child classes, the third child class is NO POJO
	ExecutionConfig conf = new ExecutionConfig();
	conf.registerPojoType(TestUserClass1.class);
	TypeSerializer<TestUserClassBase> serializer = type.createSerializer(conf);
	assert(serializer instanceof PojoSerializer);
	return serializer;
}
 

private static TypeSerializer<TestPojo> testPojoSerializerWithSubclassRegisteredSupplier() {
	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.registerPojoType(TestPojoSubclass.class);

	TypeSerializer<TestPojo> serializer =
		TypeExtractor.createTypeInfo(TestPojo.class).createSerializer(executionConfig);

	assertTrue(serializer instanceof PojoSerializer);
	return serializer;
}
 

@Override
protected TypeSerializer<TestUserInterface> createSerializer() {
	// only register one of the two child classes
	ExecutionConfig conf = new ExecutionConfig();
	conf.registerPojoType(TestUserClass2.class);
	TypeSerializer<TestUserInterface> serializer = type.createSerializer(conf);
	assert(serializer instanceof KryoSerializer);
	return serializer;
}
 

@Benchmark
@OperationsPerInvocation(value = SerializationFrameworkMiniBenchmarks.RECORDS_PER_INVOCATION)
public void serializerPojo(FlinkEnvironmentContext context) throws Exception {
	StreamExecutionEnvironment env = context.env;
	env.setParallelism(4);
	ExecutionConfig executionConfig = env.getConfig();
	executionConfig.registerPojoType(MyPojo.class);
	executionConfig.registerPojoType(MyOperation.class);

	env.addSource(new PojoSource(RECORDS_PER_INVOCATION, 10))
			.rebalance()
			.addSink(new DiscardingSink<>());

	env.execute();
}
 

@Benchmark
@OperationsPerInvocation(value = SerializationFrameworkMiniBenchmarks.RECORDS_PER_INVOCATION)
public void serializerHeavyString(FlinkEnvironmentContext context) throws Exception {
	StreamExecutionEnvironment env = context.env;
	env.setParallelism(1);
	ExecutionConfig executionConfig = env.getConfig();
	executionConfig.registerPojoType(MyPojo.class);
	executionConfig.registerPojoType(MyOperation.class);

	env.addSource(new LongStringSource(RECORDS_PER_INVOCATION, 12))
			.rebalance()
			.addSink(new DiscardingSink<>());

	env.execute();
}
 

@Override
protected TypeSerializer<TestUserClassBase> createSerializer() {
	// only register one of the three child classes, the third child class is NO POJO
	ExecutionConfig conf = new ExecutionConfig();
	conf.registerPojoType(TestUserClass1.class);
	TypeSerializer<TestUserClassBase> serializer = type.createSerializer(conf);
	assert(serializer instanceof PojoSerializer);
	return serializer;
}
 

@Override
public TypeSerializer<BasePojo> createPriorSerializer() {
	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.registerPojoType(SubclassPojoBeforeSchemaUpgrade.class);

	TypeSerializer<BasePojo> serializer = TypeExtractor.createTypeInfo(BasePojo.class).createSerializer(executionConfig);
	assertSame(serializer.getClass(), PojoSerializer.class);
	return serializer;
}
 

@Override
public TypeSerializer<BasePojo> createUpgradedSerializer() {
	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.registerPojoType(SubclassPojoAfterSchemaUpgrade.class);

	TypeSerializer<BasePojo> serializer = TypeExtractor.createTypeInfo(BasePojo.class).createSerializer(executionConfig);
	assertSame(serializer.getClass(), PojoSerializer.class);
	return serializer;
}
 

@Override
public TypeSerializer<StaticSchemaPojo> createPriorSerializer() {
	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.registerPojoType(SubclassPojoWithIntField.class);

	TypeSerializer<StaticSchemaPojo> serializer = TypeExtractor.createTypeInfo(StaticSchemaPojo.class).createSerializer(executionConfig);
	assertSame(serializer.getClass(), PojoSerializer.class);
	return serializer;
}
 

@Override
public TypeSerializer<StaticSchemaPojo> createUpgradedSerializer() {
	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.registerPojoType(SubclassPojoWithStringField.class);

	TypeSerializer<StaticSchemaPojo> serializer = TypeExtractor.createTypeInfo(StaticSchemaPojo.class).createSerializer(executionConfig);
	assertSame(serializer.getClass(), PojoSerializer.class);
	return serializer;
}
 

@Override
public TypeSerializer<StaticSchemaPojo> createPriorSerializer() {
	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.registerPojoType(StaticSchemaPojoSubclassA.class);
	executionConfig.registerPojoType(StaticSchemaPojoSubclassB.class);

	TypeSerializer<StaticSchemaPojo> serializer = TypeExtractor.createTypeInfo(StaticSchemaPojo.class).createSerializer(executionConfig);
	assertSame(serializer.getClass(), PojoSerializer.class);
	return serializer;
}
 

@Override
public TypeSerializer<StaticSchemaPojo> createPriorSerializer() {
	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.registerPojoType(StaticSchemaPojoSubclassA.class);
	executionConfig.registerPojoType(StaticSchemaPojoSubclassB.class);

	TypeSerializer<StaticSchemaPojo> serializer = TypeExtractor.createTypeInfo(StaticSchemaPojo.class).createSerializer(executionConfig);
	assertSame(serializer.getClass(), PojoSerializer.class);
	return serializer;
}
 

@Override
public TypeSerializer<StaticSchemaPojo> createUpgradedSerializer() {
	ExecutionConfig executionConfig = new ExecutionConfig();
	// missing registration for subclass A compared to setup
	executionConfig.registerPojoType(StaticSchemaPojoSubclassB.class);

	TypeSerializer<StaticSchemaPojo> serializer = TypeExtractor.createTypeInfo(StaticSchemaPojo.class).createSerializer(executionConfig);
	assertSame(serializer.getClass(), PojoSerializer.class);
	return serializer;
}
 

@Override
public TypeSerializer<StaticSchemaPojo> createUpgradedSerializer() {
	ExecutionConfig executionConfig = new ExecutionConfig();
	// new registration for subclass A compared to setup
	executionConfig.registerPojoType(StaticSchemaPojoSubclassA.class);

	TypeSerializer<StaticSchemaPojo> serializer = TypeExtractor.createTypeInfo(StaticSchemaPojo.class).createSerializer(executionConfig);
	assertSame(serializer.getClass(), PojoSerializer.class);
	return serializer;
}
 

@Override
protected TypeSerializer<TestUserInterface> createSerializer() {
	// only register one of the two child classes
	ExecutionConfig conf = new ExecutionConfig();
	conf.registerPojoType(TestUserClass2.class);
	TypeSerializer<TestUserInterface> serializer = type.createSerializer(conf);
	assert(serializer instanceof KryoSerializer);
	return serializer;
}
 

/**
 * Tests that:
 *  - Previous Pojo serializer did not have registrations, and created cached serializers for subclasses
 *  - On restore, it had those subclasses registered
 *
 * In this case, after reconfiguration, the cache should be repopulated, and registrations should
 * also exist for the subclasses.
 *
 * Note: the cache still needs to be repopulated because previous data of those subclasses were
 * written with the cached serializers. In this case, the repopulated cache has reconfigured serializers
 * for the subclasses so that previous written data can be read, but the registered serializers
 * for the subclasses do not necessarily need to be reconfigured since they will only be used to
 * write new data.
 */
@Test
public void testReconfigureWithPreviouslyNonregisteredSubclasses() throws Exception {
	// don't register any subclasses at first
	PojoSerializer<TestUserClass> pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(new ExecutionConfig());

	// create cached serializers for SubTestUserClassA and SubTestUserClassB
	pojoSerializer.getSubclassSerializer(SubTestUserClassA.class);
	pojoSerializer.getSubclassSerializer(SubTestUserClassB.class);

	// make sure serializers are in cache
	assertEquals(2, pojoSerializer.getSubclassSerializerCache().size());
	assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class));
	assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class));

	// make sure that registrations are empty
	assertTrue(pojoSerializer.getRegisteredClasses().isEmpty());
	assertEquals(0, pojoSerializer.getRegisteredSerializers().length);

	// snapshot configuration and serialize to bytes
	TypeSerializerSnapshot pojoSerializerConfigSnapshot = pojoSerializer.snapshotConfiguration();
	byte[] serializedConfig;
	try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
		TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(
			new DataOutputViewStreamWrapper(out), pojoSerializerConfigSnapshot, pojoSerializer);
		serializedConfig = out.toByteArray();
	}

	// instantiate new PojoSerializer, with new execution config that has the subclass registrations
	ExecutionConfig newExecutionConfig = new ExecutionConfig();
	newExecutionConfig.registerPojoType(SubTestUserClassA.class);
	newExecutionConfig.registerPojoType(SubTestUserClassB.class);
	pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(newExecutionConfig);

	// read configuration from bytes
	try(ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
		pojoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(
			new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), pojoSerializer);
	}

	// reconfigure - check reconfiguration result and that
	// 1) subclass serializer cache is repopulated
	// 2) registrations also contain the now registered subclasses
	@SuppressWarnings("unchecked")
	TypeSerializerSchemaCompatibility<TestUserClass> compatResult =
		pojoSerializerConfigSnapshot.resolveSchemaCompatibility(pojoSerializer);
	assertTrue(compatResult.isCompatibleWithReconfiguredSerializer());
	assertTrue(compatResult.getReconfiguredSerializer() instanceof PojoSerializer);

	PojoSerializer<TestUserClass> reconfiguredPojoSerializer = (PojoSerializer<TestUserClass>) compatResult.getReconfiguredSerializer();
	assertEquals(2, reconfiguredPojoSerializer.getSubclassSerializerCache().size());
	assertTrue(reconfiguredPojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class));
	assertTrue(reconfiguredPojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class));
	assertEquals(2, reconfiguredPojoSerializer.getRegisteredClasses().size());
	assertTrue(reconfiguredPojoSerializer.getRegisteredClasses().containsKey(SubTestUserClassA.class));
	assertTrue(reconfiguredPojoSerializer.getRegisteredClasses().containsKey(SubTestUserClassB.class));
}
 

/**
 * Tests that:
 *  - Previous Pojo serializer did not have registrations, and created cached serializers for subclasses
 *  - On restore, it had those subclasses registered
 *
 * In this case, after reconfiguration, the cache should be repopulated, and registrations should
 * also exist for the subclasses.
 *
 * Note: the cache still needs to be repopulated because previous data of those subclasses were
 * written with the cached serializers. In this case, the repopulated cache has reconfigured serializers
 * for the subclasses so that previous written data can be read, but the registered serializers
 * for the subclasses do not necessarily need to be reconfigured since they will only be used to
 * write new data.
 */
@Test
public void testReconfigureWithPreviouslyNonregisteredSubclasses() throws Exception {
	// don't register any subclasses at first
	PojoSerializer<TestUserClass> pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(new ExecutionConfig());

	// create cached serializers for SubTestUserClassA and SubTestUserClassB
	pojoSerializer.getSubclassSerializer(SubTestUserClassA.class);
	pojoSerializer.getSubclassSerializer(SubTestUserClassB.class);

	// make sure serializers are in cache
	assertEquals(2, pojoSerializer.getSubclassSerializerCache().size());
	assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class));
	assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class));

	// make sure that registrations are empty
	assertTrue(pojoSerializer.getRegisteredClasses().isEmpty());
	assertEquals(0, pojoSerializer.getRegisteredSerializers().length);

	// snapshot configuration and serialize to bytes
	TypeSerializerSnapshot pojoSerializerConfigSnapshot = pojoSerializer.snapshotConfiguration();
	byte[] serializedConfig;
	try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
		TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(
			new DataOutputViewStreamWrapper(out), pojoSerializerConfigSnapshot, pojoSerializer);
		serializedConfig = out.toByteArray();
	}

	// instantiate new PojoSerializer, with new execution config that has the subclass registrations
	ExecutionConfig newExecutionConfig = new ExecutionConfig();
	newExecutionConfig.registerPojoType(SubTestUserClassA.class);
	newExecutionConfig.registerPojoType(SubTestUserClassB.class);
	pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(newExecutionConfig);

	// read configuration from bytes
	try(ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
		pojoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(
			new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), pojoSerializer);
	}

	// reconfigure - check reconfiguration result and that
	// 1) subclass serializer cache is repopulated
	// 2) registrations also contain the now registered subclasses
	@SuppressWarnings("unchecked")
	TypeSerializerSchemaCompatibility<TestUserClass> compatResult =
		pojoSerializerConfigSnapshot.resolveSchemaCompatibility(pojoSerializer);
	assertTrue(compatResult.isCompatibleWithReconfiguredSerializer());
	assertTrue(compatResult.getReconfiguredSerializer() instanceof PojoSerializer);

	PojoSerializer<TestUserClass> reconfiguredPojoSerializer = (PojoSerializer<TestUserClass>) compatResult.getReconfiguredSerializer();
	assertEquals(2, reconfiguredPojoSerializer.getSubclassSerializerCache().size());
	assertTrue(reconfiguredPojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class));
	assertTrue(reconfiguredPojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class));
	assertEquals(2, reconfiguredPojoSerializer.getRegisteredClasses().size());
	assertTrue(reconfiguredPojoSerializer.getRegisteredClasses().containsKey(SubTestUserClassA.class));
	assertTrue(reconfiguredPojoSerializer.getRegisteredClasses().containsKey(SubTestUserClassB.class));
}
 

/**
 * Tests that:
 *  - Previous Pojo serializer did not have registrations, and created cached serializers for subclasses
 *  - On restore, it had those subclasses registered
 *
 * In this case, after reconfiguration, the cache should be repopulated, and registrations should
 * also exist for the subclasses.
 *
 * Note: the cache still needs to be repopulated because previous data of those subclasses were
 * written with the cached serializers. In this case, the repopulated cache has reconfigured serializers
 * for the subclasses so that previous written data can be read, but the registered serializers
 * for the subclasses do not necessarily need to be reconfigured since they will only be used to
 * write new data.
 */
@Test
public void testReconfigureWithPreviouslyNonregisteredSubclasses() throws Exception {
	// don't register any subclasses at first
	PojoSerializer<TestUserClass> pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(new ExecutionConfig());

	// create cached serializers for SubTestUserClassA and SubTestUserClassB
	pojoSerializer.getSubclassSerializer(SubTestUserClassA.class);
	pojoSerializer.getSubclassSerializer(SubTestUserClassB.class);

	// make sure serializers are in cache
	assertEquals(2, pojoSerializer.getSubclassSerializerCache().size());
	assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class));
	assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class));

	// make sure that registrations are empty
	assertTrue(pojoSerializer.getRegisteredClasses().isEmpty());
	assertEquals(0, pojoSerializer.getRegisteredSerializers().length);

	// snapshot configuration and serialize to bytes
	TypeSerializerSnapshot pojoSerializerConfigSnapshot = pojoSerializer.snapshotConfiguration();
	byte[] serializedConfig;
	try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
		TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(
			new DataOutputViewStreamWrapper(out), pojoSerializerConfigSnapshot, pojoSerializer);
		serializedConfig = out.toByteArray();
	}

	// instantiate new PojoSerializer, with new execution config that has the subclass registrations
	ExecutionConfig newExecutionConfig = new ExecutionConfig();
	newExecutionConfig.registerPojoType(SubTestUserClassA.class);
	newExecutionConfig.registerPojoType(SubTestUserClassB.class);
	pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(newExecutionConfig);

	// read configuration from bytes
	try(ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
		pojoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(
			new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), pojoSerializer);
	}

	// reconfigure - check reconfiguration result and that
	// 1) subclass serializer cache is repopulated
	// 2) registrations also contain the now registered subclasses
	@SuppressWarnings("unchecked")
	TypeSerializerSchemaCompatibility<TestUserClass> compatResult =
		pojoSerializerConfigSnapshot.resolveSchemaCompatibility(pojoSerializer);
	assertTrue(compatResult.isCompatibleWithReconfiguredSerializer());
	assertTrue(compatResult.getReconfiguredSerializer() instanceof PojoSerializer);

	PojoSerializer<TestUserClass> reconfiguredPojoSerializer = (PojoSerializer<TestUserClass>) compatResult.getReconfiguredSerializer();
	assertEquals(2, reconfiguredPojoSerializer.getSubclassSerializerCache().size());
	assertTrue(reconfiguredPojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class));
	assertTrue(reconfiguredPojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class));
	assertEquals(2, reconfiguredPojoSerializer.getRegisteredClasses().size());
	assertTrue(reconfiguredPojoSerializer.getRegisteredClasses().containsKey(SubTestUserClassA.class));
	assertTrue(reconfiguredPojoSerializer.getRegisteredClasses().containsKey(SubTestUserClassB.class));
}