org.apache.flink.api.common.typeutils.TypeSerializerSnapshot Java Examples

The following examples show how to use org.apache.flink.api.common.typeutils.TypeSerializerSnapshot. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: PojoSerializerSnapshot.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Finds which registered subclasses exists both in the new {@link PojoSerializer} as well as in the previous one
 * (represented by this snapshot), and returns an {@link IntermediateCompatibilityResult}
 * of the serializers of this preexisting registered subclasses.
 */
private static <T> IntermediateCompatibilityResult<T> getCompatibilityOfPreExistingRegisteredSubclasses(
		PojoSerializer<T> newPojoSerializer,
		LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> registeredSubclassSerializerSnapshots) {

	final LinkedHashMap<Class<?>, TypeSerializerSnapshot<?>> unwrappedSerializerSnapshots = registeredSubclassSerializerSnapshots.unwrapOptionals();

	final ArrayList<TypeSerializerSnapshot<?>> associatedSubclassSerializerSnapshots = new ArrayList<>();
	final ArrayList<TypeSerializer<?>> associatedNewSubclassSerializers = new ArrayList<>();

	final LinkedHashMap<Class<?>, TypeSerializer<?>> newSubclassSerializerRegistry = newPojoSerializer.getBundledSubclassSerializerRegistry();

	for (Map.Entry<Class<?>, TypeSerializerSnapshot<?>> entry : unwrappedSerializerSnapshots.entrySet()) {
		TypeSerializer<?> newRegisteredSerializer = newSubclassSerializerRegistry.get(entry.getKey());
		if (newRegisteredSerializer != null) {
			associatedSubclassSerializerSnapshots.add(entry.getValue());
			associatedNewSubclassSerializers.add(newRegisteredSerializer);
		}
	}

	return CompositeTypeSerializerUtil.constructIntermediateCompatibilityResult(
		associatedNewSubclassSerializers.toArray(new TypeSerializer<?>[associatedNewSubclassSerializers.size()]),
		associatedSubclassSerializerSnapshots.toArray(new TypeSerializerSnapshot<?>[associatedSubclassSerializerSnapshots.size()]));
}
 
Example #2
Source File: KryoSerializerCompatibilityTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testMigrationStrategyForRemovedAvroDependency() throws Exception {
	KryoSerializer<TestClass> kryoSerializerForA = new KryoSerializer<>(TestClass.class, new ExecutionConfig());

	// read configuration again from bytes
	TypeSerializerSnapshot kryoSerializerConfigSnapshot;
	try (InputStream in = getClass().getResourceAsStream("/kryo-serializer-flink1.3-snapshot")) {
		kryoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(
			new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), kryoSerializerForA);
	}

	@SuppressWarnings("unchecked")
	TypeSerializerSchemaCompatibility<TestClass> compatResult =
		kryoSerializerConfigSnapshot.resolveSchemaCompatibility(kryoSerializerForA);
	assertTrue(compatResult.isCompatibleAsIs());
}
 
Example #3
Source File: PojoSerializerSnapshot.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Creates a reconfigured version of the {@link PojoSerializer}.
 *
 * @param originalNewPojoSerializer the original new {@link PojoSerializer} to create a reconfigured version of.
 * @param fieldSerializerCompatibility compatibility of preexisting fields' serializers.
 * @param registeredSerializerSnapshots snapshot of previous registered subclasses' serializers.
 * @param preExistingRegistrationsCompatibility compatibility of preexisting subclasses' serializers.
 * @param nonRegisteredSubclassSerializerSnapshots snapshot of previous non-registered subclasses' serializers.
 *
 * @return a reconfigured version of the original new {@link PojoSerializer}.
 */
private static <T> PojoSerializer<T> constructReconfiguredPojoSerializer(
		PojoSerializer<T> originalNewPojoSerializer,
		IntermediateCompatibilityResult<T> fieldSerializerCompatibility,
		LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> registeredSerializerSnapshots,
		IntermediateCompatibilityResult<T> preExistingRegistrationsCompatibility,
		LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> nonRegisteredSubclassSerializerSnapshots) {

	@SuppressWarnings("unchecked")
	final TypeSerializer<Object>[] reconfiguredFieldSerializers = constructReconfiguredFieldSerializers(fieldSerializerCompatibility);

	Tuple2<LinkedHashMap<Class<?>, Integer>, TypeSerializer<Object>[]> reconfiguredSubclassRegistry = constructReconfiguredSubclassRegistry(
		originalNewPojoSerializer.getBundledSubclassSerializerRegistry(),
		registeredSerializerSnapshots,
		preExistingRegistrationsCompatibility);

	return new PojoSerializer<>(
		originalNewPojoSerializer.getPojoClass(),
		originalNewPojoSerializer.getFields(),
		reconfiguredFieldSerializers,
		reconfiguredSubclassRegistry.f0,
		reconfiguredSubclassRegistry.f1,
		restoreSerializers(nonRegisteredSubclassSerializerSnapshots.unwrapOptionals()),
		originalNewPojoSerializer.getExecutionConfig());
}
 
Example #4
Source File: Lockable.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public <U> TypeSerializerSnapshot<Lockable<E>> transformLegacySerializerSnapshot(TypeSerializerSnapshot<U> legacySnapshot) {
	if (legacySnapshot instanceof LockableTypeSerializerSnapshot) {
		return (TypeSerializerSnapshot<Lockable<E>>) legacySnapshot;
	}

	// In Flink 1.6, this serializer was directly returning the elementSerializer's snapshot
	// instead of wrapping it in a LockableTypeSerializer(Config)Snapshot.
	// This caused state information to be written as <LockableTypeSerializer, SomeArbitrarySerializerSnapshot>,
	// Therefore we need to preform the following transformation:
	// 	1. set the prior serializer on the legacySnapshot to be the elementSerializer
	// 	2. return a LockableTypeSerializerSnapshot that has the legacySnapshot as a nested snapshot.
	if (legacySnapshot instanceof TypeSerializerConfigSnapshot) {
		setElementSerializerAsPriorSerializer(legacySnapshot, this.elementSerializer);
	}
	LockableTypeSerializerSnapshot<E> lockableSnapshot = new LockableTypeSerializerSnapshot<>();
	CompositeTypeSerializerUtil.setNestedSerializersSnapshots(lockableSnapshot, legacySnapshot);
	return lockableSnapshot;
}
 
Example #5
Source File: LegacyStateMetaInfoReaders.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Nonnull
@Override
public StateMetaInfoSnapshot readStateMetaInfoSnapshot(
	@Nonnull DataInputView in, @Nonnull ClassLoader userCodeClassLoader) throws IOException {

	final StateDescriptor.Type stateDescType = StateDescriptor.Type.values()[in.readInt()];
	final String stateName = in.readUTF();
	List<Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> serializersAndConfigs =
		TypeSerializerSerializationUtil.readSerializersAndConfigsWithResilience(in, userCodeClassLoader);

	Map<String, String> optionsMap = Collections.singletonMap(
		StateMetaInfoSnapshot.CommonOptionsKeys.KEYED_STATE_TYPE.toString(),
		stateDescType.toString());

	Map<String, TypeSerializerSnapshot<?>> serializerConfigSnapshotMap = new HashMap<>(2);
	serializerConfigSnapshotMap.put(StateMetaInfoSnapshot.CommonSerializerKeys.NAMESPACE_SERIALIZER.toString(),
		serializersAndConfigs.get(0).f1);
	serializerConfigSnapshotMap.put(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER.toString(),
		serializersAndConfigs.get(1).f1);

	return new StateMetaInfoSnapshot(
		stateName,
		StateMetaInfoSnapshot.BackendStateType.KEY_VALUE,
		optionsMap,
		serializerConfigSnapshotMap);
}
 
Example #6
Source File: LegacyStateMetaInfoReaders.java    From flink with Apache License 2.0 6 votes vote down vote up
@Nonnull
@Override
public StateMetaInfoSnapshot readStateMetaInfoSnapshot(
	@Nonnull DataInputView in, @Nonnull ClassLoader userCodeClassLoader) throws IOException {

	final StateDescriptor.Type stateDescType = StateDescriptor.Type.values()[in.readInt()];
	final String stateName = in.readUTF();
	List<Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> serializersAndConfigs =
		TypeSerializerSerializationUtil.readSerializersAndConfigsWithResilience(in, userCodeClassLoader);

	Map<String, String> optionsMap = Collections.singletonMap(
		StateMetaInfoSnapshot.CommonOptionsKeys.KEYED_STATE_TYPE.toString(),
		stateDescType.toString());

	Map<String, TypeSerializerSnapshot<?>> serializerConfigSnapshotMap = new HashMap<>(2);
	serializerConfigSnapshotMap.put(StateMetaInfoSnapshot.CommonSerializerKeys.NAMESPACE_SERIALIZER.toString(),
		serializersAndConfigs.get(0).f1);
	serializerConfigSnapshotMap.put(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER.toString(),
		serializersAndConfigs.get(1).f1);

	return new StateMetaInfoSnapshot(
		stateName,
		StateMetaInfoSnapshot.BackendStateType.KEY_VALUE,
		optionsMap,
		serializerConfigSnapshotMap);
}
 
Example #7
Source File: EitherSerializerConfigSnapshot.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public TypeSerializerSchemaCompatibility<Either<L, R>> resolveSchemaCompatibility(
		TypeSerializer<Either<L, R>> newSerializer) {

	// this class was shared between the Java Either Serializer and the
	// Scala Either serializer
	if (newSerializer.getClass() == EitherSerializer.class) {
		List<Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> nestedSerializersAndConfigs = getNestedSerializersAndConfigs();
		return CompositeTypeSerializerUtil.delegateCompatibilityCheckToNewSnapshot(
			newSerializer,
			new JavaEitherSerializerSnapshot<>(),
			nestedSerializersAndConfigs.get(0).f1,
			nestedSerializersAndConfigs.get(1).f1);
	}
	else {
		// fall back to the backwards compatibility path
		return super.resolveSchemaCompatibility(newSerializer);
	}
}
 
Example #8
Source File: PojoSerializerSnapshot.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private static Tuple2<LinkedHashMap<Class<?>, Integer>, TypeSerializer<Object>[]> constructReconfiguredSubclassRegistry(
		LinkedHashMap<Class<?>, TypeSerializer<?>> newSubclassRegistrations,
		LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> registeredSerializerSnapshots,
		IntermediateCompatibilityResult<?> preExistingRegistrationsCompatibility) {

	checkArgument(!preExistingRegistrationsCompatibility.isIncompatible() && !preExistingRegistrationsCompatibility.isCompatibleAfterMigration());

	LinkedHashMap<Class<?>, TypeSerializer<?>> reconfiguredSubclassSerializerRegistry =
		restoreSerializers(registeredSerializerSnapshots.unwrapOptionals());

	Iterator<TypeSerializer<?>> serializersForPreexistingRegistrations =
		Arrays.asList(preExistingRegistrationsCompatibility.getNestedSerializers()).iterator();

	for (Map.Entry<Class<?>, TypeSerializer<?>> registration : newSubclassRegistrations.entrySet()) {
		// new registrations should simply be appended to the subclass serializer registry with their new serializers;
		// preexisting registrations should use the compatibility-checked serializer
		TypeSerializer<?> newRegistration = (reconfiguredSubclassSerializerRegistry.containsKey(registration.getKey()))
			? serializersForPreexistingRegistrations.next()
			: registration.getValue();
		reconfiguredSubclassSerializerRegistry.put(registration.getKey(), newRegistration);
	}

	return decomposeSubclassSerializerRegistry(reconfiguredSubclassSerializerRegistry);
}
 
Example #9
Source File: PojoSerializerSnapshotTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private static PojoSerializerSnapshot<TestPojo> buildTestSnapshot(List<TestPojoField> fieldsToContainInSnapshot) {

		int numFields = fieldsToContainInSnapshot.size();
		ArrayList<Field> fields = new ArrayList<>(numFields);
		ArrayList<TypeSerializerSnapshot<?>> fieldSerializerSnapshots = new ArrayList<>(numFields);
		fieldsToContainInSnapshot.forEach(testPojoField -> {
			fields.add(testPojoField.field);
			fieldSerializerSnapshots.add(testPojoField.serializerSnapshot);
		});

		return new PojoSerializerSnapshot<>(
			TestPojo.class,
			fields.toArray(new Field[numFields]),
			fieldSerializerSnapshots.toArray(new TypeSerializerSnapshot[numFields]),
			new LinkedHashMap<>(),
			new LinkedHashMap<>());
	}
 
Example #10
Source File: PojoSerializer.java    From flink with Apache License 2.0 6 votes vote down vote up
public PojoSerializerConfigSnapshot(
		Class<T> pojoType,
		LinkedHashMap<String, Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> fieldToSerializerConfigSnapshot,
		LinkedHashMap<Class<?>, Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> registeredSubclassesToSerializerConfigSnapshots,
		HashMap<Class<?>, Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> nonRegisteredSubclassesToSerializerConfigSnapshots,
		boolean ignoreTypeSerializerSerialization) {

	super(pojoType);

	this.fieldToSerializerConfigSnapshot =
			Preconditions.checkNotNull(fieldToSerializerConfigSnapshot);
	this.registeredSubclassesToSerializerConfigSnapshots =
			Preconditions.checkNotNull(registeredSubclassesToSerializerConfigSnapshots);
	this.nonRegisteredSubclassesToSerializerConfigSnapshots =
			Preconditions.checkNotNull(nonRegisteredSubclassesToSerializerConfigSnapshots);

	this.ignoreTypeSerializerSerialization = ignoreTypeSerializerSerialization;
}
 
Example #11
Source File: EnumSerializerUpgradeTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private static TypeSerializerSchemaCompatibility checkCompatibility(String enumSourceA, String enumSourceB)
	throws IOException, ClassNotFoundException {

	ClassLoader classLoader = ClassLoaderUtils.compileAndLoadJava(
		temporaryFolder.newFolder(), ENUM_NAME + ".java", enumSourceA);

	EnumSerializer enumSerializer = new EnumSerializer(classLoader.loadClass(ENUM_NAME));

	TypeSerializerSnapshot snapshot = enumSerializer.snapshotConfiguration();
	byte[] snapshotBytes;
	try (
		ByteArrayOutputStream outBuffer = new ByteArrayOutputStream();
		DataOutputViewStreamWrapper outputViewStreamWrapper = new DataOutputViewStreamWrapper(outBuffer)) {

		TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(
			outputViewStreamWrapper, snapshot, enumSerializer);
		snapshotBytes = outBuffer.toByteArray();
	}

	ClassLoader classLoader2 = ClassLoaderUtils.compileAndLoadJava(
		temporaryFolder.newFolder(), ENUM_NAME + ".java", enumSourceB);

	TypeSerializerSnapshot restoredSnapshot;
	try (
		ByteArrayInputStream inBuffer = new ByteArrayInputStream(snapshotBytes);
		DataInputViewStreamWrapper inputViewStreamWrapper = new DataInputViewStreamWrapper(inBuffer)) {

		restoredSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(
			inputViewStreamWrapper, classLoader2, enumSerializer);
	}

	EnumSerializer enumSerializer2 = new EnumSerializer(classLoader2.loadClass(ENUM_NAME));
	return restoredSnapshot.resolveSchemaCompatibility(enumSerializer2);
}
 
Example #12
Source File: AvroSerializerSnapshotTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void nonValidSchemaEvaluationShouldResultInCompatibleSerializers() {
	final AvroSerializer<GenericRecord> originalSerializer = new AvroSerializer<>(GenericRecord.class, FIRST_REQUIRED_LAST_OPTIONAL);
	final AvroSerializer<GenericRecord> newSerializer = new AvroSerializer<>(GenericRecord.class, BOTH_REQUIRED);

	TypeSerializerSnapshot<GenericRecord> originalSnapshot = originalSerializer.snapshotConfiguration();

	assertThat(originalSnapshot.resolveSchemaCompatibility(newSerializer), isIncompatible());
}
 
Example #13
Source File: RegisteredPriorityQueueStateBackendMetaInfo.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
public RegisteredPriorityQueueStateBackendMetaInfo(StateMetaInfoSnapshot snapshot) {
	this(
		snapshot.getName(),
		StateSerializerProvider.fromPreviousSerializerSnapshot(
			(TypeSerializerSnapshot<T>) Preconditions.checkNotNull(
				snapshot.getTypeSerializerSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER))));

	Preconditions.checkState(StateMetaInfoSnapshot.BackendStateType.PRIORITY_QUEUE == snapshot.getBackendStateType());
}
 
Example #14
Source File: KryoSerializer.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public TypeSerializerSnapshot<T> snapshotConfiguration() {
	return new KryoSerializerSnapshot<>(
		type,
		defaultSerializers,
		defaultSerializerClasses,
		kryoRegistrations);
}
 
Example #15
Source File: PojoSerializerSnapshot.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Finds which Pojo fields exists both in the new {@link PojoSerializer} as well as in the previous one
 * (represented by this snapshot), and returns an {@link IntermediateCompatibilityResult}
 * of the serializers of those preexisting fields.
 */
private static <T> IntermediateCompatibilityResult<T> getCompatibilityOfPreExistingFields(
		PojoSerializer<T> newPojoSerializer,
		LinkedOptionalMap<Field, TypeSerializerSnapshot<?>> fieldSerializerSnapshots) {

	// the present entries dictates the preexisting fields, because removed fields would be
	// represented as absent keys in the optional map.
	final Set<LinkedOptionalMap.KeyValue<Field, TypeSerializerSnapshot<?>>> presentFieldSnapshots =
		fieldSerializerSnapshots.getPresentEntries();

	final ArrayList<TypeSerializerSnapshot<?>> associatedFieldSerializerSnapshots = new ArrayList<>(presentFieldSnapshots.size());
	final ArrayList<TypeSerializer<?>> associatedNewFieldSerializers = new ArrayList<>(presentFieldSnapshots.size());

	final Map<Field, TypeSerializer<?>> newFieldSerializersIndex = buildNewFieldSerializersIndex(newPojoSerializer);
	for (LinkedOptionalMap.KeyValue<Field, TypeSerializerSnapshot<?>> presentFieldEntry : presentFieldSnapshots) {
		TypeSerializer<?> associatedNewFieldSerializer = newFieldSerializersIndex.get(presentFieldEntry.getKey());
		checkState(
			associatedNewFieldSerializer != null,
			"a present field should have its associated new field serializer available.");

		associatedFieldSerializerSnapshots.add(presentFieldEntry.getValue());
		associatedNewFieldSerializers.add(associatedNewFieldSerializer);
	}

	return CompositeTypeSerializerUtil.constructIntermediateCompatibilityResult(
		associatedNewFieldSerializers.toArray(new TypeSerializer<?>[associatedNewFieldSerializers.size()]),
		associatedFieldSerializerSnapshots.toArray(new TypeSerializerSnapshot<?>[associatedFieldSerializerSnapshots.size()]));
}
 
Example #16
Source File: PojoSerializerSnapshot.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Constructor for backwards compatibility paths with the {@link PojoSerializer.PojoSerializerConfigSnapshot}.
 * This is used in {@link PojoSerializer.PojoSerializerConfigSnapshot#resolveSchemaCompatibility(TypeSerializer)}
 * to delegate the compatibility check to this snapshot class.
 */
PojoSerializerSnapshot(
		Class<T> pojoClass,
		Field[] fields,
		TypeSerializerSnapshot<?>[] existingFieldSerializerSnapshots,
		LinkedHashMap<Class<?>, TypeSerializerSnapshot<?>> existingRegisteredSubclassSerializerSnapshots,
		Map<Class<?>, TypeSerializerSnapshot<?>> existingNonRegisteredSubclassSerializerSnapshots) {

	this.snapshotData = PojoSerializerSnapshotData.createFrom(
		pojoClass,
		fields,
		existingFieldSerializerSnapshots,
		existingRegisteredSubclassSerializerSnapshots,
		existingNonRegisteredSubclassSerializerSnapshots);
}
 
Example #17
Source File: PojoSerializerSnapshot.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Checks whether the new {@link PojoSerializer} has a different subclass registration order
 * compared to the previous one.
 */
private static boolean newPojoHasDifferentSubclassRegistrationOrder(
		LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> registeredSubclassSerializerSnapshots,
		PojoSerializer<?> newPojoSerializer) {
	Set<Class<?>> previousRegistrationOrder = registeredSubclassSerializerSnapshots.unwrapOptionals().keySet();
	Set<Class<?>> newRegistrationOrder = newPojoSerializer.getRegisteredClasses().keySet();
	return !isPreviousRegistrationPrefixOfNewRegistration(previousRegistrationOrder, newRegistrationOrder);
}
 
Example #18
Source File: MapViewSerializer.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * We need to override this as a {@link LegacySerializerSnapshotTransformer}
 * because in Flink 1.6.x and below, this serializer was incorrectly returning
 * directly the snapshot of the nested map serializer as its own snapshot.
 *
 * <p>This method transforms the incorrect map serializer snapshot
 * to be a proper {@link MapViewSerializerSnapshot}.
 */
@Override
public <U> TypeSerializerSnapshot<MapView<K, V>> transformLegacySerializerSnapshot(
		TypeSerializerSnapshot<U> legacySnapshot) {
	if (legacySnapshot instanceof MapViewSerializerSnapshot) {
		return (TypeSerializerSnapshot<MapView<K, V>>) legacySnapshot;
	} else if (legacySnapshot instanceof MapSerializerConfigSnapshot) {
		// first, transform the incorrect map serializer's snapshot
		// into a proper ListSerializerSnapshot
		MapSerializerSnapshot<K, V> transformedNestedMapSerializerSnapshot = new MapSerializerSnapshot<>();
		MapSerializerConfigSnapshot<K, V> snapshot = (MapSerializerConfigSnapshot<K, V>) legacySnapshot;
		CompositeTypeSerializerUtil.setNestedSerializersSnapshots(
				transformedNestedMapSerializerSnapshot,
				snapshot.getNestedSerializersAndConfigs().get(0).f1,
				snapshot.getNestedSerializersAndConfigs().get(1).f1
		);

		// then, wrap the transformed MapSerializerSnapshot
		// as a nested snapshot in the final resulting MapViewSerializerSnapshot
		MapViewSerializerSnapshot<K, V> transformedMapViewSerializerSnapshot = new MapViewSerializerSnapshot<>();
		CompositeTypeSerializerUtil.setNestedSerializersSnapshots(
				transformedMapViewSerializerSnapshot,
				transformedNestedMapSerializerSnapshot
		);

		return transformedMapViewSerializerSnapshot;
	} else {
		throw new UnsupportedOperationException(
				legacySnapshot.getClass().getCanonicalName() + " is not supported.");
	}
}
 
Example #19
Source File: PojoSerializerSnapshotData.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a {@link PojoSerializerSnapshotData} from configuration of a {@link PojoSerializer}.
 *
 * <p>This factory method is meant to be used in regular write paths, i.e. when taking a snapshot
 * of the {@link PojoSerializer}. All registered subclass classes, and non-registered
 * subclass classes are all present. Some POJO fields may be absent, if the originating
 * {@link PojoSerializer} was a restored one with already missing fields, and was never replaced
 * by a new {@link PojoSerializer} (i.e. because the serialized old data was never accessed).
 */
static <T> PojoSerializerSnapshotData<T> createFrom(
		Class<T> pojoClass,
		Field[] fields,
		TypeSerializer<?>[] fieldSerializers,
		LinkedHashMap<Class<?>, TypeSerializer<?>> registeredSubclassSerializers,
		Map<Class<?>, TypeSerializer<?>> nonRegisteredSubclassSerializers) {

	final LinkedOptionalMap<Field, TypeSerializerSnapshot<?>> fieldSerializerSnapshots = new LinkedOptionalMap<>(fields.length);

	for (int i = 0; i < fields.length; i++) {
		Field field = fields[i];
		String fieldName = (field == null) ? getDummyNameForMissingField(i) : field.getName();
		fieldSerializerSnapshots.put(fieldName, field, TypeSerializerUtils.snapshotBackwardsCompatible(fieldSerializers[i]));
	}

	LinkedHashMap<Class<?>, TypeSerializerSnapshot<?>> registeredSubclassSerializerSnapshots = new LinkedHashMap<>(registeredSubclassSerializers.size());
	registeredSubclassSerializers.forEach((k, v) -> registeredSubclassSerializerSnapshots.put(k, TypeSerializerUtils.snapshotBackwardsCompatible(v)));

	Map<Class<?>, TypeSerializerSnapshot<?>> nonRegisteredSubclassSerializerSnapshots = new HashMap<>(nonRegisteredSubclassSerializers.size());
	nonRegisteredSubclassSerializers.forEach((k, v) -> nonRegisteredSubclassSerializerSnapshots.put(k, TypeSerializerUtils.snapshotBackwardsCompatible(v)));

	return new PojoSerializerSnapshotData<>(
		pojoClass,
		fieldSerializerSnapshots,
		optionalMapOf(registeredSubclassSerializerSnapshots, Class::getName),
		optionalMapOf(nonRegisteredSubclassSerializerSnapshots, Class::getName));
}
 
Example #20
Source File: AvroSerializerSnapshotTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings({"unchecked", "rawtypes"})
public void changingFromGenericToSpecificWithCompatibleSchemaShouldResultInCompatibleSerializers() {
	// starting with a generic serializer
	AvroSerializer<Object> generic = new AvroSerializer(GenericRecord.class, User.SCHEMA$);
	TypeSerializerSnapshot<Object> genericSnapshot = generic.snapshotConfiguration();

	// then upgrading to a specific serializer
	AvroSerializer<Object> specificSerializer = new AvroSerializer(User.class);
	specificSerializer.snapshotConfiguration();

	assertThat(genericSnapshot.resolveSchemaCompatibility(specificSerializer), isCompatibleAsIs());
}
 
Example #21
Source File: AvroSerializerSnapshotTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void validSchemaEvaluationShouldResultInCRequiresMigration() {
	final AvroSerializer<GenericRecord> originalSerializer = new AvroSerializer<>(GenericRecord.class, FIRST_NAME);
	final AvroSerializer<GenericRecord> newSerializer = new AvroSerializer<>(GenericRecord.class, FIRST_REQUIRED_LAST_OPTIONAL);

	TypeSerializerSnapshot<GenericRecord> originalSnapshot = originalSerializer.snapshotConfiguration();

	assertThat(originalSnapshot.resolveSchemaCompatibility(newSerializer), isCompatibleAfterMigration());
}
 
Example #22
Source File: KryoSerializerCompatibilityTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Verifies that reconfiguration result is INCOMPATIBLE if data type has changed.
 */
@Test
public void testMigrationStrategyWithDifferentKryoType() throws Exception {
	KryoSerializer<TestClassA> kryoSerializerForA = new KryoSerializer<>(TestClassA.class, new ExecutionConfig());

	// snapshot configuration and serialize to bytes
	TypeSerializerSnapshot kryoSerializerConfigSnapshot = kryoSerializerForA.snapshotConfiguration();
	byte[] serializedConfig;
	try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
		TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(
			new DataOutputViewStreamWrapper(out), kryoSerializerConfigSnapshot, kryoSerializerForA);
		serializedConfig = out.toByteArray();
	}

	KryoSerializer<TestClassB> kryoSerializerForB = new KryoSerializer<>(TestClassB.class, new ExecutionConfig());

	// read configuration again from bytes
	try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
		kryoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(
			new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), kryoSerializerForB);
	}

	@SuppressWarnings("unchecked")
	TypeSerializerSchemaCompatibility<TestClassB> compatResult =
		kryoSerializerConfigSnapshot.resolveSchemaCompatibility(kryoSerializerForB);
	assertTrue(compatResult.isIncompatible());
}
 
Example #23
Source File: PojoSerializerSnapshotData.java    From flink with Apache License 2.0 5 votes vote down vote up
private PojoSerializerSnapshotData(
		Class<T> typeClass,
		LinkedOptionalMap<Field, TypeSerializerSnapshot<?>> fieldSerializerSnapshots,
		LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> registeredSubclassSerializerSnapshots,
		LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> nonRegisteredSubclassSerializerSnapshots) {

	this.pojoClass = checkNotNull(typeClass);
	this.fieldSerializerSnapshots = checkNotNull(fieldSerializerSnapshots);
	this.registeredSubclassSerializerSnapshots = checkNotNull(registeredSubclassSerializerSnapshots);
	this.nonRegisteredSubclassSerializerSnapshots = checkNotNull(nonRegisteredSubclassSerializerSnapshots);
}
 
Example #24
Source File: EnumSerializerUpgradeTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private static TypeSerializerSchemaCompatibility checkCompatibility(String enumSourceA, String enumSourceB)
	throws IOException, ClassNotFoundException {

	ClassLoader classLoader = ClassLoaderUtils.compileAndLoadJava(
		temporaryFolder.newFolder(), ENUM_NAME + ".java", enumSourceA);

	EnumSerializer enumSerializer = new EnumSerializer(classLoader.loadClass(ENUM_NAME));

	TypeSerializerSnapshot snapshot = enumSerializer.snapshotConfiguration();
	byte[] snapshotBytes;
	try (
		ByteArrayOutputStream outBuffer = new ByteArrayOutputStream();
		DataOutputViewStreamWrapper outputViewStreamWrapper = new DataOutputViewStreamWrapper(outBuffer)) {

		TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(
			outputViewStreamWrapper, snapshot, enumSerializer);
		snapshotBytes = outBuffer.toByteArray();
	}

	ClassLoader classLoader2 = ClassLoaderUtils.compileAndLoadJava(
		temporaryFolder.newFolder(), ENUM_NAME + ".java", enumSourceB);

	TypeSerializerSnapshot restoredSnapshot;
	try (
		ByteArrayInputStream inBuffer = new ByteArrayInputStream(snapshotBytes);
		DataInputViewStreamWrapper inputViewStreamWrapper = new DataInputViewStreamWrapper(inBuffer)) {

		restoredSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(
			inputViewStreamWrapper, classLoader2, enumSerializer);
	}

	EnumSerializer enumSerializer2 = new EnumSerializer(classLoader2.loadClass(ENUM_NAME));
	return restoredSnapshot.resolveSchemaCompatibility(enumSerializer2);
}
 
Example #25
Source File: RegisteredBroadcastStateBackendMetaInfo.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
public RegisteredBroadcastStateBackendMetaInfo(@Nonnull StateMetaInfoSnapshot snapshot) {
	this(
		snapshot.getName(),
		OperatorStateHandle.Mode.valueOf(
			snapshot.getOption(StateMetaInfoSnapshot.CommonOptionsKeys.OPERATOR_STATE_DISTRIBUTION_MODE)),
		StateSerializerProvider.fromPreviousSerializerSnapshot(
			(TypeSerializerSnapshot<K>) Preconditions.checkNotNull(
				snapshot.getTypeSerializerSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.KEY_SERIALIZER))),
		StateSerializerProvider.fromPreviousSerializerSnapshot(
			(TypeSerializerSnapshot<V>) Preconditions.checkNotNull(
				snapshot.getTypeSerializerSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER))));

	Preconditions.checkState(StateMetaInfoSnapshot.BackendStateType.BROADCAST == snapshot.getBackendStateType());
}
 
Example #26
Source File: PojoSerializer.java    From flink with Apache License 2.0 4 votes vote down vote up
public LinkedHashMap<Class<?>, Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> getRegisteredSubclassesToSerializerConfigSnapshots() {
	return registeredSubclassesToSerializerConfigSnapshots;
}
 
Example #27
Source File: CharSerializer.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Override
public TypeSerializerSnapshot<Character> snapshotConfiguration() {
	return new CharSerializerSnapshot();
}
 
Example #28
Source File: NFAStateSerializer.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Override
public TypeSerializerSnapshot<NFAState> snapshotConfiguration() {
	return new NFAStateSerializerSnapshot(this);
}
 
Example #29
Source File: IntValueArraySerializer.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public TypeSerializerSnapshot<IntValueArray> snapshotConfiguration() {
	return new IntValueArraySerializerSnapshot();
}
 
Example #30
Source File: SharedBufferEdge.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Override
public TypeSerializerSnapshot<SharedBufferEdge> snapshotConfiguration() {
	return new SharedBufferEdgeSerializerSnapshot(this);
}