Java Code Examples for org.apache.flink.api.common.typeutils.TypeSerializerSnapshot

The following examples show how to use org.apache.flink.api.common.typeutils.TypeSerializerSnapshot. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: PojoSerializer.java    License: Apache License 2.0 6 votes vote down vote up
public PojoSerializerConfigSnapshot(
		Class<T> pojoType,
		LinkedHashMap<String, Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> fieldToSerializerConfigSnapshot,
		LinkedHashMap<Class<?>, Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> registeredSubclassesToSerializerConfigSnapshots,
		HashMap<Class<?>, Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> nonRegisteredSubclassesToSerializerConfigSnapshots,
		boolean ignoreTypeSerializerSerialization) {

	super(pojoType);

	this.fieldToSerializerConfigSnapshot =
			Preconditions.checkNotNull(fieldToSerializerConfigSnapshot);
	this.registeredSubclassesToSerializerConfigSnapshots =
			Preconditions.checkNotNull(registeredSubclassesToSerializerConfigSnapshots);
	this.nonRegisteredSubclassesToSerializerConfigSnapshots =
			Preconditions.checkNotNull(nonRegisteredSubclassesToSerializerConfigSnapshots);

	this.ignoreTypeSerializerSerialization = ignoreTypeSerializerSerialization;
}
 
Example 2
Source Project: flink   Source File: KryoSerializerCompatibilityTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMigrationStrategyForRemovedAvroDependency() throws Exception {
	KryoSerializer<TestClass> kryoSerializerForA = new KryoSerializer<>(TestClass.class, new ExecutionConfig());

	// read configuration again from bytes
	TypeSerializerSnapshot kryoSerializerConfigSnapshot;
	try (InputStream in = getClass().getResourceAsStream("/kryo-serializer-flink1.3-snapshot")) {
		kryoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(
			new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), kryoSerializerForA);
	}

	@SuppressWarnings("unchecked")
	TypeSerializerSchemaCompatibility<TestClass> compatResult =
		kryoSerializerConfigSnapshot.resolveSchemaCompatibility(kryoSerializerForA);
	assertTrue(compatResult.isCompatibleAsIs());
}
 
Example 3
Source Project: flink   Source File: PojoSerializerSnapshot.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a reconfigured version of the {@link PojoSerializer}.
 *
 * @param originalNewPojoSerializer the original new {@link PojoSerializer} to create a reconfigured version of.
 * @param fieldSerializerCompatibility compatibility of preexisting fields' serializers.
 * @param registeredSerializerSnapshots snapshot of previous registered subclasses' serializers.
 * @param preExistingRegistrationsCompatibility compatibility of preexisting subclasses' serializers.
 * @param nonRegisteredSubclassSerializerSnapshots snapshot of previous non-registered subclasses' serializers.
 *
 * @return a reconfigured version of the original new {@link PojoSerializer}.
 */
private static <T> PojoSerializer<T> constructReconfiguredPojoSerializer(
		PojoSerializer<T> originalNewPojoSerializer,
		IntermediateCompatibilityResult<T> fieldSerializerCompatibility,
		LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> registeredSerializerSnapshots,
		IntermediateCompatibilityResult<T> preExistingRegistrationsCompatibility,
		LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> nonRegisteredSubclassSerializerSnapshots) {

	@SuppressWarnings("unchecked")
	final TypeSerializer<Object>[] reconfiguredFieldSerializers = constructReconfiguredFieldSerializers(fieldSerializerCompatibility);

	Tuple2<LinkedHashMap<Class<?>, Integer>, TypeSerializer<Object>[]> reconfiguredSubclassRegistry = constructReconfiguredSubclassRegistry(
		originalNewPojoSerializer.getBundledSubclassSerializerRegistry(),
		registeredSerializerSnapshots,
		preExistingRegistrationsCompatibility);

	return new PojoSerializer<>(
		originalNewPojoSerializer.getPojoClass(),
		originalNewPojoSerializer.getFields(),
		reconfiguredFieldSerializers,
		reconfiguredSubclassRegistry.f0,
		reconfiguredSubclassRegistry.f1,
		restoreSerializers(nonRegisteredSubclassSerializerSnapshots.unwrapOptionals()),
		originalNewPojoSerializer.getExecutionConfig());
}
 
Example 4
Source Project: flink   Source File: Lockable.java    License: Apache License 2.0 6 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public <U> TypeSerializerSnapshot<Lockable<E>> transformLegacySerializerSnapshot(TypeSerializerSnapshot<U> legacySnapshot) {
	if (legacySnapshot instanceof LockableTypeSerializerSnapshot) {
		return (TypeSerializerSnapshot<Lockable<E>>) legacySnapshot;
	}

	// In Flink 1.6, this serializer was directly returning the elementSerializer's snapshot
	// instead of wrapping it in a LockableTypeSerializer(Config)Snapshot.
	// This caused state information to be written as <LockableTypeSerializer, SomeArbitrarySerializerSnapshot>,
	// Therefore we need to preform the following transformation:
	// 	1. set the prior serializer on the legacySnapshot to be the elementSerializer
	// 	2. return a LockableTypeSerializerSnapshot that has the legacySnapshot as a nested snapshot.
	if (legacySnapshot instanceof TypeSerializerConfigSnapshot) {
		setElementSerializerAsPriorSerializer(legacySnapshot, this.elementSerializer);
	}
	LockableTypeSerializerSnapshot<E> lockableSnapshot = new LockableTypeSerializerSnapshot<>();
	CompositeTypeSerializerUtil.setNestedSerializersSnapshots(lockableSnapshot, legacySnapshot);
	return lockableSnapshot;
}
 
Example 5
Source Project: Flink-CEPplus   Source File: LegacyStateMetaInfoReaders.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
@Override
public StateMetaInfoSnapshot readStateMetaInfoSnapshot(
	@Nonnull DataInputView in, @Nonnull ClassLoader userCodeClassLoader) throws IOException {

	final StateDescriptor.Type stateDescType = StateDescriptor.Type.values()[in.readInt()];
	final String stateName = in.readUTF();
	List<Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> serializersAndConfigs =
		TypeSerializerSerializationUtil.readSerializersAndConfigsWithResilience(in, userCodeClassLoader);

	Map<String, String> optionsMap = Collections.singletonMap(
		StateMetaInfoSnapshot.CommonOptionsKeys.KEYED_STATE_TYPE.toString(),
		stateDescType.toString());

	Map<String, TypeSerializerSnapshot<?>> serializerConfigSnapshotMap = new HashMap<>(2);
	serializerConfigSnapshotMap.put(StateMetaInfoSnapshot.CommonSerializerKeys.NAMESPACE_SERIALIZER.toString(),
		serializersAndConfigs.get(0).f1);
	serializerConfigSnapshotMap.put(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER.toString(),
		serializersAndConfigs.get(1).f1);

	return new StateMetaInfoSnapshot(
		stateName,
		StateMetaInfoSnapshot.BackendStateType.KEY_VALUE,
		optionsMap,
		serializerConfigSnapshotMap);
}
 
Example 6
Source Project: flink   Source File: EitherSerializerConfigSnapshot.java    License: Apache License 2.0 6 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public TypeSerializerSchemaCompatibility<Either<L, R>> resolveSchemaCompatibility(
		TypeSerializer<Either<L, R>> newSerializer) {

	// this class was shared between the Java Either Serializer and the
	// Scala Either serializer
	if (newSerializer.getClass() == EitherSerializer.class) {
		List<Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> nestedSerializersAndConfigs = getNestedSerializersAndConfigs();
		return CompositeTypeSerializerUtil.delegateCompatibilityCheckToNewSnapshot(
			newSerializer,
			new JavaEitherSerializerSnapshot<>(),
			nestedSerializersAndConfigs.get(0).f1,
			nestedSerializersAndConfigs.get(1).f1);
	}
	else {
		// fall back to the backwards compatibility path
		return super.resolveSchemaCompatibility(newSerializer);
	}
}
 
Example 7
Source Project: flink   Source File: PojoSerializerSnapshot.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Finds which registered subclasses exists both in the new {@link PojoSerializer} as well as in the previous one
 * (represented by this snapshot), and returns an {@link IntermediateCompatibilityResult}
 * of the serializers of this preexisting registered subclasses.
 */
private static <T> IntermediateCompatibilityResult<T> getCompatibilityOfPreExistingRegisteredSubclasses(
		PojoSerializer<T> newPojoSerializer,
		LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> registeredSubclassSerializerSnapshots) {

	final LinkedHashMap<Class<?>, TypeSerializerSnapshot<?>> unwrappedSerializerSnapshots = registeredSubclassSerializerSnapshots.unwrapOptionals();

	final ArrayList<TypeSerializerSnapshot<?>> associatedSubclassSerializerSnapshots = new ArrayList<>();
	final ArrayList<TypeSerializer<?>> associatedNewSubclassSerializers = new ArrayList<>();

	final LinkedHashMap<Class<?>, TypeSerializer<?>> newSubclassSerializerRegistry = newPojoSerializer.getBundledSubclassSerializerRegistry();

	for (Map.Entry<Class<?>, TypeSerializerSnapshot<?>> entry : unwrappedSerializerSnapshots.entrySet()) {
		TypeSerializer<?> newRegisteredSerializer = newSubclassSerializerRegistry.get(entry.getKey());
		if (newRegisteredSerializer != null) {
			associatedSubclassSerializerSnapshots.add(entry.getValue());
			associatedNewSubclassSerializers.add(newRegisteredSerializer);
		}
	}

	return CompositeTypeSerializerUtil.constructIntermediateCompatibilityResult(
		associatedNewSubclassSerializers.toArray(new TypeSerializer<?>[associatedNewSubclassSerializers.size()]),
		associatedSubclassSerializerSnapshots.toArray(new TypeSerializerSnapshot<?>[associatedSubclassSerializerSnapshots.size()]));
}
 
Example 8
Source Project: Flink-CEPplus   Source File: PojoSerializerSnapshot.java    License: Apache License 2.0 6 votes vote down vote up
private static Tuple2<LinkedHashMap<Class<?>, Integer>, TypeSerializer<Object>[]> constructReconfiguredSubclassRegistry(
		LinkedHashMap<Class<?>, TypeSerializer<?>> newSubclassRegistrations,
		LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> registeredSerializerSnapshots,
		IntermediateCompatibilityResult<?> preExistingRegistrationsCompatibility) {

	checkArgument(!preExistingRegistrationsCompatibility.isIncompatible() && !preExistingRegistrationsCompatibility.isCompatibleAfterMigration());

	LinkedHashMap<Class<?>, TypeSerializer<?>> reconfiguredSubclassSerializerRegistry =
		restoreSerializers(registeredSerializerSnapshots.unwrapOptionals());

	Iterator<TypeSerializer<?>> serializersForPreexistingRegistrations =
		Arrays.asList(preExistingRegistrationsCompatibility.getNestedSerializers()).iterator();

	for (Map.Entry<Class<?>, TypeSerializer<?>> registration : newSubclassRegistrations.entrySet()) {
		// new registrations should simply be appended to the subclass serializer registry with their new serializers;
		// preexisting registrations should use the compatibility-checked serializer
		TypeSerializer<?> newRegistration = (reconfiguredSubclassSerializerRegistry.containsKey(registration.getKey()))
			? serializersForPreexistingRegistrations.next()
			: registration.getValue();
		reconfiguredSubclassSerializerRegistry.put(registration.getKey(), newRegistration);
	}

	return decomposeSubclassSerializerRegistry(reconfiguredSubclassSerializerRegistry);
}
 
Example 9
Source Project: flink   Source File: LegacyStateMetaInfoReaders.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
@Override
public StateMetaInfoSnapshot readStateMetaInfoSnapshot(
	@Nonnull DataInputView in, @Nonnull ClassLoader userCodeClassLoader) throws IOException {

	final StateDescriptor.Type stateDescType = StateDescriptor.Type.values()[in.readInt()];
	final String stateName = in.readUTF();
	List<Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> serializersAndConfigs =
		TypeSerializerSerializationUtil.readSerializersAndConfigsWithResilience(in, userCodeClassLoader);

	Map<String, String> optionsMap = Collections.singletonMap(
		StateMetaInfoSnapshot.CommonOptionsKeys.KEYED_STATE_TYPE.toString(),
		stateDescType.toString());

	Map<String, TypeSerializerSnapshot<?>> serializerConfigSnapshotMap = new HashMap<>(2);
	serializerConfigSnapshotMap.put(StateMetaInfoSnapshot.CommonSerializerKeys.NAMESPACE_SERIALIZER.toString(),
		serializersAndConfigs.get(0).f1);
	serializerConfigSnapshotMap.put(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER.toString(),
		serializersAndConfigs.get(1).f1);

	return new StateMetaInfoSnapshot(
		stateName,
		StateMetaInfoSnapshot.BackendStateType.KEY_VALUE,
		optionsMap,
		serializerConfigSnapshotMap);
}
 
Example 10
Source Project: flink   Source File: PojoSerializerSnapshotTest.java    License: Apache License 2.0 6 votes vote down vote up
private static PojoSerializerSnapshot<TestPojo> buildTestSnapshot(List<TestPojoField> fieldsToContainInSnapshot) {

		int numFields = fieldsToContainInSnapshot.size();
		ArrayList<Field> fields = new ArrayList<>(numFields);
		ArrayList<TypeSerializerSnapshot<?>> fieldSerializerSnapshots = new ArrayList<>(numFields);
		fieldsToContainInSnapshot.forEach(testPojoField -> {
			fields.add(testPojoField.field);
			fieldSerializerSnapshots.add(testPojoField.serializerSnapshot);
		});

		return new PojoSerializerSnapshot<>(
			TestPojo.class,
			fields.toArray(new Field[numFields]),
			fieldSerializerSnapshots.toArray(new TypeSerializerSnapshot[numFields]),
			new LinkedHashMap<>(),
			new LinkedHashMap<>());
	}
 
Example 11
Source Project: flink   Source File: EnumSerializerUpgradeTest.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private static TypeSerializerSchemaCompatibility checkCompatibility(String enumSourceA, String enumSourceB)
	throws IOException, ClassNotFoundException {

	ClassLoader classLoader = ClassLoaderUtils.compileAndLoadJava(
		temporaryFolder.newFolder(), ENUM_NAME + ".java", enumSourceA);

	EnumSerializer enumSerializer = new EnumSerializer(classLoader.loadClass(ENUM_NAME));

	TypeSerializerSnapshot snapshot = enumSerializer.snapshotConfiguration();
	byte[] snapshotBytes;
	try (
		ByteArrayOutputStream outBuffer = new ByteArrayOutputStream();
		DataOutputViewStreamWrapper outputViewStreamWrapper = new DataOutputViewStreamWrapper(outBuffer)) {

		TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(
			outputViewStreamWrapper, snapshot, enumSerializer);
		snapshotBytes = outBuffer.toByteArray();
	}

	ClassLoader classLoader2 = ClassLoaderUtils.compileAndLoadJava(
		temporaryFolder.newFolder(), ENUM_NAME + ".java", enumSourceB);

	TypeSerializerSnapshot restoredSnapshot;
	try (
		ByteArrayInputStream inBuffer = new ByteArrayInputStream(snapshotBytes);
		DataInputViewStreamWrapper inputViewStreamWrapper = new DataInputViewStreamWrapper(inBuffer)) {

		restoredSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(
			inputViewStreamWrapper, classLoader2, enumSerializer);
	}

	EnumSerializer enumSerializer2 = new EnumSerializer(classLoader2.loadClass(ENUM_NAME));
	return restoredSnapshot.resolveSchemaCompatibility(enumSerializer2);
}
 
Example 12
Source Project: flink   Source File: KryoSerializer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public TypeSerializerSnapshot<T> snapshotConfiguration() {
	return new KryoSerializerSnapshot<>(
		type,
		defaultSerializers,
		defaultSerializerClasses,
		kryoRegistrations);
}
 
Example 13
Source Project: flink   Source File: MapViewSerializer.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * We need to override this as a {@link LegacySerializerSnapshotTransformer}
 * because in Flink 1.6.x and below, this serializer was incorrectly returning
 * directly the snapshot of the nested map serializer as its own snapshot.
 *
 * <p>This method transforms the incorrect map serializer snapshot
 * to be a proper {@link MapViewSerializerSnapshot}.
 */
@Override
public <U> TypeSerializerSnapshot<MapView<K, V>> transformLegacySerializerSnapshot(
		TypeSerializerSnapshot<U> legacySnapshot) {
	if (legacySnapshot instanceof MapViewSerializerSnapshot) {
		return (TypeSerializerSnapshot<MapView<K, V>>) legacySnapshot;
	} else if (legacySnapshot instanceof MapSerializerConfigSnapshot) {
		// first, transform the incorrect map serializer's snapshot
		// into a proper ListSerializerSnapshot
		MapSerializerSnapshot<K, V> transformedNestedMapSerializerSnapshot = new MapSerializerSnapshot<>();
		MapSerializerConfigSnapshot<K, V> snapshot = (MapSerializerConfigSnapshot<K, V>) legacySnapshot;
		CompositeTypeSerializerUtil.setNestedSerializersSnapshots(
				transformedNestedMapSerializerSnapshot,
				snapshot.getNestedSerializersAndConfigs().get(0).f1,
				snapshot.getNestedSerializersAndConfigs().get(1).f1
		);

		// then, wrap the transformed MapSerializerSnapshot
		// as a nested snapshot in the final resulting MapViewSerializerSnapshot
		MapViewSerializerSnapshot<K, V> transformedMapViewSerializerSnapshot = new MapViewSerializerSnapshot<>();
		CompositeTypeSerializerUtil.setNestedSerializersSnapshots(
				transformedMapViewSerializerSnapshot,
				transformedNestedMapSerializerSnapshot
		);

		return transformedMapViewSerializerSnapshot;
	} else {
		throw new UnsupportedOperationException(
				legacySnapshot.getClass().getCanonicalName() + " is not supported.");
	}
}
 
Example 14
Source Project: Flink-CEPplus   Source File: PojoSerializerSnapshotData.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a {@link PojoSerializerSnapshotData} from configuration of a {@link PojoSerializer}.
 *
 * <p>This factory method is meant to be used in regular write paths, i.e. when taking a snapshot
 * of the {@link PojoSerializer}. All registered subclass classes, and non-registered
 * subclass classes are all present. Some POJO fields may be absent, if the originating
 * {@link PojoSerializer} was a restored one with already missing fields, and was never replaced
 * by a new {@link PojoSerializer} (i.e. because the serialized old data was never accessed).
 */
static <T> PojoSerializerSnapshotData<T> createFrom(
		Class<T> pojoClass,
		Field[] fields,
		TypeSerializer<?>[] fieldSerializers,
		LinkedHashMap<Class<?>, TypeSerializer<?>> registeredSubclassSerializers,
		Map<Class<?>, TypeSerializer<?>> nonRegisteredSubclassSerializers) {

	final LinkedOptionalMap<Field, TypeSerializerSnapshot<?>> fieldSerializerSnapshots = new LinkedOptionalMap<>(fields.length);

	for (int i = 0; i < fields.length; i++) {
		Field field = fields[i];
		String fieldName = (field == null) ? getDummyNameForMissingField(i) : field.getName();
		fieldSerializerSnapshots.put(fieldName, field, TypeSerializerUtils.snapshotBackwardsCompatible(fieldSerializers[i]));
	}

	LinkedHashMap<Class<?>, TypeSerializerSnapshot<?>> registeredSubclassSerializerSnapshots = new LinkedHashMap<>(registeredSubclassSerializers.size());
	registeredSubclassSerializers.forEach((k, v) -> registeredSubclassSerializerSnapshots.put(k, TypeSerializerUtils.snapshotBackwardsCompatible(v)));

	Map<Class<?>, TypeSerializerSnapshot<?>> nonRegisteredSubclassSerializerSnapshots = new HashMap<>(nonRegisteredSubclassSerializers.size());
	nonRegisteredSubclassSerializers.forEach((k, v) -> nonRegisteredSubclassSerializerSnapshots.put(k, TypeSerializerUtils.snapshotBackwardsCompatible(v)));

	return new PojoSerializerSnapshotData<>(
		pojoClass,
		fieldSerializerSnapshots,
		optionalMapOf(registeredSubclassSerializerSnapshots, Class::getName),
		optionalMapOf(nonRegisteredSubclassSerializerSnapshots, Class::getName));
}
 
Example 15
Source Project: flink   Source File: PojoSerializerSnapshot.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Checks whether the new {@link PojoSerializer} has a different subclass registration order
 * compared to the previous one.
 */
private static boolean newPojoHasDifferentSubclassRegistrationOrder(
		LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> registeredSubclassSerializerSnapshots,
		PojoSerializer<?> newPojoSerializer) {
	Set<Class<?>> previousRegistrationOrder = registeredSubclassSerializerSnapshots.unwrapOptionals().keySet();
	Set<Class<?>> newRegistrationOrder = newPojoSerializer.getRegisteredClasses().keySet();
	return !isPreviousRegistrationPrefixOfNewRegistration(previousRegistrationOrder, newRegistrationOrder);
}
 
Example 16
Source Project: flink   Source File: AvroSerializerSnapshotTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void validSchemaEvaluationShouldResultInCRequiresMigration() {
	final AvroSerializer<GenericRecord> originalSerializer = new AvroSerializer<>(GenericRecord.class, FIRST_NAME);
	final AvroSerializer<GenericRecord> newSerializer = new AvroSerializer<>(GenericRecord.class, FIRST_REQUIRED_LAST_OPTIONAL);

	TypeSerializerSnapshot<GenericRecord> originalSnapshot = originalSerializer.snapshotConfiguration();

	assertThat(originalSnapshot.resolveSchemaCompatibility(newSerializer), isCompatibleAfterMigration());
}
 
Example 17
@SuppressWarnings("unchecked")
public RegisteredBroadcastStateBackendMetaInfo(@Nonnull StateMetaInfoSnapshot snapshot) {
	this(
		snapshot.getName(),
		OperatorStateHandle.Mode.valueOf(
			snapshot.getOption(StateMetaInfoSnapshot.CommonOptionsKeys.OPERATOR_STATE_DISTRIBUTION_MODE)),
		StateSerializerProvider.fromPreviousSerializerSnapshot(
			(TypeSerializerSnapshot<K>) Preconditions.checkNotNull(
				snapshot.getTypeSerializerSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.KEY_SERIALIZER))),
		StateSerializerProvider.fromPreviousSerializerSnapshot(
			(TypeSerializerSnapshot<V>) Preconditions.checkNotNull(
				snapshot.getTypeSerializerSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER))));

	Preconditions.checkState(StateMetaInfoSnapshot.BackendStateType.BROADCAST == snapshot.getBackendStateType());
}
 
Example 18
Source Project: Flink-CEPplus   Source File: EnumSerializerUpgradeTest.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private static TypeSerializerSchemaCompatibility checkCompatibility(String enumSourceA, String enumSourceB)
	throws IOException, ClassNotFoundException {

	ClassLoader classLoader = ClassLoaderUtils.compileAndLoadJava(
		temporaryFolder.newFolder(), ENUM_NAME + ".java", enumSourceA);

	EnumSerializer enumSerializer = new EnumSerializer(classLoader.loadClass(ENUM_NAME));

	TypeSerializerSnapshot snapshot = enumSerializer.snapshotConfiguration();
	byte[] snapshotBytes;
	try (
		ByteArrayOutputStream outBuffer = new ByteArrayOutputStream();
		DataOutputViewStreamWrapper outputViewStreamWrapper = new DataOutputViewStreamWrapper(outBuffer)) {

		TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(
			outputViewStreamWrapper, snapshot, enumSerializer);
		snapshotBytes = outBuffer.toByteArray();
	}

	ClassLoader classLoader2 = ClassLoaderUtils.compileAndLoadJava(
		temporaryFolder.newFolder(), ENUM_NAME + ".java", enumSourceB);

	TypeSerializerSnapshot restoredSnapshot;
	try (
		ByteArrayInputStream inBuffer = new ByteArrayInputStream(snapshotBytes);
		DataInputViewStreamWrapper inputViewStreamWrapper = new DataInputViewStreamWrapper(inBuffer)) {

		restoredSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(
			inputViewStreamWrapper, classLoader2, enumSerializer);
	}

	EnumSerializer enumSerializer2 = new EnumSerializer(classLoader2.loadClass(ENUM_NAME));
	return restoredSnapshot.resolveSchemaCompatibility(enumSerializer2);
}
 
Example 19
Source Project: flink   Source File: PojoSerializerSnapshotData.java    License: Apache License 2.0 5 votes vote down vote up
private PojoSerializerSnapshotData(
		Class<T> typeClass,
		LinkedOptionalMap<Field, TypeSerializerSnapshot<?>> fieldSerializerSnapshots,
		LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> registeredSubclassSerializerSnapshots,
		LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> nonRegisteredSubclassSerializerSnapshots) {

	this.pojoClass = checkNotNull(typeClass);
	this.fieldSerializerSnapshots = checkNotNull(fieldSerializerSnapshots);
	this.registeredSubclassSerializerSnapshots = checkNotNull(registeredSubclassSerializerSnapshots);
	this.nonRegisteredSubclassSerializerSnapshots = checkNotNull(nonRegisteredSubclassSerializerSnapshots);
}
 
Example 20
Source Project: flink   Source File: KryoSerializerCompatibilityTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Verifies that reconfiguration result is INCOMPATIBLE if data type has changed.
 */
@Test
public void testMigrationStrategyWithDifferentKryoType() throws Exception {
	KryoSerializer<TestClassA> kryoSerializerForA = new KryoSerializer<>(TestClassA.class, new ExecutionConfig());

	// snapshot configuration and serialize to bytes
	TypeSerializerSnapshot kryoSerializerConfigSnapshot = kryoSerializerForA.snapshotConfiguration();
	byte[] serializedConfig;
	try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
		TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(
			new DataOutputViewStreamWrapper(out), kryoSerializerConfigSnapshot, kryoSerializerForA);
		serializedConfig = out.toByteArray();
	}

	KryoSerializer<TestClassB> kryoSerializerForB = new KryoSerializer<>(TestClassB.class, new ExecutionConfig());

	// read configuration again from bytes
	try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
		kryoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(
			new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), kryoSerializerForB);
	}

	@SuppressWarnings("unchecked")
	TypeSerializerSchemaCompatibility<TestClassB> compatResult =
		kryoSerializerConfigSnapshot.resolveSchemaCompatibility(kryoSerializerForB);
	assertTrue(compatResult.isIncompatible());
}
 
Example 21
Source Project: flink   Source File: PojoSerializerSnapshot.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Constructor for backwards compatibility paths with the {@link PojoSerializer.PojoSerializerConfigSnapshot}.
 * This is used in {@link PojoSerializer.PojoSerializerConfigSnapshot#resolveSchemaCompatibility(TypeSerializer)}
 * to delegate the compatibility check to this snapshot class.
 */
PojoSerializerSnapshot(
		Class<T> pojoClass,
		Field[] fields,
		TypeSerializerSnapshot<?>[] existingFieldSerializerSnapshots,
		LinkedHashMap<Class<?>, TypeSerializerSnapshot<?>> existingRegisteredSubclassSerializerSnapshots,
		Map<Class<?>, TypeSerializerSnapshot<?>> existingNonRegisteredSubclassSerializerSnapshots) {

	this.snapshotData = PojoSerializerSnapshotData.createFrom(
		pojoClass,
		fields,
		existingFieldSerializerSnapshots,
		existingRegisteredSubclassSerializerSnapshots,
		existingNonRegisteredSubclassSerializerSnapshots);
}
 
Example 22
Source Project: flink   Source File: PojoSerializerSnapshot.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Finds which Pojo fields exists both in the new {@link PojoSerializer} as well as in the previous one
 * (represented by this snapshot), and returns an {@link IntermediateCompatibilityResult}
 * of the serializers of those preexisting fields.
 */
private static <T> IntermediateCompatibilityResult<T> getCompatibilityOfPreExistingFields(
		PojoSerializer<T> newPojoSerializer,
		LinkedOptionalMap<Field, TypeSerializerSnapshot<?>> fieldSerializerSnapshots) {

	// the present entries dictates the preexisting fields, because removed fields would be
	// represented as absent keys in the optional map.
	final Set<LinkedOptionalMap.KeyValue<Field, TypeSerializerSnapshot<?>>> presentFieldSnapshots =
		fieldSerializerSnapshots.getPresentEntries();

	final ArrayList<TypeSerializerSnapshot<?>> associatedFieldSerializerSnapshots = new ArrayList<>(presentFieldSnapshots.size());
	final ArrayList<TypeSerializer<?>> associatedNewFieldSerializers = new ArrayList<>(presentFieldSnapshots.size());

	final Map<Field, TypeSerializer<?>> newFieldSerializersIndex = buildNewFieldSerializersIndex(newPojoSerializer);
	for (LinkedOptionalMap.KeyValue<Field, TypeSerializerSnapshot<?>> presentFieldEntry : presentFieldSnapshots) {
		TypeSerializer<?> associatedNewFieldSerializer = newFieldSerializersIndex.get(presentFieldEntry.getKey());
		checkState(
			associatedNewFieldSerializer != null,
			"a present field should have its associated new field serializer available.");

		associatedFieldSerializerSnapshots.add(presentFieldEntry.getValue());
		associatedNewFieldSerializers.add(associatedNewFieldSerializer);
	}

	return CompositeTypeSerializerUtil.constructIntermediateCompatibilityResult(
		associatedNewFieldSerializers.toArray(new TypeSerializer<?>[associatedNewFieldSerializers.size()]),
		associatedFieldSerializerSnapshots.toArray(new TypeSerializerSnapshot<?>[associatedFieldSerializerSnapshots.size()]));
}
 
Example 23
@SuppressWarnings("unchecked")
public RegisteredPriorityQueueStateBackendMetaInfo(StateMetaInfoSnapshot snapshot) {
	this(
		snapshot.getName(),
		StateSerializerProvider.fromPreviousSerializerSnapshot(
			(TypeSerializerSnapshot<T>) Preconditions.checkNotNull(
				snapshot.getTypeSerializerSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER))));

	Preconditions.checkState(StateMetaInfoSnapshot.BackendStateType.PRIORITY_QUEUE == snapshot.getBackendStateType());
}
 
Example 24
Source Project: flink   Source File: AvroSerializerSnapshotTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void nonValidSchemaEvaluationShouldResultInCompatibleSerializers() {
	final AvroSerializer<GenericRecord> originalSerializer = new AvroSerializer<>(GenericRecord.class, FIRST_REQUIRED_LAST_OPTIONAL);
	final AvroSerializer<GenericRecord> newSerializer = new AvroSerializer<>(GenericRecord.class, BOTH_REQUIRED);

	TypeSerializerSnapshot<GenericRecord> originalSnapshot = originalSerializer.snapshotConfiguration();

	assertThat(originalSnapshot.resolveSchemaCompatibility(newSerializer), isIncompatible());
}
 
Example 25
Source Project: flink   Source File: AvroSerializerSnapshotTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings({"unchecked", "rawtypes"})
public void changingFromGenericToSpecificWithCompatibleSchemaShouldResultInCompatibleSerializers() {
	// starting with a generic serializer
	AvroSerializer<Object> generic = new AvroSerializer(GenericRecord.class, User.SCHEMA$);
	TypeSerializerSnapshot<Object> genericSnapshot = generic.snapshotConfiguration();

	// then upgrading to a specific serializer
	AvroSerializer<Object> specificSerializer = new AvroSerializer(User.class);
	specificSerializer.snapshotConfiguration();

	assertThat(genericSnapshot.resolveSchemaCompatibility(specificSerializer), isCompatibleAsIs());
}
 
Example 26
Source Project: Flink-CEPplus   Source File: FlinkKafkaProducer.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public TypeSerializerSnapshot<FlinkKafkaProducer.KafkaTransactionState> snapshotConfiguration() {
	return new TransactionStateSerializerSnapshot();
}
 
Example 27
Source Project: flink   Source File: InternalTimersSnapshot.java    License: Apache License 2.0 4 votes vote down vote up
public void setKeySerializerSnapshot(TypeSerializerSnapshot<K> keySerializerConfigSnapshot) {
	this.keySerializerSnapshot = keySerializerConfigSnapshot;
}
 
Example 28
Source Project: flink   Source File: MapSerializer.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public TypeSerializerSnapshot<Map<K, V>> snapshotConfiguration() {
	return new MapSerializerSnapshot<>(this);
}
 
Example 29
@Override
public TypeSerializerSnapshot<Integer> snapshotConfiguration() {
	return new SchemaCompatibilityTestingSnapshot(tokenForEqualityChecks, resolver);
}
 
Example 30
Source Project: flink   Source File: BooleanValueSerializer.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public TypeSerializerSnapshot<BooleanValue> snapshotConfiguration() {
	return new BooleanValueSerializerSnapshot();
}