org.apache.flink.api.common.typeutils.TypeSerializer Java Examples

@Test
public void testVectorsSerDeser() throws IOException {
	// Prepare data
	SparseVector sparseVector = new SparseVector(10, new HashMap<Integer, Double>() {{
		ThreadLocalRandom rand = ThreadLocalRandom.current();
		for (int i = 0; i < 10; i += 2) {
			this.put(i, rand.nextDouble());
		}
	}});
	DenseVector denseVector = DenseVector.rand(10);

	// Prepare serializer
	ExecutionConfig config = new ExecutionConfig();
	TypeSerializer<Vector> vecSer = VectorTypes.VECTOR.createSerializer(config);
	TypeSerializer<SparseVector> sparseSer = VectorTypes.SPARSE_VECTOR.createSerializer(config);
	TypeSerializer<DenseVector> denseSer = VectorTypes.DENSE_VECTOR.createSerializer(config);

	// Do tests.
	doVectorSerDeserTest(vecSer, sparseVector);
	doVectorSerDeserTest(vecSer, denseVector);
	doVectorSerDeserTest(sparseSer, sparseVector);
	doVectorSerDeserTest(denseSer, denseVector);
}
 

private <K, V> void deserializeBroadcastStateValues(
	final BackendWritableBroadcastState<K, V> broadcastStateForName,
	final FSDataInputStream in,
	final OperatorStateHandle.StateMetaInfo metaInfo) throws Exception {

	if (metaInfo != null) {
		long[] offsets = metaInfo.getOffsets();
		if (offsets != null) {

			TypeSerializer<K> keySerializer = broadcastStateForName.getStateMetaInfo().getKeySerializer();
			TypeSerializer<V> valueSerializer = broadcastStateForName.getStateMetaInfo().getValueSerializer();

			in.seek(offsets[0]);

			DataInputView div = new DataInputViewStreamWrapper(in);
			int size = div.readInt();
			for (int i = 0; i < size; i++) {
				broadcastStateForName.put(keySerializer.deserialize(div), valueSerializer.deserialize(div));
			}
		}
	}
}
 

@Test
public void testKeyedStateMetaInfoSerialization() throws Exception {

	String name = "test";
	TypeSerializer<?> namespaceSerializer = LongSerializer.INSTANCE;
	TypeSerializer<?> stateSerializer = DoubleSerializer.INSTANCE;

	StateMetaInfoSnapshot metaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(
		StateDescriptor.Type.VALUE, name, namespaceSerializer, stateSerializer).snapshot();

	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		StateMetaInfoSnapshotReadersWriters.getWriter().
			writeStateMetaInfoSnapshot(metaInfo, new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		final StateMetaInfoReader reader = StateMetaInfoSnapshotReadersWriters.getReader(
			CURRENT_STATE_META_INFO_SNAPSHOT_VERSION, StateMetaInfoSnapshotReadersWriters.StateTypeHint.KEYED_STATE);
		metaInfo = reader.readStateMetaInfoSnapshot(
			new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
	}

	Assert.assertEquals(name, metaInfo.getName());
}
 

public ReusingMergeOuterJoinIterator(
		OuterJoinType outerJoinType,
		MutableObjectIterator<T1> input1,
		MutableObjectIterator<T2> input2,
		TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1,
		TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
		TypePairComparator<T1, T2> pairComparator,
		MemoryManager memoryManager,
		IOManager ioManager,
		int numMemoryPages,
		AbstractInvokable parentTask)
		throws MemoryAllocationException {
	super(outerJoinType, input1, input2, serializer1, comparator1, serializer2, comparator2, pairComparator, memoryManager, ioManager, numMemoryPages, parentTask);

	this.copy1 = serializer1.createInstance();
	this.spillHeadCopy = serializer1.createInstance();
	this.copy2 = serializer2.createInstance();
	this.blockHeadCopy = serializer2.createInstance();
}
 

@Test
public void testRestoreSerializerWithSameFields() {
	final PojoSerializerSnapshot<TestPojo> testSnapshot = buildTestSnapshot(Arrays.asList(
		ID_FIELD,
		NAME_FIELD,
		HEIGHT_FIELD
	));

	final TypeSerializer<TestPojo> restoredSerializer = testSnapshot.restoreSerializer();
	assertSame(restoredSerializer.getClass(), PojoSerializer.class);
	final PojoSerializer<TestPojo> restoredPojoSerializer = (PojoSerializer<TestPojo>) restoredSerializer;

	final Field[] restoredFields = restoredPojoSerializer.getFields();
	assertArrayEquals(
		new Field[] { ID_FIELD.field, NAME_FIELD.field, HEIGHT_FIELD.field },
		restoredFields);

	final TypeSerializer<?>[] restoredFieldSerializers = restoredPojoSerializer.getFieldSerializers();
	assertArrayEquals(
		new TypeSerializer[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE, DoubleSerializer.INSTANCE },
		restoredFieldSerializers);
}
 

private <K, N, U> void assertKeyGroupKeyNamespaceUserKeyBytes(
	K key,
	int keyGroup,
	int prefixBytes,
	TypeSerializer<K> keySerializer,
	N namespace,
	TypeSerializer<N> namespaceSerializer,
	U userKey,
	TypeSerializer<U> userKeySerializer,
	DataInputDeserializer deserializer,
	boolean ambiguousCompositeKeyPossible) throws IOException {
	assertKeyGroupKeyNamespaceBytes(
		key,
		keyGroup,
		prefixBytes,
		keySerializer,
		namespace,
		namespaceSerializer,
		deserializer,
		ambiguousCompositeKeyPossible);
	Assert.assertEquals(userKey, userKeySerializer.deserialize(deserializer));
}
 

public <T> TypeSerializer<T> getTypeSerializerOut(ClassLoader cl) {
	try {
		return InstantiationUtil.readObjectFromConfig(this.config, TYPE_SERIALIZER_OUT_1, cl);
	} catch (Exception e) {
		throw new StreamTaskException("Could not instantiate serializer.", e);
	}
}
 

public SpillingResettableIterator(Iterator<T> input, TypeSerializer<T> serializer, 
		MemoryManager memoryManager, IOManager ioManager,
		int numPages, AbstractInvokable parentTask)
throws MemoryAllocationException
{
	this(input, serializer, memoryManager, ioManager, memoryManager.allocatePages(parentTask, numPages), true);
}
 

public static <V> ValueTimeWrapper<V> deserialize(
	final TypeSerializer<V> valueSerializer,
	final DataInputView source) throws IOException {

	final V value = valueSerializer.deserialize(source);
	final long timestamp = source.readLong();
	final int counter = source.readInt();

	return new ValueTimeWrapper<>(value, timestamp, counter);
}
 

@Override
@SuppressWarnings("unchecked")
public TypeSerializerSchemaCompatibility<C> resolveSchemaCompatibility(TypeSerializer<C> newSerializer) {
	if (!(newSerializer instanceof ListSerializer)) {
		return super.resolveSchemaCompatibility(newSerializer);
	}

	return CompositeTypeSerializerUtil.delegateCompatibilityCheckToNewSnapshot(
		newSerializer,
		(CompositeTypeSerializerSnapshot<C, ? extends TypeSerializer>) new ListSerializerSnapshot<>(),
		getSingleNestedSerializerAndConfig().f1);
}
 

public NonReusingBuildFirstHashJoinIterator(
		MutableObjectIterator<V1> firstInput,
		MutableObjectIterator<V2> secondInput,
		TypeSerializer<V1> serializer1,
		TypeComparator<V1> comparator1,
		TypeSerializer<V2> serializer2,
		TypeComparator<V2> comparator2,
		TypePairComparator<V2, V1> pairComparator,
		MemoryManager memManager, IOManager ioManager,
		AbstractInvokable ownerTask,
		double memoryFraction,
		boolean probeSideOuterJoin,
		boolean buildSideOuterJoin,
		boolean useBitmapFilters) throws MemoryAllocationException {
	
	this.memManager = memManager;
	this.firstInput = firstInput;
	this.secondInput = secondInput;
	this.probeSideSerializer = serializer2;

	if(useBitmapFilters && probeSideOuterJoin) {
		throw new IllegalArgumentException("Bitmap filter may not be activated for joining with empty build side");
	}
	this.probeSideOuterJoin = probeSideOuterJoin;
	this.buildSideOuterJoin = buildSideOuterJoin;

	this.hashJoin = getHashJoin(serializer1, comparator1, serializer2, comparator2,
			pairComparator, memManager, ioManager, ownerTask, memoryFraction, useBitmapFilters);
}
 

@Override
protected TupleComparator<Tuple3<Integer, Long, Double>> createComparator(boolean ascending) {
	return new TupleComparator<Tuple3<Integer, Long, Double>>(
			new int[]{0, 1},
			new TypeComparator[]{
				new IntComparator(ascending),
				new LongComparator(ascending)
			},
			new TypeSerializer[]{ IntSerializer.INSTANCE, LongSerializer.INSTANCE });
}
 

private <K> AbstractKeyedStateBackend<K> createKeyedBackend(TypeSerializer<K> keySerializer, Environment env) throws Exception {
	return createKeyedBackend(
		keySerializer,
		10,
		new KeyGroupRange(0, 9),
		env);
}
 

public static <N> void writeNameSpace(
	N namespace,
	TypeSerializer<N> namespaceSerializer,
	DataOutputSerializer keySerializationDataOutputView,
	boolean ambiguousKeyPossible) throws IOException {

	int beforeWrite = keySerializationDataOutputView.length();
	namespaceSerializer.serialize(namespace, keySerializationDataOutputView);

	if (ambiguousKeyPossible) {
		//write length of namespace
		writeLengthFrom(beforeWrite, keySerializationDataOutputView);
	}
}
 

@Test
public void testDeserializingKryoSerializerWithoutAvro() throws Exception {
	final String resource = "serialized-kryo-serializer-1.3";

	TypeSerializer<?> serializer;

	try (InputStream in = getClass().getClassLoader().getResourceAsStream(resource)) {
		DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(in);

		serializer = TypeSerializerSerializationUtil.tryReadSerializer(inView, getClass().getClassLoader());
	}

	assertNotNull(serializer);
	assertTrue(serializer instanceof KryoSerializer);
}
 

@Nullable
@Override
public Map<UK, TtlValue<UV>> getUnexpiredOrNull(@Nonnull Map<UK, TtlValue<UV>> ttlValue) {
	Map<UK, TtlValue<UV>> unexpired = new HashMap<>();
	TypeSerializer<TtlValue<UV>> valueSerializer =
		((MapSerializer<UK, TtlValue<UV>>) original.getValueSerializer()).getValueSerializer();
	for (Map.Entry<UK, TtlValue<UV>> e : ttlValue.entrySet()) {
		if (!expired(e.getValue())) {
			// we have to do the defensive copy to update the value
			unexpired.put(e.getKey(), valueSerializer.copy(e.getValue()));
		}
	}
	return ttlValue.size() == unexpired.size() ? ttlValue : unexpired;
}
 

public ReusingBlockResettableIterator(MemoryManager memoryManager, Iterator<T> input,
		TypeSerializer<T> serializer, int numPages,
		AbstractInvokable ownerTask)
throws MemoryAllocationException
{
	this(memoryManager, serializer, numPages, ownerTask);
	this.input = input;
}
 

private static TypeSerializer<?>[] init(
	@Nonnull TypeSerializer<?> keySerializer,
	@Nonnull TypeSerializer<?> namespaceSerializer) {
	TypeSerializer<?>[] timerSerializers = new TypeSerializer[2];
	timerSerializers[KEY_SERIALIZER_SNAPSHOT_INDEX] = keySerializer;
	timerSerializers[NAMESPACE_SERIALIZER_SNAPSHOT_INDEX] = namespaceSerializer;
	return timerSerializers;
}
 

@SuppressWarnings("unchecked")
@Override
protected Serializer createOuterSerializerWithNestedSerializers(TypeSerializer<?>[] nestedSerializers) {
	TypeSerializer<?> valueSerializer = nestedSerializers[0];
	TypeSerializer<Long> timestampSerializer = (TypeSerializer<Long>) nestedSerializers[1];
	return new Serializer(valueSerializer, timestampSerializer);
}
 

@Override
public TypeSerializer<TaggedBootstrapData> createSerializer(ExecutionConfig executionConfig) {
  final List<TypeSerializer<?>> payloadSerializers =
      payloadTypeInfos.stream()
          .map(typeInfo -> typeInfo.createSerializer(executionConfig))
          .collect(Collectors.toList());

  return new TaggedBootstrapDataSerializer(payloadSerializers);
}
 

public ReusingBlockResettableIterator(MemoryManager memoryManager, TypeSerializer<T>
		serializer, int numPages, AbstractInvokable ownerTask)
throws MemoryAllocationException
{
	super(memoryManager, serializer, numPages, ownerTask);
	
	this.reuseElement = serializer.createInstance();
}
 

/**
 * Creates an array of serializers for provided list of registered subclasses.
 * Order of returned serializers will correspond to order of provided subclasses.
 */
private static TypeSerializer<?>[] createRegisteredSubclassSerializers(
		LinkedHashSet<Class<?>> registeredSubclasses,
		ExecutionConfig executionConfig) {

	final TypeSerializer<?>[] subclassSerializers = new TypeSerializer[registeredSubclasses.size()];

	int i = 0;
	for (Class<?> registeredClass : registeredSubclasses) {
		subclassSerializers[i] = TypeExtractor.createTypeInfo(registeredClass).createSerializer(executionConfig);
		i++;
	}

	return subclassSerializers;
}
 

private void setTypeSerializer(String key, TypeSerializer<?> typeWrapper) {
	try {
		InstantiationUtil.writeObjectToConfig(typeWrapper, this.config, key);
	} catch (IOException e) {
		throw new StreamTaskException("Could not serialize type serializer.", e);
	}
}
 

@Override
public TypeSerializerSchemaCompatibility<TaggedUnion<T1, T2>> resolveSchemaCompatibility(TypeSerializer<TaggedUnion<T1, T2>> newSerializer) {
	List<Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> nestedSerializersAndConfigs = getNestedSerializersAndConfigs();

	return CompositeTypeSerializerUtil.delegateCompatibilityCheckToNewSnapshot(
		newSerializer,
		new UnionSerializerSnapshot<>(),
		nestedSerializersAndConfigs.get(0).f1,
		nestedSerializersAndConfigs.get(1).f1
	);
}
 

/**
 * Constructs a new {@code StateMap} instance with the specified capacity.
 *
 * @param capacity      the initial capacity of this hash map.
 * @param stateSerializer the serializer of the key.
 * @throws IllegalArgumentException when the capacity is less than zero.
 */
@SuppressWarnings("unchecked")
private CopyOnWriteStateMap(
	int capacity, TypeSerializer<S> stateSerializer) {
	this.stateSerializer = Preconditions.checkNotNull(stateSerializer);

	// initialized maps to EMPTY_TABLE.
	this.primaryTable = (StateMapEntry<K, N, S>[]) EMPTY_TABLE;
	this.incrementalRehashTable = (StateMapEntry<K, N, S>[]) EMPTY_TABLE;

	// initialize sizes to 0.
	this.primaryTableSize = 0;
	this.incrementalRehashTableSize = 0;

	this.rehashIndex = 0;
	this.stateMapVersion = 0;
	this.highestRequiredSnapshotVersion = 0;
	this.snapshotVersions = new TreeSet<>();

	if (capacity < 0) {
		throw new IllegalArgumentException("Capacity: " + capacity);
	}

	if (capacity == 0) {
		threshold = -1;
		return;
	}

	if (capacity < MINIMUM_CAPACITY) {
		capacity = MINIMUM_CAPACITY;
	} else if (capacity > MAXIMUM_CAPACITY) {
		capacity = MAXIMUM_CAPACITY;
	} else {
		capacity = MathUtils.roundUpToPowerOfTwo(capacity);
	}
	primaryTable = makeTable(capacity);
}
 

@SuppressWarnings("unchecked")
static <K, N, SV, S extends State, IS extends S> IS create(
	StateDescriptor<S, SV> stateDesc,
	StateTable<K, N, SV> stateTable,
	TypeSerializer<K> keySerializer) {
	return (IS) new HeapValueState<>(
		stateTable,
		keySerializer,
		stateTable.getStateSerializer(),
		stateTable.getNamespaceSerializer(),
		stateDesc.getDefaultValue());
}
 

public AbstractInternalTimersSnapshotWriter(
		InternalTimersSnapshot<K, N> timersSnapshot,
		TypeSerializer<K> keySerializer,
		TypeSerializer<N> namespaceSerializer) {
	this.timersSnapshot = checkNotNull(timersSnapshot);
	this.keySerializer = checkNotNull(keySerializer);
	this.namespaceSerializer = checkNotNull(namespaceSerializer);
}
 

@SuppressWarnings("unchecked")
@Override
protected TupleSerializer<Tuple3<Integer, String, Double>> createSerializer() {
	return new TupleSerializer<Tuple3<Integer, String, Double>>(
			(Class<Tuple3<Integer, String, Double>>) (Class<?>) Tuple3.class,
			new TypeSerializer[]{
				new IntSerializer(),
				new StringSerializer(),
				new DoubleSerializer()});
}
 

@Override
protected TimerSerializer<K, N> createOuterSerializerWithNestedSerializers(TypeSerializer<?>[] nestedSerializers) {
	@SuppressWarnings("unchecked")
	final TypeSerializer<K> keySerializer = (TypeSerializer<K>) nestedSerializers[0];

	@SuppressWarnings("unchecked")
	final TypeSerializer<N> namespaceSerializer = (TypeSerializer<N>) nestedSerializers[1];

	return new TimerSerializer<>(keySerializer, namespaceSerializer);
}
 

/**
 * Initializes the KeyGroupedIterator. It requires an iterator which returns its result
 * sorted by the key fields.
 * 
 * @param iterator An iterator over records, which are sorted by the key fields, in any order.
 * @param serializer The serializer for the data type iterated over.
 * @param comparator The comparator for the data type iterated over.
 */
public ReusingKeyGroupedIterator(MutableObjectIterator<E> iterator, TypeSerializer<E>
		serializer, TypeComparator<E> comparator)
{
	if (iterator == null || serializer == null || comparator == null) {
		throw new NullPointerException();
	}
	
	this.iterator = iterator;
	this.serializer = serializer;
	this.comparator = comparator;
	this.reuse = this.serializer.createInstance();
}