org.apache.flink.api.common.typeutils.TypeSerializer Java Examples

The following examples show how to use org.apache.flink.api.common.typeutils.TypeSerializer. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: Alink   Author: alibaba   File: VectorTypesTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testVectorsSerDeser() throws IOException {
	// Prepare data
	SparseVector sparseVector = new SparseVector(10, new HashMap<Integer, Double>() {{
		ThreadLocalRandom rand = ThreadLocalRandom.current();
		for (int i = 0; i < 10; i += 2) {
			this.put(i, rand.nextDouble());
		}
	}});
	DenseVector denseVector = DenseVector.rand(10);

	// Prepare serializer
	ExecutionConfig config = new ExecutionConfig();
	TypeSerializer<Vector> vecSer = VectorTypes.VECTOR.createSerializer(config);
	TypeSerializer<SparseVector> sparseSer = VectorTypes.SPARSE_VECTOR.createSerializer(config);
	TypeSerializer<DenseVector> denseSer = VectorTypes.DENSE_VECTOR.createSerializer(config);

	// Do tests.
	doVectorSerDeserTest(vecSer, sparseVector);
	doVectorSerDeserTest(vecSer, denseVector);
	doVectorSerDeserTest(sparseSer, sparseVector);
	doVectorSerDeserTest(denseSer, denseVector);
}
 
Example #2
Source Project: Flink-CEPplus   Author: ljygz   File: OperatorStateRestoreOperation.java    License: Apache License 2.0 6 votes vote down vote up
private <K, V> void deserializeBroadcastStateValues(
	final BackendWritableBroadcastState<K, V> broadcastStateForName,
	final FSDataInputStream in,
	final OperatorStateHandle.StateMetaInfo metaInfo) throws Exception {

	if (metaInfo != null) {
		long[] offsets = metaInfo.getOffsets();
		if (offsets != null) {

			TypeSerializer<K> keySerializer = broadcastStateForName.getStateMetaInfo().getKeySerializer();
			TypeSerializer<V> valueSerializer = broadcastStateForName.getStateMetaInfo().getValueSerializer();

			in.seek(offsets[0]);

			DataInputView div = new DataInputViewStreamWrapper(in);
			int size = div.readInt();
			for (int i = 0; i < size; i++) {
				broadcastStateForName.put(keySerializer.deserialize(div), valueSerializer.deserialize(div));
			}
		}
	}
}
 
Example #3
Source Project: Flink-CEPplus   Author: ljygz   File: SerializationProxiesTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testKeyedStateMetaInfoSerialization() throws Exception {

	String name = "test";
	TypeSerializer<?> namespaceSerializer = LongSerializer.INSTANCE;
	TypeSerializer<?> stateSerializer = DoubleSerializer.INSTANCE;

	StateMetaInfoSnapshot metaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(
		StateDescriptor.Type.VALUE, name, namespaceSerializer, stateSerializer).snapshot();

	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		StateMetaInfoSnapshotReadersWriters.getWriter().
			writeStateMetaInfoSnapshot(metaInfo, new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		final StateMetaInfoReader reader = StateMetaInfoSnapshotReadersWriters.getReader(
			CURRENT_STATE_META_INFO_SNAPSHOT_VERSION, StateMetaInfoSnapshotReadersWriters.StateTypeHint.KEYED_STATE);
		metaInfo = reader.readStateMetaInfoSnapshot(
			new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
	}

	Assert.assertEquals(name, metaInfo.getName());
}
 
Example #4
Source Project: flink   Author: apache   File: ReusingMergeOuterJoinIterator.java    License: Apache License 2.0 6 votes vote down vote up
public ReusingMergeOuterJoinIterator(
		OuterJoinType outerJoinType,
		MutableObjectIterator<T1> input1,
		MutableObjectIterator<T2> input2,
		TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1,
		TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
		TypePairComparator<T1, T2> pairComparator,
		MemoryManager memoryManager,
		IOManager ioManager,
		int numMemoryPages,
		AbstractInvokable parentTask)
		throws MemoryAllocationException {
	super(outerJoinType, input1, input2, serializer1, comparator1, serializer2, comparator2, pairComparator, memoryManager, ioManager, numMemoryPages, parentTask);

	this.copy1 = serializer1.createInstance();
	this.spillHeadCopy = serializer1.createInstance();
	this.copy2 = serializer2.createInstance();
	this.blockHeadCopy = serializer2.createInstance();
}
 
Example #5
Source Project: flink   Author: apache   File: PojoSerializerSnapshotTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRestoreSerializerWithSameFields() {
	final PojoSerializerSnapshot<TestPojo> testSnapshot = buildTestSnapshot(Arrays.asList(
		ID_FIELD,
		NAME_FIELD,
		HEIGHT_FIELD
	));

	final TypeSerializer<TestPojo> restoredSerializer = testSnapshot.restoreSerializer();
	assertSame(restoredSerializer.getClass(), PojoSerializer.class);
	final PojoSerializer<TestPojo> restoredPojoSerializer = (PojoSerializer<TestPojo>) restoredSerializer;

	final Field[] restoredFields = restoredPojoSerializer.getFields();
	assertArrayEquals(
		new Field[] { ID_FIELD.field, NAME_FIELD.field, HEIGHT_FIELD.field },
		restoredFields);

	final TypeSerializer<?>[] restoredFieldSerializers = restoredPojoSerializer.getFieldSerializers();
	assertArrayEquals(
		new TypeSerializer[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE, DoubleSerializer.INSTANCE },
		restoredFieldSerializers);
}
 
Example #6
Source Project: flink   Author: apache   File: RocksDBSerializedCompositeKeyBuilderTest.java    License: Apache License 2.0 6 votes vote down vote up
private <K, N, U> void assertKeyGroupKeyNamespaceUserKeyBytes(
	K key,
	int keyGroup,
	int prefixBytes,
	TypeSerializer<K> keySerializer,
	N namespace,
	TypeSerializer<N> namespaceSerializer,
	U userKey,
	TypeSerializer<U> userKeySerializer,
	DataInputDeserializer deserializer,
	boolean ambiguousCompositeKeyPossible) throws IOException {
	assertKeyGroupKeyNamespaceBytes(
		key,
		keyGroup,
		prefixBytes,
		keySerializer,
		namespace,
		namespaceSerializer,
		deserializer,
		ambiguousCompositeKeyPossible);
	Assert.assertEquals(userKey, userKeySerializer.deserialize(deserializer));
}
 
Example #7
Source Project: flink   Author: flink-tpc-ds   File: StreamConfig.java    License: Apache License 2.0 5 votes vote down vote up
public <T> TypeSerializer<T> getTypeSerializerOut(ClassLoader cl) {
	try {
		return InstantiationUtil.readObjectFromConfig(this.config, TYPE_SERIALIZER_OUT_1, cl);
	} catch (Exception e) {
		throw new StreamTaskException("Could not instantiate serializer.", e);
	}
}
 
Example #8
Source Project: Flink-CEPplus   Author: ljygz   File: SpillingResettableIterator.java    License: Apache License 2.0 5 votes vote down vote up
public SpillingResettableIterator(Iterator<T> input, TypeSerializer<T> serializer, 
		MemoryManager memoryManager, IOManager ioManager,
		int numPages, AbstractInvokable parentTask)
throws MemoryAllocationException
{
	this(input, serializer, memoryManager, ioManager, memoryManager.allocatePages(parentTask, numPages), true);
}
 
Example #9
Source Project: flink   Author: flink-tpc-ds   File: SharedBuffer.java    License: Apache License 2.0 5 votes vote down vote up
public static <V> ValueTimeWrapper<V> deserialize(
	final TypeSerializer<V> valueSerializer,
	final DataInputView source) throws IOException {

	final V value = valueSerializer.deserialize(source);
	final long timestamp = source.readLong();
	final int counter = source.readInt();

	return new ValueTimeWrapper<>(value, timestamp, counter);
}
 
Example #10
Source Project: Flink-CEPplus   Author: ljygz   File: CollectionSerializerConfigSnapshot.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public TypeSerializerSchemaCompatibility<C> resolveSchemaCompatibility(TypeSerializer<C> newSerializer) {
	if (!(newSerializer instanceof ListSerializer)) {
		return super.resolveSchemaCompatibility(newSerializer);
	}

	return CompositeTypeSerializerUtil.delegateCompatibilityCheckToNewSnapshot(
		newSerializer,
		(CompositeTypeSerializerSnapshot<C, ? extends TypeSerializer>) new ListSerializerSnapshot<>(),
		getSingleNestedSerializerAndConfig().f1);
}
 
Example #11
Source Project: flink   Author: flink-tpc-ds   File: NonReusingBuildFirstHashJoinIterator.java    License: Apache License 2.0 5 votes vote down vote up
public NonReusingBuildFirstHashJoinIterator(
		MutableObjectIterator<V1> firstInput,
		MutableObjectIterator<V2> secondInput,
		TypeSerializer<V1> serializer1,
		TypeComparator<V1> comparator1,
		TypeSerializer<V2> serializer2,
		TypeComparator<V2> comparator2,
		TypePairComparator<V2, V1> pairComparator,
		MemoryManager memManager, IOManager ioManager,
		AbstractInvokable ownerTask,
		double memoryFraction,
		boolean probeSideOuterJoin,
		boolean buildSideOuterJoin,
		boolean useBitmapFilters) throws MemoryAllocationException {
	
	this.memManager = memManager;
	this.firstInput = firstInput;
	this.secondInput = secondInput;
	this.probeSideSerializer = serializer2;

	if(useBitmapFilters && probeSideOuterJoin) {
		throw new IllegalArgumentException("Bitmap filter may not be activated for joining with empty build side");
	}
	this.probeSideOuterJoin = probeSideOuterJoin;
	this.buildSideOuterJoin = buildSideOuterJoin;

	this.hashJoin = getHashJoin(serializer1, comparator1, serializer2, comparator2,
			pairComparator, memManager, ioManager, ownerTask, memoryFraction, useBitmapFilters);
}
 
Example #12
Source Project: Flink-CEPplus   Author: ljygz   File: TupleComparatorILD2Test.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected TupleComparator<Tuple3<Integer, Long, Double>> createComparator(boolean ascending) {
	return new TupleComparator<Tuple3<Integer, Long, Double>>(
			new int[]{0, 1},
			new TypeComparator[]{
				new IntComparator(ascending),
				new LongComparator(ascending)
			},
			new TypeSerializer[]{ IntSerializer.INSTANCE, LongSerializer.INSTANCE });
}
 
Example #13
Source Project: flink   Author: flink-tpc-ds   File: StateBackendMigrationTestBase.java    License: Apache License 2.0 5 votes vote down vote up
private <K> AbstractKeyedStateBackend<K> createKeyedBackend(TypeSerializer<K> keySerializer, Environment env) throws Exception {
	return createKeyedBackend(
		keySerializer,
		10,
		new KeyGroupRange(0, 9),
		env);
}
 
Example #14
Source Project: flink   Author: apache   File: RocksDBKeySerializationUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static <N> void writeNameSpace(
	N namespace,
	TypeSerializer<N> namespaceSerializer,
	DataOutputSerializer keySerializationDataOutputView,
	boolean ambiguousKeyPossible) throws IOException {

	int beforeWrite = keySerializationDataOutputView.length();
	namespaceSerializer.serialize(namespace, keySerializationDataOutputView);

	if (ambiguousKeyPossible) {
		//write length of namespace
		writeLengthFrom(beforeWrite, keySerializationDataOutputView);
	}
}
 
Example #15
Source Project: flink   Author: flink-tpc-ds   File: KryoSerializerCompatibilityTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDeserializingKryoSerializerWithoutAvro() throws Exception {
	final String resource = "serialized-kryo-serializer-1.3";

	TypeSerializer<?> serializer;

	try (InputStream in = getClass().getClassLoader().getResourceAsStream(resource)) {
		DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(in);

		serializer = TypeSerializerSerializationUtil.tryReadSerializer(inView, getClass().getClassLoader());
	}

	assertNotNull(serializer);
	assertTrue(serializer instanceof KryoSerializer);
}
 
Example #16
Source Project: flink   Author: flink-tpc-ds   File: TtlMapState.java    License: Apache License 2.0 5 votes vote down vote up
@Nullable
@Override
public Map<UK, TtlValue<UV>> getUnexpiredOrNull(@Nonnull Map<UK, TtlValue<UV>> ttlValue) {
	Map<UK, TtlValue<UV>> unexpired = new HashMap<>();
	TypeSerializer<TtlValue<UV>> valueSerializer =
		((MapSerializer<UK, TtlValue<UV>>) original.getValueSerializer()).getValueSerializer();
	for (Map.Entry<UK, TtlValue<UV>> e : ttlValue.entrySet()) {
		if (!expired(e.getValue())) {
			// we have to do the defensive copy to update the value
			unexpired.put(e.getKey(), valueSerializer.copy(e.getValue()));
		}
	}
	return ttlValue.size() == unexpired.size() ? ttlValue : unexpired;
}
 
Example #17
Source Project: flink   Author: flink-tpc-ds   File: ReusingBlockResettableIterator.java    License: Apache License 2.0 5 votes vote down vote up
public ReusingBlockResettableIterator(MemoryManager memoryManager, Iterator<T> input,
		TypeSerializer<T> serializer, int numPages,
		AbstractInvokable ownerTask)
throws MemoryAllocationException
{
	this(memoryManager, serializer, numPages, ownerTask);
	this.input = input;
}
 
Example #18
Source Project: Flink-CEPplus   Author: ljygz   File: TimerSerializer.java    License: Apache License 2.0 5 votes vote down vote up
private static TypeSerializer<?>[] init(
	@Nonnull TypeSerializer<?> keySerializer,
	@Nonnull TypeSerializer<?> namespaceSerializer) {
	TypeSerializer<?>[] timerSerializers = new TypeSerializer[2];
	timerSerializers[KEY_SERIALIZER_SNAPSHOT_INDEX] = keySerializer;
	timerSerializers[NAMESPACE_SERIALIZER_SNAPSHOT_INDEX] = namespaceSerializer;
	return timerSerializers;
}
 
Example #19
Source Project: Flink-CEPplus   Author: ljygz   File: ValueWithTs.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
protected Serializer createOuterSerializerWithNestedSerializers(TypeSerializer<?>[] nestedSerializers) {
	TypeSerializer<?> valueSerializer = nestedSerializers[0];
	TypeSerializer<Long> timestampSerializer = (TypeSerializer<Long>) nestedSerializers[1];
	return new Serializer(valueSerializer, timestampSerializer);
}
 
Example #20
Source Project: flink-statefun   Author: apache   File: TaggedBootstrapDataTypeInfo.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public TypeSerializer<TaggedBootstrapData> createSerializer(ExecutionConfig executionConfig) {
  final List<TypeSerializer<?>> payloadSerializers =
      payloadTypeInfos.stream()
          .map(typeInfo -> typeInfo.createSerializer(executionConfig))
          .collect(Collectors.toList());

  return new TaggedBootstrapDataSerializer(payloadSerializers);
}
 
Example #21
Source Project: Flink-CEPplus   Author: ljygz   File: ReusingBlockResettableIterator.java    License: Apache License 2.0 5 votes vote down vote up
public ReusingBlockResettableIterator(MemoryManager memoryManager, TypeSerializer<T>
		serializer, int numPages, AbstractInvokable ownerTask)
throws MemoryAllocationException
{
	super(memoryManager, serializer, numPages, ownerTask);
	
	this.reuseElement = serializer.createInstance();
}
 
Example #22
Source Project: Flink-CEPplus   Author: ljygz   File: PojoSerializer.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates an array of serializers for provided list of registered subclasses.
 * Order of returned serializers will correspond to order of provided subclasses.
 */
private static TypeSerializer<?>[] createRegisteredSubclassSerializers(
		LinkedHashSet<Class<?>> registeredSubclasses,
		ExecutionConfig executionConfig) {

	final TypeSerializer<?>[] subclassSerializers = new TypeSerializer[registeredSubclasses.size()];

	int i = 0;
	for (Class<?> registeredClass : registeredSubclasses) {
		subclassSerializers[i] = TypeExtractor.createTypeInfo(registeredClass).createSerializer(executionConfig);
		i++;
	}

	return subclassSerializers;
}
 
Example #23
Source Project: Flink-CEPplus   Author: ljygz   File: StreamConfig.java    License: Apache License 2.0 5 votes vote down vote up
private void setTypeSerializer(String key, TypeSerializer<?> typeWrapper) {
	try {
		InstantiationUtil.writeObjectToConfig(typeWrapper, this.config, key);
	} catch (IOException e) {
		throw new StreamTaskException("Could not serialize type serializer.", e);
	}
}
 
Example #24
Source Project: flink   Author: apache   File: CoGroupedStreams.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public TypeSerializerSchemaCompatibility<TaggedUnion<T1, T2>> resolveSchemaCompatibility(TypeSerializer<TaggedUnion<T1, T2>> newSerializer) {
	List<Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> nestedSerializersAndConfigs = getNestedSerializersAndConfigs();

	return CompositeTypeSerializerUtil.delegateCompatibilityCheckToNewSnapshot(
		newSerializer,
		new UnionSerializerSnapshot<>(),
		nestedSerializersAndConfigs.get(0).f1,
		nestedSerializersAndConfigs.get(1).f1
	);
}
 
Example #25
Source Project: flink   Author: apache   File: CopyOnWriteStateMap.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Constructs a new {@code StateMap} instance with the specified capacity.
 *
 * @param capacity      the initial capacity of this hash map.
 * @param stateSerializer the serializer of the key.
 * @throws IllegalArgumentException when the capacity is less than zero.
 */
@SuppressWarnings("unchecked")
private CopyOnWriteStateMap(
	int capacity, TypeSerializer<S> stateSerializer) {
	this.stateSerializer = Preconditions.checkNotNull(stateSerializer);

	// initialized maps to EMPTY_TABLE.
	this.primaryTable = (StateMapEntry<K, N, S>[]) EMPTY_TABLE;
	this.incrementalRehashTable = (StateMapEntry<K, N, S>[]) EMPTY_TABLE;

	// initialize sizes to 0.
	this.primaryTableSize = 0;
	this.incrementalRehashTableSize = 0;

	this.rehashIndex = 0;
	this.stateMapVersion = 0;
	this.highestRequiredSnapshotVersion = 0;
	this.snapshotVersions = new TreeSet<>();

	if (capacity < 0) {
		throw new IllegalArgumentException("Capacity: " + capacity);
	}

	if (capacity == 0) {
		threshold = -1;
		return;
	}

	if (capacity < MINIMUM_CAPACITY) {
		capacity = MINIMUM_CAPACITY;
	} else if (capacity > MAXIMUM_CAPACITY) {
		capacity = MAXIMUM_CAPACITY;
	} else {
		capacity = MathUtils.roundUpToPowerOfTwo(capacity);
	}
	primaryTable = makeTable(capacity);
}
 
Example #26
Source Project: flink   Author: apache   File: HeapValueState.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <K, N, SV, S extends State, IS extends S> IS create(
	StateDescriptor<S, SV> stateDesc,
	StateTable<K, N, SV> stateTable,
	TypeSerializer<K> keySerializer) {
	return (IS) new HeapValueState<>(
		stateTable,
		keySerializer,
		stateTable.getStateSerializer(),
		stateTable.getNamespaceSerializer(),
		stateDesc.getDefaultValue());
}
 
Example #27
Source Project: Flink-CEPplus   Author: ljygz   File: InternalTimersSnapshotReaderWriters.java    License: Apache License 2.0 5 votes vote down vote up
public AbstractInternalTimersSnapshotWriter(
		InternalTimersSnapshot<K, N> timersSnapshot,
		TypeSerializer<K> keySerializer,
		TypeSerializer<N> namespaceSerializer) {
	this.timersSnapshot = checkNotNull(timersSnapshot);
	this.keySerializer = checkNotNull(keySerializer);
	this.namespaceSerializer = checkNotNull(namespaceSerializer);
}
 
Example #28
Source Project: Flink-CEPplus   Author: ljygz   File: TupleComparatorISD3Test.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
protected TupleSerializer<Tuple3<Integer, String, Double>> createSerializer() {
	return new TupleSerializer<Tuple3<Integer, String, Double>>(
			(Class<Tuple3<Integer, String, Double>>) (Class<?>) Tuple3.class,
			new TypeSerializer[]{
				new IntSerializer(),
				new StringSerializer(),
				new DoubleSerializer()});
}
 
Example #29
Source Project: flink   Author: apache   File: TimerSerializerSnapshot.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected TimerSerializer<K, N> createOuterSerializerWithNestedSerializers(TypeSerializer<?>[] nestedSerializers) {
	@SuppressWarnings("unchecked")
	final TypeSerializer<K> keySerializer = (TypeSerializer<K>) nestedSerializers[0];

	@SuppressWarnings("unchecked")
	final TypeSerializer<N> namespaceSerializer = (TypeSerializer<N>) nestedSerializers[1];

	return new TimerSerializer<>(keySerializer, namespaceSerializer);
}
 
Example #30
Source Project: flink   Author: apache   File: ReusingKeyGroupedIterator.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Initializes the KeyGroupedIterator. It requires an iterator which returns its result
 * sorted by the key fields.
 * 
 * @param iterator An iterator over records, which are sorted by the key fields, in any order.
 * @param serializer The serializer for the data type iterated over.
 * @param comparator The comparator for the data type iterated over.
 */
public ReusingKeyGroupedIterator(MutableObjectIterator<E> iterator, TypeSerializer<E>
		serializer, TypeComparator<E> comparator)
{
	if (iterator == null || serializer == null || comparator == null) {
		throw new NullPointerException();
	}
	
	this.iterator = iterator;
	this.serializer = serializer;
	this.comparator = comparator;
	this.reuse = this.serializer.createInstance();
}