Java Code Examples for org.apache.flink.api.common.ExecutionConfig#enableForceKryo()

The following examples show how to use org.apache.flink.api.common.ExecutionConfig#enableForceKryo() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Benchmark
@OperationsPerInvocation(value = SerializationFrameworkMiniBenchmarks.RECORDS_PER_INVOCATION)
public void serializerKryoThrift(FlinkEnvironmentContext context) throws Exception {
	StreamExecutionEnvironment env = context.env;
	env.setParallelism(4);
	ExecutionConfig executionConfig = env.getConfig();
	executionConfig.enableForceKryo();
	executionConfig.addDefaultKryoSerializer(org.apache.flink.benchmark.thrift.MyPojo.class, TBaseSerializer.class);
	executionConfig.addDefaultKryoSerializer(org.apache.flink.benchmark.thrift.MyOperation.class, TBaseSerializer.class);

	env.addSource(new ThriftPojoSource(RECORDS_PER_INVOCATION, 10))
			.rebalance()
			.addSink(new DiscardingSink<>());

	env.execute();
}
 
Example 2
@Benchmark
@OperationsPerInvocation(value = SerializationFrameworkMiniBenchmarks.RECORDS_PER_INVOCATION)
public void serializerKryoProtobuf(FlinkEnvironmentContext context) throws Exception {
	StreamExecutionEnvironment env = context.env;
	env.setParallelism(4);
	ExecutionConfig executionConfig = env.getConfig();
	executionConfig.enableForceKryo();
	executionConfig.registerTypeWithKryoSerializer(org.apache.flink.benchmark.protobuf.MyPojoOuterClass.MyPojo.class, ProtobufSerializer.class);
	executionConfig.registerTypeWithKryoSerializer(org.apache.flink.benchmark.protobuf.MyPojoOuterClass.MyOperation.class, ProtobufSerializer.class);

	env.addSource(new ProtobufPojoSource(RECORDS_PER_INVOCATION, 10))
			.rebalance()
			.addSink(new DiscardingSink<>());

	env.execute();
}
 
Example 3
@Benchmark
@OperationsPerInvocation(value = SerializationFrameworkMiniBenchmarks.RECORDS_PER_INVOCATION)
public void serializerKryo(FlinkEnvironmentContext context) throws Exception {
	StreamExecutionEnvironment env = context.env;
	env.setParallelism(4);
	ExecutionConfig executionConfig = env.getConfig();
	executionConfig.enableForceKryo();
	executionConfig.registerKryoType(MyPojo.class);
	executionConfig.registerKryoType(MyOperation.class);

	env.addSource(new PojoSource(RECORDS_PER_INVOCATION, 10))
			.rebalance()
			.addSink(new DiscardingSink<>());

	env.execute();
}
 
Example 4
Source Project: flink   File: BaseMapSerializerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testExecutionConfigWithKryo() throws Exception {
	// serialize base array
	ExecutionConfig config = new ExecutionConfig();
	config.enableForceKryo();
	config.registerTypeWithKryoSerializer(MyObj.class, new MyObjSerializer());
	final BaseMapSerializer serializer = createSerializerWithConfig(config);

	int inputKey = 998244353;
	MyObj inputObj = new MyObj(114514, 1919810);
	Map<Object, Object> javaMap = new HashMap<>();
	javaMap.put(inputKey, new BinaryGeneric<>(inputObj, new KryoSerializer<>(MyObj.class, config)));
	BaseMap inputMap = new GenericMap(javaMap);

	byte[] serialized;
	try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
		serializer.serialize(inputMap, new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	// deserialize base array using restored serializer
	final BaseMapSerializer restoreSerializer =
		(BaseMapSerializer) snapshotAndReconfigure(serializer, () -> createSerializerWithConfig(config));

	BaseMap outputMap;
	try (ByteArrayInputStream in = new ByteArrayInputStream(serialized)) {
		outputMap = restoreSerializer.deserialize(new DataInputViewStreamWrapper(in));
	}

	TypeSerializer restoreKeySer = restoreSerializer.getKeySerializer();
	TypeSerializer restoreValueSer = restoreSerializer.getValueSerializer();
	assertEquals(serializer.getKeySerializer(), restoreKeySer);
	assertEquals(serializer.getValueSerializer(), restoreValueSer);

	MyObj outputObj = BinaryGeneric.getJavaObjectFromBinaryGeneric(
		(BinaryGeneric) outputMap.toJavaMap(
			DataTypes.INT().getLogicalType(), DataTypes.ANY(TypeInformation.of(MyObj.class)).getLogicalType())
			.get(inputKey), new KryoSerializer<>(MyObj.class, config));
	assertEquals(inputObj, outputObj);
}
 
Example 5
Source Project: flink   File: BaseArraySerializerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testExecutionConfigWithKryo() throws Exception {
	// serialize base array
	ExecutionConfig config = new ExecutionConfig();
	config.enableForceKryo();
	config.registerTypeWithKryoSerializer(MyObj.class, new MyObjSerializer());
	final BaseArraySerializer serializer = createSerializerWithConfig(config);

	MyObj inputObj = new MyObj(114514, 1919810);
	BaseArray inputArray = new GenericArray(new BinaryGeneric[] {
		new BinaryGeneric<>(inputObj, new KryoSerializer<>(MyObj.class, config))
	}, 1);

	byte[] serialized;
	try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
		serializer.serialize(inputArray, new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	// deserialize base array using restored serializer
	final BaseArraySerializer restoreSerializer =
		(BaseArraySerializer) snapshotAndReconfigure(serializer, () -> createSerializerWithConfig(config));

	BaseArray outputArray;
	try (ByteArrayInputStream in = new ByteArrayInputStream(serialized)) {
		outputArray = restoreSerializer.deserialize(new DataInputViewStreamWrapper(in));
	}

	TypeSerializer restoreEleSer = restoreSerializer.getEleSer();
	assertEquals(serializer.getEleSer(), restoreEleSer);

	MyObj outputObj = BinaryGeneric.getJavaObjectFromBinaryGeneric(
		outputArray.getGeneric(0), new KryoSerializer<>(MyObj.class, config));
	assertEquals(inputObj, outputObj);
}
 
Example 6
Source Project: flink   File: ArrayDataSerializerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testExecutionConfigWithKryo() throws Exception {
	// serialize base array
	ExecutionConfig config = new ExecutionConfig();
	config.enableForceKryo();
	config.registerTypeWithKryoSerializer(MyObj.class, new MyObjSerializer());
	final ArrayDataSerializer serializer = createSerializerWithConfig(config);

	MyObj inputObj = new MyObj(114514, 1919810);
	ArrayData inputArray = new GenericArrayData(new RawValueData[] {
		RawValueData.fromObject(inputObj)
	});

	byte[] serialized;
	try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
		serializer.serialize(inputArray, new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	// deserialize base array using restored serializer
	final ArrayDataSerializer restoreSerializer =
		(ArrayDataSerializer) snapshotAndReconfigure(serializer, () -> createSerializerWithConfig(config));

	ArrayData outputArray;
	try (ByteArrayInputStream in = new ByteArrayInputStream(serialized)) {
		outputArray = restoreSerializer.deserialize(new DataInputViewStreamWrapper(in));
	}

	TypeSerializer restoreEleSer = restoreSerializer.getEleSer();
	assertEquals(serializer.getEleSer(), restoreEleSer);

	MyObj outputObj = outputArray
		.<MyObj>getRawValue(0)
		.toObject(new KryoSerializer<>(MyObj.class, config));
	assertEquals(inputObj, outputObj);
}
 
Example 7
Source Project: flink   File: MapDataSerializerTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testExecutionConfigWithKryo() throws Exception {
	// serialize base array
	ExecutionConfig config = new ExecutionConfig();
	config.enableForceKryo();
	config.registerTypeWithKryoSerializer(MyObj.class, new MyObjSerializer());
	final MapDataSerializer serializer = createSerializerWithConfig(config);

	int inputKey = 998244353;
	MyObj inputObj = new MyObj(114514, 1919810);
	Map<Object, Object> javaMap = new HashMap<>();
	javaMap.put(inputKey, RawValueData.fromObject(inputObj));
	MapData inputMap = new GenericMapData(javaMap);

	byte[] serialized;
	try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
		serializer.serialize(inputMap, new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	// deserialize base array using restored serializer
	final MapDataSerializer restoreSerializer =
		(MapDataSerializer) snapshotAndReconfigure(serializer, () -> createSerializerWithConfig(config));

	MapData outputMap;
	try (ByteArrayInputStream in = new ByteArrayInputStream(serialized)) {
		outputMap = restoreSerializer.deserialize(new DataInputViewStreamWrapper(in));
	}

	TypeSerializer restoreKeySer = restoreSerializer.getKeySerializer();
	TypeSerializer restoreValueSer = restoreSerializer.getValueSerializer();
	assertEquals(serializer.getKeySerializer(), restoreKeySer);
	assertEquals(serializer.getValueSerializer(), restoreValueSer);

	Map<Object, Object> outputJavaMap = convertToJavaMap(
		outputMap,
		DataTypes.INT().getLogicalType(),
		DataTypes.RAW(TypeInformation.of(MyObj.class)).getLogicalType());
	MyObj outputObj = ((RawValueData<MyObj>) outputJavaMap.get(inputKey))
		.toObject(new KryoSerializer<>(MyObj.class, config));
	assertEquals(inputObj, outputObj);
}