Java Code Examples for org.apache.flink.api.common.ExecutionConfig#enableForceKryo()

The following examples show how to use org.apache.flink.api.common.ExecutionConfig#enableForceKryo() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SerializationFrameworkAllBenchmarks.java    From flink-benchmarks with Apache License 2.0 6 votes vote down vote up 
@Benchmark
@OperationsPerInvocation(value = SerializationFrameworkMiniBenchmarks.RECORDS_PER_INVOCATION)
public void serializerKryoThrift(FlinkEnvironmentContext context) throws Exception {
	StreamExecutionEnvironment env = context.env;
	env.setParallelism(4);
	ExecutionConfig executionConfig = env.getConfig();
	executionConfig.enableForceKryo();
	executionConfig.addDefaultKryoSerializer(org.apache.flink.benchmark.thrift.MyPojo.class, TBaseSerializer.class);
	executionConfig.addDefaultKryoSerializer(org.apache.flink.benchmark.thrift.MyOperation.class, TBaseSerializer.class);

	env.addSource(new ThriftPojoSource(RECORDS_PER_INVOCATION, 10))
			.rebalance()
			.addSink(new DiscardingSink<>());

	env.execute();
}
Example 2
Source File: SerializationFrameworkAllBenchmarks.java    From flink-benchmarks with Apache License 2.0 6 votes vote down vote up 
@Benchmark
@OperationsPerInvocation(value = SerializationFrameworkMiniBenchmarks.RECORDS_PER_INVOCATION)
public void serializerKryoProtobuf(FlinkEnvironmentContext context) throws Exception {
	StreamExecutionEnvironment env = context.env;
	env.setParallelism(4);
	ExecutionConfig executionConfig = env.getConfig();
	executionConfig.enableForceKryo();
	executionConfig.registerTypeWithKryoSerializer(org.apache.flink.benchmark.protobuf.MyPojoOuterClass.MyPojo.class, ProtobufSerializer.class);
	executionConfig.registerTypeWithKryoSerializer(org.apache.flink.benchmark.protobuf.MyPojoOuterClass.MyOperation.class, ProtobufSerializer.class);

	env.addSource(new ProtobufPojoSource(RECORDS_PER_INVOCATION, 10))
			.rebalance()
			.addSink(new DiscardingSink<>());

	env.execute();
}
Example 3
Source File: SerializationFrameworkMiniBenchmarks.java    From flink-benchmarks with Apache License 2.0 6 votes vote down vote up 
@Benchmark
@OperationsPerInvocation(value = SerializationFrameworkMiniBenchmarks.RECORDS_PER_INVOCATION)
public void serializerKryo(FlinkEnvironmentContext context) throws Exception {
	StreamExecutionEnvironment env = context.env;
	env.setParallelism(4);
	ExecutionConfig executionConfig = env.getConfig();
	executionConfig.enableForceKryo();
	executionConfig.registerKryoType(MyPojo.class);
	executionConfig.registerKryoType(MyOperation.class);

	env.addSource(new PojoSource(RECORDS_PER_INVOCATION, 10))
			.rebalance()
			.addSink(new DiscardingSink<>());

	env.execute();
}
Example 4
Source File: BaseMapSerializerTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testExecutionConfigWithKryo() throws Exception {
	// serialize base array
	ExecutionConfig config = new ExecutionConfig();
	config.enableForceKryo();
	config.registerTypeWithKryoSerializer(MyObj.class, new MyObjSerializer());
	final BaseMapSerializer serializer = createSerializerWithConfig(config);

	int inputKey = 998244353;
	MyObj inputObj = new MyObj(114514, 1919810);
	Map<Object, Object> javaMap = new HashMap<>();
	javaMap.put(inputKey, new BinaryGeneric<>(inputObj, new KryoSerializer<>(MyObj.class, config)));
	BaseMap inputMap = new GenericMap(javaMap);

	byte[] serialized;
	try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
		serializer.serialize(inputMap, new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	// deserialize base array using restored serializer
	final BaseMapSerializer restoreSerializer =
		(BaseMapSerializer) snapshotAndReconfigure(serializer, () -> createSerializerWithConfig(config));

	BaseMap outputMap;
	try (ByteArrayInputStream in = new ByteArrayInputStream(serialized)) {
		outputMap = restoreSerializer.deserialize(new DataInputViewStreamWrapper(in));
	}

	TypeSerializer restoreKeySer = restoreSerializer.getKeySerializer();
	TypeSerializer restoreValueSer = restoreSerializer.getValueSerializer();
	assertEquals(serializer.getKeySerializer(), restoreKeySer);
	assertEquals(serializer.getValueSerializer(), restoreValueSer);

	MyObj outputObj = BinaryGeneric.getJavaObjectFromBinaryGeneric(
		(BinaryGeneric) outputMap.toJavaMap(
			DataTypes.INT().getLogicalType(), DataTypes.ANY(TypeInformation.of(MyObj.class)).getLogicalType())
			.get(inputKey), new KryoSerializer<>(MyObj.class, config));
	assertEquals(inputObj, outputObj);
}
Example 5
Source File: BaseArraySerializerTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testExecutionConfigWithKryo() throws Exception {
	// serialize base array
	ExecutionConfig config = new ExecutionConfig();
	config.enableForceKryo();
	config.registerTypeWithKryoSerializer(MyObj.class, new MyObjSerializer());
	final BaseArraySerializer serializer = createSerializerWithConfig(config);

	MyObj inputObj = new MyObj(114514, 1919810);
	BaseArray inputArray = new GenericArray(new BinaryGeneric[] {
		new BinaryGeneric<>(inputObj, new KryoSerializer<>(MyObj.class, config))
	}, 1);

	byte[] serialized;
	try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
		serializer.serialize(inputArray, new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	// deserialize base array using restored serializer
	final BaseArraySerializer restoreSerializer =
		(BaseArraySerializer) snapshotAndReconfigure(serializer, () -> createSerializerWithConfig(config));

	BaseArray outputArray;
	try (ByteArrayInputStream in = new ByteArrayInputStream(serialized)) {
		outputArray = restoreSerializer.deserialize(new DataInputViewStreamWrapper(in));
	}

	TypeSerializer restoreEleSer = restoreSerializer.getEleSer();
	assertEquals(serializer.getEleSer(), restoreEleSer);

	MyObj outputObj = BinaryGeneric.getJavaObjectFromBinaryGeneric(
		outputArray.getGeneric(0), new KryoSerializer<>(MyObj.class, config));
	assertEquals(inputObj, outputObj);
}
Example 6
Source File: ArrayDataSerializerTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testExecutionConfigWithKryo() throws Exception {
	// serialize base array
	ExecutionConfig config = new ExecutionConfig();
	config.enableForceKryo();
	config.registerTypeWithKryoSerializer(MyObj.class, new MyObjSerializer());
	final ArrayDataSerializer serializer = createSerializerWithConfig(config);

	MyObj inputObj = new MyObj(114514, 1919810);
	ArrayData inputArray = new GenericArrayData(new RawValueData[] {
		RawValueData.fromObject(inputObj)
	});

	byte[] serialized;
	try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
		serializer.serialize(inputArray, new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	// deserialize base array using restored serializer
	final ArrayDataSerializer restoreSerializer =
		(ArrayDataSerializer) snapshotAndReconfigure(serializer, () -> createSerializerWithConfig(config));

	ArrayData outputArray;
	try (ByteArrayInputStream in = new ByteArrayInputStream(serialized)) {
		outputArray = restoreSerializer.deserialize(new DataInputViewStreamWrapper(in));
	}

	TypeSerializer restoreEleSer = restoreSerializer.getEleSer();
	assertEquals(serializer.getEleSer(), restoreEleSer);

	MyObj outputObj = outputArray
		.<MyObj>getRawValue(0)
		.toObject(new KryoSerializer<>(MyObj.class, config));
	assertEquals(inputObj, outputObj);
}
Example 7
Source File: MapDataSerializerTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Test
public void testExecutionConfigWithKryo() throws Exception {
	// serialize base array
	ExecutionConfig config = new ExecutionConfig();
	config.enableForceKryo();
	config.registerTypeWithKryoSerializer(MyObj.class, new MyObjSerializer());
	final MapDataSerializer serializer = createSerializerWithConfig(config);

	int inputKey = 998244353;
	MyObj inputObj = new MyObj(114514, 1919810);
	Map<Object, Object> javaMap = new HashMap<>();
	javaMap.put(inputKey, RawValueData.fromObject(inputObj));
	MapData inputMap = new GenericMapData(javaMap);

	byte[] serialized;
	try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
		serializer.serialize(inputMap, new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	// deserialize base array using restored serializer
	final MapDataSerializer restoreSerializer =
		(MapDataSerializer) snapshotAndReconfigure(serializer, () -> createSerializerWithConfig(config));

	MapData outputMap;
	try (ByteArrayInputStream in = new ByteArrayInputStream(serialized)) {
		outputMap = restoreSerializer.deserialize(new DataInputViewStreamWrapper(in));
	}

	TypeSerializer restoreKeySer = restoreSerializer.getKeySerializer();
	TypeSerializer restoreValueSer = restoreSerializer.getValueSerializer();
	assertEquals(serializer.getKeySerializer(), restoreKeySer);
	assertEquals(serializer.getValueSerializer(), restoreValueSer);

	Map<Object, Object> outputJavaMap = convertToJavaMap(
		outputMap,
		DataTypes.INT().getLogicalType(),
		DataTypes.RAW(TypeInformation.of(MyObj.class)).getLogicalType());
	MyObj outputObj = ((RawValueData<MyObj>) outputJavaMap.get(inputKey))
		.toObject(new KryoSerializer<>(MyObj.class, config));
	assertEquals(inputObj, outputObj);
}