Java Code Examples for org.apache.flink.api.common.ExecutionConfig#registerKryoType()

The following examples show how to use org.apache.flink.api.common.ExecutionConfig#registerKryoType() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Test
public void testValueStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = new StreamingRuntimeContext(
			createDescriptorCapturingMockOp(descriptorCapture, config),
			createMockEnvironment(),
			Collections.<String, Accumulator<?, ?>>emptyMap());

	ValueStateDescriptor<TaskInfo> descr = new ValueStateDescriptor<>("name", TaskInfo.class);
	context.getState(descr);

	StateDescriptor<?, ?> descrIntercepted = (StateDescriptor<?, ?>) descriptorCapture.get();
	TypeSerializer<?> serializer = descrIntercepted.getSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(serializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 
Example 2
@Test
public void testAggregatingStateInstantiation() throws Exception {
	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = createRuntimeContext(descriptorCapture, config);

	@SuppressWarnings("unchecked")
	AggregateFunction<String, TaskInfo, String> aggregate = (AggregateFunction<String, TaskInfo, String>) mock(AggregateFunction.class);

	AggregatingStateDescriptor<String, TaskInfo, String> descr =
			new AggregatingStateDescriptor<>("name", aggregate, TaskInfo.class);

	context.getAggregatingState(descr);

	AggregatingStateDescriptor<?, ?, ?> descrIntercepted = (AggregatingStateDescriptor<?, ?, ?>) descriptorCapture.get();
	TypeSerializer<?> serializer = descrIntercepted.getSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(serializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 
Example 3
Source Project: flink   File: StateDescriptorTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testInitializeSerializerAfterSerializationWithCustomConfig() throws Exception {
	// guard our test assumptions.
	assertEquals("broken test assumption", -1,
			new KryoSerializer<>(String.class, new ExecutionConfig()).getKryo()
					.getRegistration(File.class).getId());

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(File.class);

	final TestStateDescriptor<Path> original = new TestStateDescriptor<>("test", Path.class);
	TestStateDescriptor<Path> clone = CommonTestUtils.createCopySerializable(original);

	clone.initializeSerializerUnlessSet(config);

	// serialized one (later initialized) carries the registration
	assertTrue(((KryoSerializer<?>) clone.getSerializer()).getKryo()
			.getRegistration(File.class).getId() > 0);
}
 
Example 4
@Test
public void testListStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = new StreamingRuntimeContext(
			createDescriptorCapturingMockOp(descriptorCapture, config),
			createMockEnvironment(),
			Collections.<String, Accumulator<?, ?>>emptyMap());

	ListStateDescriptor<TaskInfo> descr = new ListStateDescriptor<>("name", TaskInfo.class);
	context.getListState(descr);

	ListStateDescriptor<?> descrIntercepted = (ListStateDescriptor<?>) descriptorCapture.get();
	TypeSerializer<?> serializer = descrIntercepted.getSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(serializer instanceof ListSerializer);

	TypeSerializer<?> elementSerializer = descrIntercepted.getElementSerializer();
	assertTrue(elementSerializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) elementSerializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 
Example 5
@Test
public void testMapStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = new StreamingRuntimeContext(
			createDescriptorCapturingMockOp(descriptorCapture, config),
			createMockEnvironment(),
			Collections.<String, Accumulator<?, ?>>emptyMap());

	MapStateDescriptor<String, TaskInfo> descr =
			new MapStateDescriptor<>("name", String.class, TaskInfo.class);

	context.getMapState(descr);

	MapStateDescriptor<?, ?> descrIntercepted = (MapStateDescriptor<?, ?>) descriptorCapture.get();
	TypeSerializer<?> valueSerializer = descrIntercepted.getValueSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(valueSerializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) valueSerializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 
Example 6
@Benchmark
@OperationsPerInvocation(value = SerializationFrameworkMiniBenchmarks.RECORDS_PER_INVOCATION)
public void serializerKryo(FlinkEnvironmentContext context) throws Exception {
	StreamExecutionEnvironment env = context.env;
	env.setParallelism(4);
	ExecutionConfig executionConfig = env.getConfig();
	executionConfig.enableForceKryo();
	executionConfig.registerKryoType(MyPojo.class);
	executionConfig.registerKryoType(MyOperation.class);

	env.addSource(new PojoSource(RECORDS_PER_INVOCATION, 10))
			.rebalance()
			.addSink(new DiscardingSink<>());

	env.execute();
}
 
Example 7
Source Project: flink   File: StateDescriptorTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testInitializeSerializerAfterSerializationWithCustomConfig() throws Exception {
	// guard our test assumptions.
	assertEquals("broken test assumption", -1,
			new KryoSerializer<>(String.class, new ExecutionConfig()).getKryo()
					.getRegistration(File.class).getId());

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(File.class);

	final TestStateDescriptor<Path> original = new TestStateDescriptor<>("test", Path.class);
	TestStateDescriptor<Path> clone = CommonTestUtils.createCopySerializable(original);

	clone.initializeSerializerUnlessSet(config);

	// serialized one (later initialized) carries the registration
	assertTrue(((KryoSerializer<?>) clone.getSerializer()).getKryo()
			.getRegistration(File.class).getId() > 0);
}
 
Example 8
@Test
public void testValueStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = new StreamingRuntimeContext(
			createDescriptorCapturingMockOp(descriptorCapture, config),
			createMockEnvironment(),
			Collections.<String, Accumulator<?, ?>>emptyMap());

	ValueStateDescriptor<TaskInfo> descr = new ValueStateDescriptor<>("name", TaskInfo.class);
	context.getState(descr);

	StateDescriptor<?, ?> descrIntercepted = (StateDescriptor<?, ?>) descriptorCapture.get();
	TypeSerializer<?> serializer = descrIntercepted.getSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(serializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 
Example 9
@Test
public void testListStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = createRuntimeContext(descriptorCapture, config);

	ListStateDescriptor<TaskInfo> descr = new ListStateDescriptor<>("name", TaskInfo.class);
	context.getListState(descr);

	ListStateDescriptor<?> descrIntercepted = (ListStateDescriptor<?>) descriptorCapture.get();
	TypeSerializer<?> serializer = descrIntercepted.getSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(serializer instanceof ListSerializer);

	TypeSerializer<?> elementSerializer = descrIntercepted.getElementSerializer();
	assertTrue(elementSerializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) elementSerializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 
Example 10
private static ExecutionConfig registerClassThatIsNotInClassPath(ClassLoader tempClassLoader) {
	Object objectForClassNotInClassPath =
		CommonTestUtils.createObjectForClassNotInClassPath(tempClassLoader);

	ExecutionConfig conf = new ExecutionConfig();
	conf.registerKryoType(objectForClassNotInClassPath.getClass());
	return conf;
}
 
Example 11
@Test
public void testReducingStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = new StreamingRuntimeContext(
			createDescriptorCapturingMockOp(descriptorCapture, config),
			createMockEnvironment(),
			Collections.<String, Accumulator<?, ?>>emptyMap());

	@SuppressWarnings("unchecked")
	ReduceFunction<TaskInfo> reducer = (ReduceFunction<TaskInfo>) mock(ReduceFunction.class);

	ReducingStateDescriptor<TaskInfo> descr =
			new ReducingStateDescriptor<>("name", reducer, TaskInfo.class);

	context.getReducingState(descr);

	StateDescriptor<?, ?> descrIntercepted = (StateDescriptor<?, ?>) descriptorCapture.get();
	TypeSerializer<?> serializer = descrIntercepted.getSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(serializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 
Example 12
Source Project: flink   File: KryoSerializerUpgradeTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public TypeSerializer<Animal> createPriorSerializer() {
	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.registerKryoType(Dog.class);
	executionConfig.registerKryoType(Cat.class);
	executionConfig.registerKryoType(Parrot.class);

	return new KryoSerializer<>(Animal.class, executionConfig);
}
 
Example 13
Source Project: flink   File: KryoSerializerUpgradeTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public TypeSerializer<Animal> createUpgradedSerializer() {
	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.registerKryoType(DummyClassOne.class);
	executionConfig.registerTypeWithKryoSerializer(
			Dog.class,
			KryoPojosForMigrationTests.DogV2KryoSerializer.class);
	executionConfig.registerKryoType(DummyClassTwo.class);
	executionConfig.registerKryoType(Cat.class);
	executionConfig.registerTypeWithKryoSerializer(
			Parrot.class,
			KryoPojosForMigrationTests.ParrotKryoSerializer.class);

	return new KryoSerializer<>(Animal.class, executionConfig);
}
 
Example 14
@Test
public void testReducingStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = new StreamingRuntimeContext(
			createDescriptorCapturingMockOp(descriptorCapture, config),
			createMockEnvironment(),
			Collections.<String, Accumulator<?, ?>>emptyMap());

	@SuppressWarnings("unchecked")
	ReduceFunction<TaskInfo> reducer = (ReduceFunction<TaskInfo>) mock(ReduceFunction.class);

	ReducingStateDescriptor<TaskInfo> descr =
			new ReducingStateDescriptor<>("name", reducer, TaskInfo.class);

	context.getReducingState(descr);

	StateDescriptor<?, ?> descrIntercepted = (StateDescriptor<?, ?>) descriptorCapture.get();
	TypeSerializer<?> serializer = descrIntercepted.getSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(serializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 
Example 15
@Test
public void testAggregatingStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = new StreamingRuntimeContext(
			createDescriptorCapturingMockOp(descriptorCapture, config),
			createMockEnvironment(),
			Collections.<String, Accumulator<?, ?>>emptyMap());

	@SuppressWarnings("unchecked")
	AggregateFunction<String, TaskInfo, String> aggregate = (AggregateFunction<String, TaskInfo, String>) mock(AggregateFunction.class);

	AggregatingStateDescriptor<String, TaskInfo, String> descr =
			new AggregatingStateDescriptor<>("name", aggregate, TaskInfo.class);

	context.getAggregatingState(descr);

	AggregatingStateDescriptor<?, ?, ?> descrIntercepted = (AggregatingStateDescriptor<?, ?, ?>) descriptorCapture.get();
	TypeSerializer<?> serializer = descrIntercepted.getSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(serializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 
Example 16
Source Project: flink   File: KryoSerializerUpgradeTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public TypeSerializer<Animal> createPriorSerializer() {
	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.registerTypeWithKryoSerializer(
			Dog.class,
			KryoPojosForMigrationTests.DogKryoSerializer.class);
	executionConfig.registerKryoType(Cat.class);
	executionConfig.registerTypeWithKryoSerializer(
			Parrot.class,
			KryoPojosForMigrationTests.ParrotKryoSerializer.class);

	return new KryoSerializer<>(Animal.class, executionConfig);
}
 
Example 17
Source Project: flink   File: KryoClearedBufferTest.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Tests that the kryo output buffer is cleared in case of an exception. Flink uses the
 * EOFException to signal that a buffer is full. In such a case, the record which was tried
 * to be written will be rewritten. Therefore, eventually buffered data of this record has
 * to be cleared.
 */
@Test
public void testOutputBufferedBeingClearedInCaseOfException() throws Exception {
	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.registerTypeWithKryoSerializer(TestRecord.class, new TestRecordSerializer());
	executionConfig.registerKryoType(TestRecord.class);

	KryoSerializer<TestRecord> kryoSerializer = new KryoSerializer<TestRecord>(
		TestRecord.class,
		executionConfig);

	int size = 94;
	int bufferSize = 150;

	TestRecord testRecord = new TestRecord(size);

	TestDataOutputView target = new TestDataOutputView(bufferSize);

	kryoSerializer.serialize(testRecord, target);

	try {
		kryoSerializer.serialize(testRecord, target);
		Assert.fail("Expected an EOFException.");
	} catch(EOFException eofException) {
		// expected exception
		// now the Kryo Output should have been cleared
	}

	TestRecord actualRecord = kryoSerializer.deserialize(
			new DataInputViewStreamWrapper(new ByteArrayInputStream(target.getBuffer())));

	Assert.assertEquals(testRecord, actualRecord);

	target.clear();

	// if the kryo output has been cleared then we can serialize our test record into the target
	// because the target buffer 150 bytes can host one TestRecord (total serialization size 100)
	kryoSerializer.serialize(testRecord, target);

	byte[] buffer = target.getBuffer();
	int counter = 0;

	for (int i = 0; i < buffer.length; i++) {
		if(buffer[i] == 42) {
			counter++;
		}
	}

	Assert.assertEquals(size, counter);
}
 
Example 18
/**
 * Tests that the kryo output buffer is cleared in case of an exception. Flink uses the
 * EOFException to signal that a buffer is full. In such a case, the record which was tried
 * to be written will be rewritten. Therefore, eventually buffered data of this record has
 * to be cleared.
 */
@Test
public void testOutputBufferedBeingClearedInCaseOfException() throws Exception {
	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.registerTypeWithKryoSerializer(TestRecord.class, new TestRecordSerializer());
	executionConfig.registerKryoType(TestRecord.class);

	KryoSerializer<TestRecord> kryoSerializer = new KryoSerializer<TestRecord>(
		TestRecord.class,
		executionConfig);

	int size = 94;
	int bufferSize = 150;

	TestRecord testRecord = new TestRecord(size);

	TestDataOutputView target = new TestDataOutputView(bufferSize);

	kryoSerializer.serialize(testRecord, target);

	try {
		kryoSerializer.serialize(testRecord, target);
		Assert.fail("Expected an EOFException.");
	} catch(EOFException eofException) {
		// expected exception
		// now the Kryo Output should have been cleared
	}

	TestRecord actualRecord = kryoSerializer.deserialize(
			new DataInputViewStreamWrapper(new ByteArrayInputStream(target.getBuffer())));

	Assert.assertEquals(testRecord, actualRecord);

	target.clear();

	// if the kryo output has been cleared then we can serialize our test record into the target
	// because the target buffer 150 bytes can host one TestRecord (total serialization size 100)
	kryoSerializer.serialize(testRecord, target);

	byte[] buffer = target.getBuffer();
	int counter = 0;

	for (int i = 0; i < buffer.length; i++) {
		if(buffer[i] == 42) {
			counter++;
		}
	}

	Assert.assertEquals(size, counter);
}
 
Example 19
@Test
public void testReducingStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = createRuntimeContext(descriptorCapture, config);

	@SuppressWarnings("unchecked")
	ReduceFunction<TaskInfo> reducer = (ReduceFunction<TaskInfo>) mock(ReduceFunction.class);

	ReducingStateDescriptor<TaskInfo> descr =
			new ReducingStateDescriptor<>("name", reducer, TaskInfo.class);

	context.getReducingState(descr);

	StateDescriptor<?, ?> descrIntercepted = (StateDescriptor<?, ?>) descriptorCapture.get();
	TypeSerializer<?> serializer = descrIntercepted.getSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(serializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 
Example 20
@Test
public void testMapStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = createRuntimeContext(descriptorCapture, config);

	MapStateDescriptor<String, TaskInfo> descr =
			new MapStateDescriptor<>("name", String.class, TaskInfo.class);

	context.getMapState(descr);

	MapStateDescriptor<?, ?> descrIntercepted = (MapStateDescriptor<?, ?>) descriptorCapture.get();
	TypeSerializer<?> valueSerializer = descrIntercepted.getValueSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(valueSerializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) valueSerializer).getKryo().getRegistration(Path.class).getId() > 0);
}