Java Code Examples for org.apache.flink.util.SerializedValue#deserializeValue()

void dispatchEventToHandlers(OperatorID operatorID, SerializedValue<OperatorEvent> serializedEvent) throws FlinkException {
	final OperatorEvent evt;
	try {
		evt = serializedEvent.deserializeValue(classLoader);
	}
	catch (IOException | ClassNotFoundException e) {
		throw new FlinkException("Could not deserialize operator event", e);
	}

	final OperatorEventHandler handler = handlers.get(operatorID);
	if (handler != null) {
		handler.handleOperatorEvent(evt);
	}
	else {
		throw new FlinkException("Operator not registered for operator events");
	}
}
 

public void doTestPropagationFromCheckpointConfig(boolean failTaskOnCheckpointErrors) throws Exception {
	StreamExecutionEnvironment streamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment();
	streamExecutionEnvironment.setParallelism(1);
	streamExecutionEnvironment.getCheckpointConfig().setCheckpointInterval(1000);
	streamExecutionEnvironment.getCheckpointConfig().setFailOnCheckpointingErrors(failTaskOnCheckpointErrors);
	streamExecutionEnvironment.addSource(new SourceFunction<Integer>() {

		@Override
		public void run(SourceContext<Integer> ctx) throws Exception {
		}

		@Override
		public void cancel() {
		}

	}).addSink(new DiscardingSink<>());

	StreamGraph streamGraph = streamExecutionEnvironment.getStreamGraph();
	JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(streamGraph);
	SerializedValue<ExecutionConfig> serializedExecutionConfig = jobGraph.getSerializedExecutionConfig();
	ExecutionConfig executionConfig =
		serializedExecutionConfig.deserializeValue(Thread.currentThread().getContextClassLoader());

	Assert.assertEquals(failTaskOnCheckpointErrors, executionConfig.isFailTaskOnCheckpointError());
}
 

public LocalShuffle(int split, RichSinkFunction<T> userSink)
        throws IOException, ClassNotFoundException, IllegalAccessException, NoSuchFieldException
{
    this.sinks = new ArrayList<>(split);
    SerializedValue<RichSinkFunction<T>> serializedValue = new SerializedValue<>(userSink);
    for (int i = 0; i < split; i++) {
        StreamingFileSink<T> sink = (StreamingFileSink<T>) serializedValue.deserializeValue(this.getClass().getClassLoader());
        Field field = StreamingFileSink.class.getDeclaredField("bucketsBuilder");
        field.setAccessible(true);
        StreamingFileSink<T> mockSink = new StreamingFileSink<T>((StreamingFileSink.BulkFormatBuilder<T, ?>) field.get(sink), 0)
        {
            @Override
            public RuntimeContext getRuntimeContext()
            {
                return LocalShuffle.this.getRuntimeContext();
            }
        };
    }
}
 

public static OperatorCoordinatorHolder create(
		SerializedValue<OperatorCoordinator.Provider> serializedProvider,
		ExecutionJobVertex jobVertex,
		ClassLoader classLoader) throws IOException, ClassNotFoundException {

	try (TemporaryClassLoaderContext ignored = TemporaryClassLoaderContext.of(classLoader)) {
		final OperatorCoordinator.Provider provider = serializedProvider.deserializeValue(classLoader);
		final OperatorID opId = provider.getOperatorId();

		final BiFunction<SerializedValue<OperatorEvent>, Integer, CompletableFuture<Acknowledge>> eventSender =
			(serializedEvent, subtask) -> {
				final Execution executionAttempt = jobVertex.getTaskVertices()[subtask].getCurrentExecutionAttempt();
				return executionAttempt.sendOperatorEvent(opId, serializedEvent);
			};

		return create(
				opId,
				provider,
				eventSender,
				jobVertex.getName(),
				jobVertex.getParallelism(),
				jobVertex.getMaxParallelism());
	}
}
 

@Test
public void testSerializationDeserialization() throws Exception {
	final String json = objectMapper.writeValueAsString(new SerializedValue<>(new TestClass()));

	final SerializedValue<TestClass> serializedValue =
		objectMapper.readValue(json, new TypeReference<SerializedValue<TestClass>>() {
		});
	final TestClass deserializedValue =
		serializedValue.deserializeValue(ClassLoader.getSystemClassLoader());

	assertEquals("baz", deserializedValue.foo);
	assertEquals(1, deserializedValue.bar);
}
 

@Override
public void dispatchOperatorEvent(OperatorID operator, SerializedValue<OperatorEvent> event) throws FlinkException {
	try {
		final OperatorEvent opEvent = event.deserializeValue(getUserCodeClassLoader());
		actions.add(opEvent);
	} catch (IOException | ClassNotFoundException e) {
		throw new FlinkException(e);
	}
}
 

@Override
public CompletableFuture<Acknowledge> apply(SerializedValue<OperatorEvent> event, Integer subtask) {
	final OperatorEvent deserializedEvent;
	try {
		deserializedEvent = event.deserializeValue(getClass().getClassLoader());
	} catch (IOException | ClassNotFoundException e) {
		throw new AssertionError(e);
	}
	events.add(new EventWithSubtask(deserializedEvent, subtask));

	return failureCause == null
			? CompletableFuture.completedFuture(Acknowledge.get())
			: FutureUtils.completedExceptionally(failureCause);
}
 

@Test
public void testSerializationDeserialization() throws Exception {
	final String json = objectMapper.writeValueAsString(new SerializedValue<>(new TestClass()));

	final SerializedValue<TestClass> serializedValue =
		objectMapper.readValue(json, new TypeReference<SerializedValue<TestClass>>() {
		});
	final TestClass deserializedValue =
		serializedValue.deserializeValue(ClassLoader.getSystemClassLoader());

	assertEquals("baz", deserializedValue.foo);
	assertEquals(1, deserializedValue.bar);
}
 

@Test
public void testMessageDecodingWithUnavailableClass() throws Exception {
	final ClassLoader systemClassLoader = getClass().getClassLoader();

	final String className = "UserClass";
	final URLClassLoader userClassLoader = ClassLoaderUtils.compileAndLoadJava(
		temporaryFolder.newFolder(),
		className + ".java",
		"import java.io.Serializable;\n"
			+ "public class " + className + " implements Serializable {}");

	RemoteRpcInvocation method = new RemoteRpcInvocation(
		"test",
		new Class<?>[] {
			int.class,
			Class.forName(className, false, userClassLoader)},
		new Object[] {
			1,
			Class.forName(className, false, userClassLoader).newInstance()});

	SerializedValue<RemoteRpcInvocation> serializedMethod = new SerializedValue<>(method);

	expectedException.expect(ClassNotFoundException.class);
	expectedException.expect(
		allOf(
			isA(ClassNotFoundException.class),
			hasProperty("suppressed",
				hasItemInArray(
					allOf(
						isA(ClassNotFoundException.class),
						hasProperty("message",
							containsString("Could not deserialize 1th parameter type of method test(int, ...).")))))));

	RemoteRpcInvocation deserializedMethod = serializedMethod.deserializeValue(systemClassLoader);
	deserializedMethod.getMethodName();

	userClassLoader.close();
}
 

@Override
public CompletableFuture<CoordinationResponse> deliverCoordinationRequestToCoordinator(
		OperatorID operatorId,
		SerializedValue<CoordinationRequest> serializedRequest,
		Time timeout) {
	try {
		CoordinationRequest request = serializedRequest.deserializeValue(userCodeLoader);
		return schedulerNG.deliverCoordinationRequestToCoordinator(operatorId, request);
	} catch (Exception e) {
		return FutureUtils.completedExceptionally(e);
	}
}
 

@Override
public CompletableFuture<Acknowledge> sendOperatorEventToCoordinator(
		final ExecutionAttemptID task,
		final OperatorID operatorID,
		final SerializedValue<OperatorEvent> serializedEvent) {

	try {
		final OperatorEvent evt = serializedEvent.deserializeValue(userCodeLoader);
		schedulerNG.deliverOperatorEventToCoordinator(task, operatorID, evt);
		return CompletableFuture.completedFuture(Acknowledge.get());
	} catch (Exception e) {
		return FutureUtils.completedExceptionally(e);
	}
}
 

@Test
public void testSerializationDeserialization() throws Exception {
	final String json = objectMapper.writeValueAsString(new SerializedValue<>(new TestClass()));

	final SerializedValue<TestClass> serializedValue =
		objectMapper.readValue(json, new TypeReference<SerializedValue<TestClass>>() {
		});
	final TestClass deserializedValue =
		serializedValue.deserializeValue(ClassLoader.getSystemClassLoader());

	assertEquals("baz", deserializedValue.foo);
	assertEquals(1, deserializedValue.bar);
}
 

@Test
public void testMessageDecodingWithUnavailableClass() throws Exception {
	final ClassLoader systemClassLoader = getClass().getClassLoader();

	final String className = "UserClass";
	final URLClassLoader userClassLoader = ClassLoaderUtils.compileAndLoadJava(
		temporaryFolder.newFolder(),
		className + ".java",
		"import java.io.Serializable;\n"
			+ "public class " + className + " implements Serializable {}");

	RemoteRpcInvocation method = new RemoteRpcInvocation(
		"test",
		new Class<?>[] {
			int.class,
			Class.forName(className, false, userClassLoader)},
		new Object[] {
			1,
			Class.forName(className, false, userClassLoader).newInstance()});

	SerializedValue<RemoteRpcInvocation> serializedMethod = new SerializedValue<>(method);

	expectedException.expect(ClassNotFoundException.class);
	expectedException.expect(
		allOf(
			isA(ClassNotFoundException.class),
			hasProperty("suppressed",
				hasItemInArray(
					allOf(
						isA(ClassNotFoundException.class),
						hasProperty("message",
							containsString("Could not deserialize 1th parameter type of method test(int, ...).")))))));

	RemoteRpcInvocation deserializedMethod = serializedMethod.deserializeValue(systemClassLoader);
	deserializedMethod.getMethodName();

	userClassLoader.close();
}
 

@Test
public void testMessageDecodingWithUnavailableClass() throws Exception {
	final ClassLoader systemClassLoader = getClass().getClassLoader();

	final String className = "UserClass";
	final URLClassLoader userClassLoader = ClassLoaderUtils.compileAndLoadJava(
		temporaryFolder.newFolder(),
		className + ".java",
		"import java.io.Serializable;\n"
			+ "public class " + className + " implements Serializable {}");

	RemoteRpcInvocation method = new RemoteRpcInvocation(
		"test",
		new Class<?>[] {
			int.class,
			Class.forName(className, false, userClassLoader)},
		new Object[] {
			1,
			Class.forName(className, false, userClassLoader).newInstance()});

	SerializedValue<RemoteRpcInvocation> serializedMethod = new SerializedValue<>(method);

	expectedException.expect(ClassNotFoundException.class);
	expectedException.expect(
		allOf(
			isA(ClassNotFoundException.class),
			hasProperty("suppressed",
				hasItemInArray(
					allOf(
						isA(ClassNotFoundException.class),
						hasProperty("message",
							containsString("Could not deserialize 1th parameter type of method test(int, ...).")))))));

	RemoteRpcInvocation deserializedMethod = serializedMethod.deserializeValue(systemClassLoader);
	deserializedMethod.getMethodName();

	userClassLoader.close();
}
 

/**
 * This test creates a program with 4 sources (2 with master hooks, 2 without).
 * The resulting job graph must have 2 configured master hooks.
 */
@Test
public void testHookConfiguration() throws Exception {
	// create some sources some of which configure master hooks
	final TestSource source1 = new TestSource();
	final TestSourceWithHook source2 = new TestSourceWithHook("foo");
	final TestSource source3 = new TestSource();
	final TestSourceWithHook source4 = new TestSourceWithHook("bar");

	final MapFunction<String, String> identity = new Identity<>();
	final IdentityWithHook<String> identityWithHook1 = new IdentityWithHook<>("apple");
	final IdentityWithHook<String> identityWithHook2 = new IdentityWithHook<>("orange");

	final Set<MasterTriggerRestoreHook<?>> hooks = new HashSet<MasterTriggerRestoreHook<?>>(asList(
			source2.createMasterTriggerRestoreHook(),
			source4.createMasterTriggerRestoreHook(),
			identityWithHook1.createMasterTriggerRestoreHook(),
			identityWithHook2.createMasterTriggerRestoreHook()));

	// we can instantiate a local environment here, because we never actually execute something
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
	env.enableCheckpointing(500);

	env
		.addSource(source1).map(identity)
		.union(env.addSource(source2).map(identity))
		.union(env.addSource(source3).map(identityWithHook1))
		.union(env.addSource(source4).map(identityWithHook2))
		.addSink(new DiscardingSink<String>());

	final JobGraph jg = env.getStreamGraph().getJobGraph();

	SerializedValue<Factory[]> serializedConfiguredHooks = jg.getCheckpointingSettings().getMasterHooks();
	assertNotNull(serializedConfiguredHooks);

	Factory[] configuredHooks = serializedConfiguredHooks.deserializeValue(getClass().getClassLoader());
	assertEquals(hooks.size(), configuredHooks.length);

	// check that all hooks are contained and exist exactly once
	for (Factory f : configuredHooks) {
		MasterTriggerRestoreHook<?> hook = f.create();
		assertTrue(hooks.remove(hook));
	}
	assertTrue(hooks.isEmpty());
}
 

/**
 * This test creates a program with 4 sources (2 with master hooks, 2 without).
 * The resulting job graph must have 2 configured master hooks.
 */
@Test
public void testHookConfiguration() throws Exception {
	// create some sources some of which configure master hooks
	final TestSource source1 = new TestSource();
	final TestSourceWithHook source2 = new TestSourceWithHook("foo");
	final TestSource source3 = new TestSource();
	final TestSourceWithHook source4 = new TestSourceWithHook("bar");

	final MapFunction<String, String> identity = new Identity<>();
	final IdentityWithHook<String> identityWithHook1 = new IdentityWithHook<>("apple");
	final IdentityWithHook<String> identityWithHook2 = new IdentityWithHook<>("orange");

	final Set<MasterTriggerRestoreHook<?>> hooks = new HashSet<MasterTriggerRestoreHook<?>>(asList(
			source2.createMasterTriggerRestoreHook(),
			source4.createMasterTriggerRestoreHook(),
			identityWithHook1.createMasterTriggerRestoreHook(),
			identityWithHook2.createMasterTriggerRestoreHook()));

	// we can instantiate a local environment here, because we never actually execute something
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
	env.enableCheckpointing(500);

	env
		.addSource(source1).map(identity)
		.union(env.addSource(source2).map(identity))
		.union(env.addSource(source3).map(identityWithHook1))
		.union(env.addSource(source4).map(identityWithHook2))
		.addSink(new DiscardingSink<String>());

	final JobGraph jg = env.getStreamGraph().getJobGraph();

	SerializedValue<Factory[]> serializedConfiguredHooks = jg.getCheckpointingSettings().getMasterHooks();
	assertNotNull(serializedConfiguredHooks);

	Factory[] configuredHooks = serializedConfiguredHooks.deserializeValue(getClass().getClassLoader());
	assertEquals(hooks.size(), configuredHooks.length);

	// check that all hooks are contained and exist exactly once
	for (Factory f : configuredHooks) {
		MasterTriggerRestoreHook<?> hook = f.create();
		assertTrue(hooks.remove(hook));
	}
	assertTrue(hooks.isEmpty());
}
 

/**
 * This test creates a program with 4 sources (2 with master hooks, 2 without).
 * The resulting job graph must have 2 configured master hooks.
 */
@Test
public void testHookConfiguration() throws Exception {
	// create some sources some of which configure master hooks
	final TestSource source1 = new TestSource();
	final TestSourceWithHook source2 = new TestSourceWithHook("foo");
	final TestSource source3 = new TestSource();
	final TestSourceWithHook source4 = new TestSourceWithHook("bar");

	final MapFunction<String, String> identity = new Identity<>();
	final IdentityWithHook<String> identityWithHook1 = new IdentityWithHook<>("apple");
	final IdentityWithHook<String> identityWithHook2 = new IdentityWithHook<>("orange");

	final Set<MasterTriggerRestoreHook<?>> hooks = new HashSet<MasterTriggerRestoreHook<?>>(asList(
			source2.createMasterTriggerRestoreHook(),
			source4.createMasterTriggerRestoreHook(),
			identityWithHook1.createMasterTriggerRestoreHook(),
			identityWithHook2.createMasterTriggerRestoreHook()));

	// we can instantiate a local environment here, because we never actually execute something
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
	env.enableCheckpointing(500);

	env
		.addSource(source1).map(identity)
		.union(env.addSource(source2).map(identity))
		.union(env.addSource(source3).map(identityWithHook1))
		.union(env.addSource(source4).map(identityWithHook2))
		.addSink(new DiscardingSink<String>());

	final JobGraph jg = env.getStreamGraph().getJobGraph();

	SerializedValue<Factory[]> serializedConfiguredHooks = jg.getCheckpointingSettings().getMasterHooks();
	assertNotNull(serializedConfiguredHooks);

	Factory[] configuredHooks = serializedConfiguredHooks.deserializeValue(getClass().getClassLoader());
	assertEquals(hooks.size(), configuredHooks.length);

	// check that all hooks are contained and exist exactly once
	for (Factory f : configuredHooks) {
		MasterTriggerRestoreHook<?> hook = f.create();
		assertTrue(hooks.remove(hook));
	}
	assertTrue(hooks.isEmpty());
}