org.apache.flink.runtime.checkpoint.MasterTriggerRestoreHook Java Examples

The following examples show how to use org.apache.flink.runtime.checkpoint.MasterTriggerRestoreHook. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: MasterHooks.java    From flink with Apache License 2.0 6 votes vote down vote up
private static <T> T deserializeState(MasterState state, MasterTriggerRestoreHook<?> hook) throws FlinkException {
	@SuppressWarnings("unchecked")
	final MasterTriggerRestoreHook<T> typedHook = (MasterTriggerRestoreHook<T>) hook;
	final String id = hook.getIdentifier();

	try {
		final SimpleVersionedSerializer<T> deserializer = typedHook.createCheckpointDataSerializer();
		if (deserializer == null) {
			throw new FlinkException("null serializer for state of hook " + hook.getIdentifier());
		}

		return deserializer.deserialize(state.version(), state.bytes());
	}
	catch (Throwable t) {
		throw new FlinkException("Cannot deserialize state for master hook '" + id + '\'', t);
	}
}
 
Example #2
Source File: MasterHooks.java    From flink with Apache License 2.0 6 votes vote down vote up
private static <T> T deserializeState(MasterState state, MasterTriggerRestoreHook<?> hook) throws FlinkException {
	@SuppressWarnings("unchecked")
	final MasterTriggerRestoreHook<T> typedHook = (MasterTriggerRestoreHook<T>) hook;
	final String id = hook.getIdentifier();

	try {
		final SimpleVersionedSerializer<T> deserializer = typedHook.createCheckpointDataSerializer();
		if (deserializer == null) {
			throw new FlinkException("null serializer for state of hook " + hook.getIdentifier());
		}

		return deserializer.deserialize(state.version(), state.bytes());
	}
	catch (Throwable t) {
		throw new FlinkException("Cannot deserialize state for master hook '" + id + '\'', t);
	}
}
 
Example #3
Source File: MasterHooks.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Triggers all given master hooks and returns state objects for each hook that
 * produced a state.
 *
 * @param hooks The hooks to trigger
 * @param checkpointId The checkpoint ID of the triggering checkpoint
 * @param timestamp The (informational) timestamp for the triggering checkpoint
 * @param executor An executor that can be used for asynchronous I/O calls
 * @param timeout The maximum time that a hook may take to complete
 *
 * @return A list containing all states produced by the hooks
 *
 * @throws FlinkException Thrown, if the hooks throw an exception, or the state+
 *                        deserialization fails.
 */
public static List<MasterState> triggerMasterHooks(
		Collection<MasterTriggerRestoreHook<?>> hooks,
		long checkpointId,
		long timestamp,
		Executor executor,
		Time timeout) throws FlinkException {

	final ArrayList<MasterState> states = new ArrayList<>(hooks.size());

	for (MasterTriggerRestoreHook<?> hook : hooks) {
		MasterState state = triggerHook(hook, checkpointId, timestamp, executor, timeout);
		if (state != null) {
			states.add(state);
		}
	}

	states.trimToSize();
	return states;
}
 
Example #4
Source File: MasterHooks.java    From flink with Apache License 2.0 6 votes vote down vote up
private static <T> void restoreHook(
		final Object state,
		final MasterTriggerRestoreHook<?> hook,
		final long checkpointId) throws FlinkException {

	@SuppressWarnings("unchecked")
	final T typedState = (T) state;

	@SuppressWarnings("unchecked")
	final MasterTriggerRestoreHook<T> typedHook = (MasterTriggerRestoreHook<T>) hook;

	try {
		typedHook.restoreCheckpoint(checkpointId, typedState);
	}
	catch (FlinkException e) {
		throw e;
	}
	catch (Throwable t) {
		// catch all here, including Errors that may come from dependency and classpath issues
		ExceptionUtils.rethrowIfFatalError(t);
		throw new FlinkException("Error while calling restoreCheckpoint on checkpoint hook '"
				+ hook.getIdentifier() + '\'', t);
	}
}
 
Example #5
Source File: MasterHooks.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Resets the master hooks.
 *
 * @param hooks The hooks to reset
 *
 * @throws FlinkException Thrown, if the hooks throw an exception.
 */
public static void reset(
	final Collection<MasterTriggerRestoreHook<?>> hooks,
	final Logger log) throws FlinkException {

	for (MasterTriggerRestoreHook<?> hook : hooks) {
		final String id = hook.getIdentifier();
		try {
			hook.reset();
		}
		catch (Throwable t) {
			ExceptionUtils.rethrowIfFatalErrorOrOOM(t);
			throw new FlinkException("Error while resetting checkpoint master hook '" + id + '\'', t);
		}
	}
}
 
Example #6
Source File: JobCheckpointingSettings.java    From flink with Apache License 2.0 6 votes vote down vote up
public JobCheckpointingSettings(
		List<JobVertexID> verticesToTrigger,
		List<JobVertexID> verticesToAcknowledge,
		List<JobVertexID> verticesToConfirm,
		CheckpointCoordinatorConfiguration checkpointCoordinatorConfiguration,
		@Nullable SerializedValue<StateBackend> defaultStateBackend,
		@Nullable SerializedValue<MasterTriggerRestoreHook.Factory[]> masterHooks) {


	this.verticesToTrigger = requireNonNull(verticesToTrigger);
	this.verticesToAcknowledge = requireNonNull(verticesToAcknowledge);
	this.verticesToConfirm = requireNonNull(verticesToConfirm);
	this.checkpointCoordinatorConfiguration = Preconditions.checkNotNull(checkpointCoordinatorConfiguration);
	this.defaultStateBackend = defaultStateBackend;
	this.masterHooks = masterHooks;
}
 
Example #7
Source File: MasterHooks.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Resets the master hooks.
 *
 * @param hooks The hooks to reset
 *
 * @throws FlinkException Thrown, if the hooks throw an exception.
 */
public static void reset(
	final Collection<MasterTriggerRestoreHook<?>> hooks,
	@SuppressWarnings("unused") final Logger log) throws FlinkException {

	for (MasterTriggerRestoreHook<?> hook : hooks) {
		final String id = hook.getIdentifier();
		try {
			hook.reset();
		}
		catch (Throwable t) {
			ExceptionUtils.rethrowIfFatalErrorOrOOM(t);
			throw new FlinkException("Error while resetting checkpoint master hook '" + id + '\'', t);
		}
	}
}
 
Example #8
Source File: MasterHooks.java    From flink with Apache License 2.0 6 votes vote down vote up
private static <T> void restoreHook(
		final Object state,
		final MasterTriggerRestoreHook<?> hook,
		final long checkpointId) throws FlinkException {

	@SuppressWarnings("unchecked")
	final T typedState = (T) state;

	@SuppressWarnings("unchecked")
	final MasterTriggerRestoreHook<T> typedHook = (MasterTriggerRestoreHook<T>) hook;

	try {
		typedHook.restoreCheckpoint(checkpointId, typedState);
	}
	catch (FlinkException e) {
		throw e;
	}
	catch (Throwable t) {
		// catch all here, including Errors that may come from dependency and classpath issues
		ExceptionUtils.rethrowIfFatalError(t);
		throw new FlinkException("Error while calling restoreCheckpoint on checkpoint hook '"
				+ hook.getIdentifier() + '\'', t);
	}
}
 
Example #9
Source File: JobCheckpointingSettings.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public JobCheckpointingSettings(
		List<JobVertexID> verticesToTrigger,
		List<JobVertexID> verticesToAcknowledge,
		List<JobVertexID> verticesToConfirm,
		CheckpointCoordinatorConfiguration checkpointCoordinatorConfiguration,
		@Nullable SerializedValue<StateBackend> defaultStateBackend,
		@Nullable SerializedValue<MasterTriggerRestoreHook.Factory[]> masterHooks) {


	this.verticesToTrigger = requireNonNull(verticesToTrigger);
	this.verticesToAcknowledge = requireNonNull(verticesToAcknowledge);
	this.verticesToConfirm = requireNonNull(verticesToConfirm);
	this.checkpointCoordinatorConfiguration = Preconditions.checkNotNull(checkpointCoordinatorConfiguration);
	this.defaultStateBackend = defaultStateBackend;
	this.masterHooks = masterHooks;
}
 
Example #10
Source File: JobCheckpointingSettings.java    From flink with Apache License 2.0 6 votes vote down vote up
public JobCheckpointingSettings(
		List<JobVertexID> verticesToTrigger,
		List<JobVertexID> verticesToAcknowledge,
		List<JobVertexID> verticesToConfirm,
		CheckpointCoordinatorConfiguration checkpointCoordinatorConfiguration,
		@Nullable SerializedValue<StateBackend> defaultStateBackend,
		@Nullable SerializedValue<MasterTriggerRestoreHook.Factory[]> masterHooks) {


	this.verticesToTrigger = requireNonNull(verticesToTrigger);
	this.verticesToAcknowledge = requireNonNull(verticesToAcknowledge);
	this.verticesToConfirm = requireNonNull(verticesToConfirm);
	this.checkpointCoordinatorConfiguration = Preconditions.checkNotNull(checkpointCoordinatorConfiguration);
	this.defaultStateBackend = defaultStateBackend;
	this.masterHooks = masterHooks;
}
 
Example #11
Source File: MasterHooks.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private static <T> void restoreHook(
		final Object state,
		final MasterTriggerRestoreHook<?> hook,
		final long checkpointId) throws FlinkException {

	@SuppressWarnings("unchecked")
	final T typedState = (T) state;

	@SuppressWarnings("unchecked")
	final MasterTriggerRestoreHook<T> typedHook = (MasterTriggerRestoreHook<T>) hook;

	try {
		typedHook.restoreCheckpoint(checkpointId, typedState);
	}
	catch (FlinkException e) {
		throw e;
	}
	catch (Throwable t) {
		// catch all here, including Errors that may come from dependency and classpath issues
		ExceptionUtils.rethrowIfFatalError(t);
		throw new FlinkException("Error while calling restoreCheckpoint on checkpoint hook '"
				+ hook.getIdentifier() + '\'', t);
	}
}
 
Example #12
Source File: MasterHooks.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private static <T> T deserializeState(MasterState state, MasterTriggerRestoreHook<?> hook) throws FlinkException {
	@SuppressWarnings("unchecked")
	final MasterTriggerRestoreHook<T> typedHook = (MasterTriggerRestoreHook<T>) hook;
	final String id = hook.getIdentifier();

	try {
		final SimpleVersionedSerializer<T> deserializer = typedHook.createCheckpointDataSerializer();
		if (deserializer == null) {
			throw new FlinkException("null serializer for state of hook " + hook.getIdentifier());
		}

		return deserializer.deserialize(state.version(), state.bytes());
	}
	catch (Throwable t) {
		throw new FlinkException("Cannot deserialize state for master hook '" + id + '\'', t);
	}
}
 
Example #13
Source File: MasterHooks.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Triggers all given master hooks and returns state objects for each hook that
 * produced a state.
 *
 * @param hooks The hooks to trigger
 * @param checkpointId The checkpoint ID of the triggering checkpoint
 * @param timestamp The (informational) timestamp for the triggering checkpoint
 * @param executor An executor that can be used for asynchronous I/O calls
 * @param timeout The maximum time that a hook may take to complete
 *
 * @return A list containing all states produced by the hooks
 *
 * @throws FlinkException Thrown, if the hooks throw an exception, or the state+
 *                        deserialization fails.
 */
public static List<MasterState> triggerMasterHooks(
		Collection<MasterTriggerRestoreHook<?>> hooks,
		long checkpointId,
		long timestamp,
		Executor executor,
		Time timeout) throws FlinkException {

	final ArrayList<MasterState> states = new ArrayList<>(hooks.size());

	for (MasterTriggerRestoreHook<?> hook : hooks) {
		MasterState state = triggerHook(hook, checkpointId, timestamp, executor, timeout);
		if (state != null) {
			states.add(state);
		}
	}

	states.trimToSize();
	return states;
}
 
Example #14
Source File: MasterHooks.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Resets the master hooks.
 *
 * @param hooks The hooks to reset
 *
 * @throws FlinkException Thrown, if the hooks throw an exception.
 */
public static void reset(
	final Collection<MasterTriggerRestoreHook<?>> hooks,
	final Logger log) throws FlinkException {

	for (MasterTriggerRestoreHook<?> hook : hooks) {
		final String id = hook.getIdentifier();
		try {
			hook.reset();
		}
		catch (Throwable t) {
			ExceptionUtils.rethrowIfFatalErrorOrOOM(t);
			throw new FlinkException("Error while resetting checkpoint master hook '" + id + '\'', t);
		}
	}
}
 
Example #15
Source File: MasterHooks.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Closes the master hooks.
 *
 * @param hooks The hooks to close
 *
 * @throws FlinkException Thrown, if the hooks throw an exception.
 */
public static void close(
	final Collection<MasterTriggerRestoreHook<?>> hooks,
	final Logger log) throws FlinkException {

	for (MasterTriggerRestoreHook<?> hook : hooks) {
		try {
			hook.close();
		}
		catch (Throwable t) {
			log.warn("Failed to cleanly close a checkpoint master hook (" + hook.getIdentifier() + ")", t);
		}
	}
}
 
Example #16
Source File: MasterHooks.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Closes the master hooks.
 *
 * @param hooks The hooks to close
 */
public static void close(
	final Collection<MasterTriggerRestoreHook<?>> hooks,
	final Logger log) {

	for (MasterTriggerRestoreHook<?> hook : hooks) {
		try {
			hook.close();
		}
		catch (Throwable t) {
			log.warn("Failed to cleanly close a checkpoint master hook (" + hook.getIdentifier() + ")", t);
		}
	}
}
 
Example #17
Source File: MasterHooks.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Wraps a hook such that the user-code classloader is applied when the hook is invoked.
 * @param hook the hook to wrap
 * @param userClassLoader the classloader to use
 */
public static <T> MasterTriggerRestoreHook<T> wrapHook(
		MasterTriggerRestoreHook<T> hook,
		ClassLoader userClassLoader) {

	return new WrappedMasterHook<>(hook, userClassLoader);
}
 
Example #18
Source File: MasterHooks.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Wraps a hook such that the user-code classloader is applied when the hook is invoked.
 * @param hook the hook to wrap
 * @param userClassLoader the classloader to use
 */
public static <T> MasterTriggerRestoreHook<T> wrapHook(
		MasterTriggerRestoreHook<T> hook,
		ClassLoader userClassLoader) {

	return new WrappedMasterHook<>(hook, userClassLoader);
}
 
Example #19
Source File: MasterHooks.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Closes the master hooks.
 *
 * @param hooks The hooks to close
 *
 * @throws FlinkException Thrown, if the hooks throw an exception.
 */
public static void close(
	final Collection<MasterTriggerRestoreHook<?>> hooks,
	final Logger log) throws FlinkException {

	for (MasterTriggerRestoreHook<?> hook : hooks) {
		try {
			hook.close();
		}
		catch (Throwable t) {
			log.warn("Failed to cleanly close a checkpoint master hook (" + hook.getIdentifier() + ")", t);
		}
	}
}
 
Example #20
Source File: MasterHooks.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Wraps a hook such that the user-code classloader is applied when the hook is invoked.
 * @param hook the hook to wrap
 * @param userClassLoader the classloader to use
 */
public static <T> MasterTriggerRestoreHook<T> wrapHook(
		MasterTriggerRestoreHook<T> hook,
		ClassLoader userClassLoader) {

	return new WrappedMasterHook<>(hook, userClassLoader);
}
 
Example #21
Source File: WithMasterCheckpointHookConfigTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * This test creates a program with 4 sources (2 with master hooks, 2 without).
 * The resulting job graph must have 2 configured master hooks.
 */
@Test
public void testHookConfiguration() throws Exception {
	// create some sources some of which configure master hooks
	final TestSource source1 = new TestSource();
	final TestSourceWithHook source2 = new TestSourceWithHook("foo");
	final TestSource source3 = new TestSource();
	final TestSourceWithHook source4 = new TestSourceWithHook("bar");

	final MapFunction<String, String> identity = new Identity<>();
	final IdentityWithHook<String> identityWithHook1 = new IdentityWithHook<>("apple");
	final IdentityWithHook<String> identityWithHook2 = new IdentityWithHook<>("orange");

	final Set<MasterTriggerRestoreHook<?>> hooks = new HashSet<MasterTriggerRestoreHook<?>>(asList(
			source2.createMasterTriggerRestoreHook(),
			source4.createMasterTriggerRestoreHook(),
			identityWithHook1.createMasterTriggerRestoreHook(),
			identityWithHook2.createMasterTriggerRestoreHook()));

	// we can instantiate a local environment here, because we never actually execute something
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
	env.enableCheckpointing(500);

	env
		.addSource(source1).map(identity)
		.union(env.addSource(source2).map(identity))
		.union(env.addSource(source3).map(identityWithHook1))
		.union(env.addSource(source4).map(identityWithHook2))
		.addSink(new DiscardingSink<String>());

	final JobGraph jg = env.getStreamGraph().getJobGraph();

	SerializedValue<Factory[]> serializedConfiguredHooks = jg.getCheckpointingSettings().getMasterHooks();
	assertNotNull(serializedConfiguredHooks);

	Factory[] configuredHooks = serializedConfiguredHooks.deserializeValue(getClass().getClassLoader());
	assertEquals(hooks.size(), configuredHooks.length);

	// check that all hooks are contained and exist exactly once
	for (Factory f : configuredHooks) {
		MasterTriggerRestoreHook<?> hook = f.create();
		assertTrue(hooks.remove(hook));
	}
	assertTrue(hooks.isEmpty());
}
 
Example #22
Source File: WithMasterCheckpointHookConfigTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * This test creates a program with 4 sources (2 with master hooks, 2 without).
 * The resulting job graph must have 2 configured master hooks.
 */
@Test
public void testHookConfiguration() throws Exception {
	// create some sources some of which configure master hooks
	final TestSource source1 = new TestSource();
	final TestSourceWithHook source2 = new TestSourceWithHook("foo");
	final TestSource source3 = new TestSource();
	final TestSourceWithHook source4 = new TestSourceWithHook("bar");

	final MapFunction<String, String> identity = new Identity<>();
	final IdentityWithHook<String> identityWithHook1 = new IdentityWithHook<>("apple");
	final IdentityWithHook<String> identityWithHook2 = new IdentityWithHook<>("orange");

	final Set<MasterTriggerRestoreHook<?>> hooks = new HashSet<MasterTriggerRestoreHook<?>>(asList(
			source2.createMasterTriggerRestoreHook(),
			source4.createMasterTriggerRestoreHook(),
			identityWithHook1.createMasterTriggerRestoreHook(),
			identityWithHook2.createMasterTriggerRestoreHook()));

	// we can instantiate a local environment here, because we never actually execute something
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
	env.enableCheckpointing(500);

	env
		.addSource(source1).map(identity)
		.union(env.addSource(source2).map(identity))
		.union(env.addSource(source3).map(identityWithHook1))
		.union(env.addSource(source4).map(identityWithHook2))
		.addSink(new DiscardingSink<String>());

	final JobGraph jg = env.getStreamGraph().getJobGraph();

	SerializedValue<Factory[]> serializedConfiguredHooks = jg.getCheckpointingSettings().getMasterHooks();
	assertNotNull(serializedConfiguredHooks);

	Factory[] configuredHooks = serializedConfiguredHooks.deserializeValue(getClass().getClassLoader());
	assertEquals(hooks.size(), configuredHooks.length);

	// check that all hooks are contained and exist exactly once
	for (Factory f : configuredHooks) {
		MasterTriggerRestoreHook<?> hook = f.create();
		assertTrue(hooks.remove(hook));
	}
	assertTrue(hooks.isEmpty());
}
 
Example #23
Source File: SourceExternalCheckpointTriggerTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public MasterTriggerRestoreHook<Object> createMasterTriggerRestoreHook() {
	// not relevant in this test
	throw new UnsupportedOperationException("not implemented");
}
 
Example #24
Source File: FunctionMasterCheckpointHookFactory.java    From flink with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public <V> MasterTriggerRestoreHook<V> create() {
	return (MasterTriggerRestoreHook<V>) creator.createMasterTriggerRestoreHook();
}
 
Example #25
Source File: ArchivedExecutionGraphTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@BeforeClass
public static void setupExecutionGraph() throws Exception {
	// -------------------------------------------------------------------------------------------------------------
	// Setup
	// -------------------------------------------------------------------------------------------------------------

	JobVertexID v1ID = new JobVertexID();
	JobVertexID v2ID = new JobVertexID();

	JobVertex v1 = new JobVertex("v1", v1ID);
	JobVertex v2 = new JobVertex("v2", v2ID);

	v1.setParallelism(1);
	v2.setParallelism(2);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);

	JobGraph jobGraph = new JobGraph(v1, v2);
	ExecutionConfig config = new ExecutionConfig();

	config.setExecutionMode(ExecutionMode.BATCH_FORCED);
	config.setRestartStrategy(new RestartStrategies.NoRestartStrategyConfiguration());
	config.setParallelism(4);
	config.enableObjectReuse();
	config.setGlobalJobParameters(new TestJobParameters());

	jobGraph.setExecutionConfig(config);

	runtimeGraph = TestingExecutionGraphBuilder
		.newBuilder()
		.setJobGraph(jobGraph)
		.build();

	runtimeGraph.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());

	List<ExecutionJobVertex> jobVertices = new ArrayList<>();
	jobVertices.add(runtimeGraph.getJobVertex(v1ID));
	jobVertices.add(runtimeGraph.getJobVertex(v2ID));

	CheckpointStatsTracker statsTracker = new CheckpointStatsTracker(
			0,
			jobVertices,
			mock(CheckpointCoordinatorConfiguration.class),
			new UnregisteredMetricsGroup());

	CheckpointCoordinatorConfiguration chkConfig = new CheckpointCoordinatorConfiguration(
		100,
		100,
		100,
		1,
		CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
		true,
		false,
		false,
		0);

	runtimeGraph.enableCheckpointing(
		chkConfig,
		Collections.<ExecutionJobVertex>emptyList(),
		Collections.<ExecutionJobVertex>emptyList(),
		Collections.<ExecutionJobVertex>emptyList(),
		Collections.<MasterTriggerRestoreHook<?>>emptyList(),
		new StandaloneCheckpointIDCounter(),
		new StandaloneCompletedCheckpointStore(1),
		new MemoryStateBackend(),
		statsTracker);

	runtimeGraph.setJsonPlan("{}");

	runtimeGraph.getJobVertex(v2ID).getTaskVertices()[0].getCurrentExecutionAttempt().fail(new RuntimeException("This exception was thrown on purpose."));
}
 
Example #26
Source File: JobCheckpointingSettings.java    From flink with Apache License 2.0 4 votes vote down vote up
@Nullable
public SerializedValue<MasterTriggerRestoreHook.Factory[]> getMasterHooks() {
	return masterHooks;
}
 
Example #27
Source File: MasterHooks.java    From flink with Apache License 2.0 4 votes vote down vote up
WrappedMasterHook(MasterTriggerRestoreHook<T> hook, ClassLoader userClassLoader) {
	this.hook = Preconditions.checkNotNull(hook);
	this.userClassLoader = Preconditions.checkNotNull(userClassLoader);
}
 
Example #28
Source File: MasterHooks.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
WrappedMasterHook(MasterTriggerRestoreHook<T> hook, ClassLoader userClassLoader) {
	this.hook = Preconditions.checkNotNull(hook);
	this.userClassLoader = Preconditions.checkNotNull(userClassLoader);
}
 
Example #29
Source File: JobCheckpointingSettings.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Nullable
public SerializedValue<MasterTriggerRestoreHook.Factory[]> getMasterHooks() {
	return masterHooks;
}
 
Example #30
Source File: ArchivedExecutionGraphTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@BeforeClass
public static void setupExecutionGraph() throws Exception {
	// -------------------------------------------------------------------------------------------------------------
	// Setup
	// -------------------------------------------------------------------------------------------------------------

	JobVertexID v1ID = new JobVertexID();
	JobVertexID v2ID = new JobVertexID();

	JobVertex v1 = new JobVertex("v1", v1ID);
	JobVertex v2 = new JobVertex("v2", v2ID);

	v1.setParallelism(1);
	v2.setParallelism(2);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);

	List<JobVertex> vertices = new ArrayList<>(Arrays.asList(v1, v2));

	ExecutionConfig config = new ExecutionConfig();

	config.setExecutionMode(ExecutionMode.BATCH_FORCED);
	config.setRestartStrategy(new RestartStrategies.NoRestartStrategyConfiguration());
	config.setParallelism(4);
	config.enableObjectReuse();
	config.setGlobalJobParameters(new TestJobParameters());

	runtimeGraph = new ExecutionGraph(
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		new JobID(),
		"test job",
		new Configuration(),
		new SerializedValue<>(config),
		AkkaUtils.getDefaultTimeout(),
		new NoRestartStrategy(),
		mock(SlotProvider.class));

	runtimeGraph.start(TestingComponentMainThreadExecutorServiceAdapter.forMainThread());

	runtimeGraph.attachJobGraph(vertices);

	List<ExecutionJobVertex> jobVertices = new ArrayList<>();
	jobVertices.add(runtimeGraph.getJobVertex(v1ID));
	jobVertices.add(runtimeGraph.getJobVertex(v2ID));

	CheckpointStatsTracker statsTracker = new CheckpointStatsTracker(
			0,
			jobVertices,
			mock(CheckpointCoordinatorConfiguration.class),
			new UnregisteredMetricsGroup());

	runtimeGraph.enableCheckpointing(
		100,
		100,
		100,
		1,
		CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
		Collections.<ExecutionJobVertex>emptyList(),
		Collections.<ExecutionJobVertex>emptyList(),
		Collections.<ExecutionJobVertex>emptyList(),
		Collections.<MasterTriggerRestoreHook<?>>emptyList(),
		new StandaloneCheckpointIDCounter(),
		new StandaloneCompletedCheckpointStore(1),
		new MemoryStateBackend(),
		statsTracker);

	runtimeGraph.setJsonPlan("{}");

	runtimeGraph.getJobVertex(v2ID).getTaskVertices()[0].getCurrentExecutionAttempt().fail(new RuntimeException("This exception was thrown on purpose."));
}