org.apache.flink.runtime.executiongraph.JobInformation Java Examples

The following examples show how to use org.apache.flink.runtime.executiongraph.JobInformation. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TaskDeploymentDescriptorFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
private TaskDeploymentDescriptorFactory(
		ExecutionAttemptID executionId,
		int attemptNumber,
		MaybeOffloaded<JobInformation> serializedJobInformation,
		MaybeOffloaded<TaskInformation> taskInfo,
		JobID jobID,
		boolean allowUnknownPartitions,
		int subtaskIndex,
		ExecutionEdge[][] inputEdges) {
	this.executionId = executionId;
	this.attemptNumber = attemptNumber;
	this.serializedJobInformation = serializedJobInformation;
	this.taskInfo = taskInfo;
	this.jobID = jobID;
	this.allowUnknownPartitions = allowUnknownPartitions;
	this.subtaskIndex = subtaskIndex;
	this.inputEdges = inputEdges;
}
 
Example #2
Source File: TaskDeploymentDescriptorTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testOffLoadedAndNonOffLoadedPayload() {
	final TaskDeploymentDescriptor taskDeploymentDescriptor = createTaskDeploymentDescriptor(
		new TaskDeploymentDescriptor.NonOffloaded<>(serializedJobInformation),
		new TaskDeploymentDescriptor.Offloaded<>(new PermanentBlobKey()));

	SerializedValue<JobInformation> actualSerializedJobInformation = taskDeploymentDescriptor.getSerializedJobInformation();
	assertThat(actualSerializedJobInformation, is(serializedJobInformation));

	try {
		taskDeploymentDescriptor.getSerializedTaskInformation();
		fail("Expected to fail since the task information should be offloaded.");
	} catch (IllegalStateException expected) {
		// expected
	}
}
 
Example #3
Source File: TaskDeploymentDescriptorTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testOffLoadedAndNonOffLoadedPayload() {
	final TaskDeploymentDescriptor taskDeploymentDescriptor = createTaskDeploymentDescriptor(
		new TaskDeploymentDescriptor.NonOffloaded<>(serializedJobInformation),
		new TaskDeploymentDescriptor.Offloaded<>(new PermanentBlobKey()));

	SerializedValue<JobInformation> actualSerializedJobInformation = taskDeploymentDescriptor.getSerializedJobInformation();
	assertThat(actualSerializedJobInformation, is(serializedJobInformation));

	try {
		taskDeploymentDescriptor.getSerializedTaskInformation();
		fail("Expected to fail since the task information should be offloaded.");
	} catch (IllegalStateException expected) {
		// expected
	}
}
 
Example #4
Source File: TaskDeploymentDescriptorTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testOffLoadedAndNonOffLoadedPayload() {
	final TaskDeploymentDescriptor taskDeploymentDescriptor = createTaskDeploymentDescriptor(
		new TaskDeploymentDescriptor.NonOffloaded<>(serializedJobInformation),
		new TaskDeploymentDescriptor.Offloaded<>(new PermanentBlobKey()));

	SerializedValue<JobInformation> actualSerializedJobInformation = taskDeploymentDescriptor.getSerializedJobInformation();
	assertThat(actualSerializedJobInformation, is(serializedJobInformation));

	try {
		taskDeploymentDescriptor.getSerializedTaskInformation();
		fail("Expected to fail since the task information should be offloaded.");
	} catch (IllegalStateException expected) {
		// expected
	}
}
 
Example #5
Source File: TaskDeploymentDescriptorFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
private TaskDeploymentDescriptorFactory(
		ExecutionAttemptID executionId,
		int attemptNumber,
		MaybeOffloaded<JobInformation> serializedJobInformation,
		MaybeOffloaded<TaskInformation> taskInfo,
		JobID jobID,
		boolean allowUnknownPartitions,
		int subtaskIndex,
		ExecutionEdge[][] inputEdges) {
	this.executionId = executionId;
	this.attemptNumber = attemptNumber;
	this.serializedJobInformation = serializedJobInformation;
	this.taskInfo = taskInfo;
	this.jobID = jobID;
	this.allowUnknownPartitions = allowUnknownPartitions;
	this.subtaskIndex = subtaskIndex;
	this.inputEdges = inputEdges;
}
 
Example #6
Source File: TaskDeploymentDescriptor.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Return the sub task's serialized job information.
 *
 * @return serialized job information (may be <tt>null</tt> before a call to {@link
 * #loadBigData(PermanentBlobService)}).
 */
@Nullable
public SerializedValue<JobInformation> getSerializedJobInformation() {
	if (serializedJobInformation instanceof NonOffloaded) {
		NonOffloaded<JobInformation> jobInformation =
			(NonOffloaded<JobInformation>) serializedJobInformation;
		return jobInformation.serializedValue;
	} else {
		throw new IllegalStateException(
			"Trying to work with offloaded serialized job information.");
	}
}
 
Example #7
Source File: TaskDeploymentDescriptorFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
private static MaybeOffloaded<JobInformation> getSerializedJobInformation(ExecutionGraph executionGraph) {
	Either<SerializedValue<JobInformation>, PermanentBlobKey> jobInformationOrBlobKey =
		executionGraph.getJobInformationOrBlobKey();
	if (jobInformationOrBlobKey.isLeft()) {
		return new TaskDeploymentDescriptor.NonOffloaded<>(jobInformationOrBlobKey.left());
	} else {
		return new TaskDeploymentDescriptor.Offloaded<>(jobInformationOrBlobKey.right());
	}
}
 
Example #8
Source File: TaskDeploymentDescriptor.java    From flink with Apache License 2.0 5 votes vote down vote up
public TaskDeploymentDescriptor(
	JobID jobId,
	MaybeOffloaded<JobInformation> serializedJobInformation,
	MaybeOffloaded<TaskInformation> serializedTaskInformation,
	ExecutionAttemptID executionAttemptId,
	AllocationID allocationId,
	int subtaskIndex,
	int attemptNumber,
	int targetSlotNumber,
	@Nullable JobManagerTaskRestore taskRestore,
	List<ResultPartitionDeploymentDescriptor> resultPartitionDeploymentDescriptors,
	List<InputGateDeploymentDescriptor> inputGateDeploymentDescriptors) {

	this.jobId = Preconditions.checkNotNull(jobId);

	this.serializedJobInformation = Preconditions.checkNotNull(serializedJobInformation);
	this.serializedTaskInformation = Preconditions.checkNotNull(serializedTaskInformation);

	this.executionId = Preconditions.checkNotNull(executionAttemptId);
	this.allocationId = Preconditions.checkNotNull(allocationId);

	Preconditions.checkArgument(0 <= subtaskIndex, "The subtask index must be positive.");
	this.subtaskIndex = subtaskIndex;

	Preconditions.checkArgument(0 <= attemptNumber, "The attempt number must be positive.");
	this.attemptNumber = attemptNumber;

	Preconditions.checkArgument(0 <= targetSlotNumber, "The target slot number must be positive.");
	this.targetSlotNumber = targetSlotNumber;

	this.taskRestore = taskRestore;

	this.producedPartitions = Preconditions.checkNotNull(resultPartitionDeploymentDescriptors);
	this.inputGates = Preconditions.checkNotNull(inputGateDeploymentDescriptors);
}
 
Example #9
Source File: TaskDeploymentDescriptor.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Return the sub task's serialized job information.
 *
 * @return serialized job information (may throw {@link IllegalStateException} if {@link
 * #loadBigData(PermanentBlobService)} is not called beforehand).
 * @throws IllegalStateException If job information is offloaded to BLOB store.
 */
public SerializedValue<JobInformation> getSerializedJobInformation() {
	if (serializedJobInformation instanceof NonOffloaded) {
		NonOffloaded<JobInformation> jobInformation =
			(NonOffloaded<JobInformation>) serializedJobInformation;
		return jobInformation.serializedValue;
	} else {
		throw new IllegalStateException(
			"Trying to work with offloaded serialized job information.");
	}
}
 
Example #10
Source File: TaskDeploymentDescriptorTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Nonnull
private TaskDeploymentDescriptor createTaskDeploymentDescriptor(TaskDeploymentDescriptor.MaybeOffloaded<JobInformation> jobInformation, TaskDeploymentDescriptor.MaybeOffloaded<TaskInformation> taskInformation) {
	return new TaskDeploymentDescriptor(
		jobID,
		jobInformation,
		taskInformation,
		execId,
		allocationId,
		indexInSubtaskGroup,
		attemptNumber,
		targetSlotNumber,
		taskRestore,
		producedResults,
		inputGates);
}
 
Example #11
Source File: TaskDeploymentDescriptorTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Nonnull
private TaskDeploymentDescriptor createTaskDeploymentDescriptor(TaskDeploymentDescriptor.MaybeOffloaded<JobInformation> jobInformation, TaskDeploymentDescriptor.MaybeOffloaded<TaskInformation> taskInformation) {
	return new TaskDeploymentDescriptor(
		jobID,
		jobInformation,
		taskInformation,
		execId,
		allocationId,
		indexInSubtaskGroup,
		attemptNumber,
		targetSlotNumber,
		taskRestore,
		producedResults,
		inputGates);
}
 
Example #12
Source File: TaskDeploymentDescriptor.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Return the sub task's serialized job information.
 *
 * @return serialized job information (may throw {@link IllegalStateException} if {@link
 * #loadBigData(PermanentBlobService)} is not called beforehand).
 * @throws IllegalStateException If job information is offloaded to BLOB store.
 */
public SerializedValue<JobInformation> getSerializedJobInformation() {
	if (serializedJobInformation instanceof NonOffloaded) {
		NonOffloaded<JobInformation> jobInformation =
			(NonOffloaded<JobInformation>) serializedJobInformation;
		return jobInformation.serializedValue;
	} else {
		throw new IllegalStateException(
			"Trying to work with offloaded serialized job information.");
	}
}
 
Example #13
Source File: TaskDeploymentDescriptor.java    From flink with Apache License 2.0 5 votes vote down vote up
public TaskDeploymentDescriptor(
	JobID jobId,
	MaybeOffloaded<JobInformation> serializedJobInformation,
	MaybeOffloaded<TaskInformation> serializedTaskInformation,
	ExecutionAttemptID executionAttemptId,
	AllocationID allocationId,
	int subtaskIndex,
	int attemptNumber,
	int targetSlotNumber,
	@Nullable JobManagerTaskRestore taskRestore,
	Collection<ResultPartitionDeploymentDescriptor> resultPartitionDeploymentDescriptors,
	Collection<InputGateDeploymentDescriptor> inputGateDeploymentDescriptors) {

	this.jobId = Preconditions.checkNotNull(jobId);

	this.serializedJobInformation = Preconditions.checkNotNull(serializedJobInformation);
	this.serializedTaskInformation = Preconditions.checkNotNull(serializedTaskInformation);

	this.executionId = Preconditions.checkNotNull(executionAttemptId);
	this.allocationId = Preconditions.checkNotNull(allocationId);

	Preconditions.checkArgument(0 <= subtaskIndex, "The subtask index must be positive.");
	this.subtaskIndex = subtaskIndex;

	Preconditions.checkArgument(0 <= attemptNumber, "The attempt number must be positive.");
	this.attemptNumber = attemptNumber;

	Preconditions.checkArgument(0 <= targetSlotNumber, "The target slot number must be positive.");
	this.targetSlotNumber = targetSlotNumber;

	this.taskRestore = taskRestore;

	this.producedPartitions = Preconditions.checkNotNull(resultPartitionDeploymentDescriptors);
	this.inputGates = Preconditions.checkNotNull(inputGateDeploymentDescriptors);
}
 
Example #14
Source File: TaskDeploymentDescriptorFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
private static MaybeOffloaded<JobInformation> getSerializedJobInformation(ExecutionGraph executionGraph) {
	Either<SerializedValue<JobInformation>, PermanentBlobKey> jobInformationOrBlobKey =
		executionGraph.getJobInformationOrBlobKey();
	if (jobInformationOrBlobKey.isLeft()) {
		return new TaskDeploymentDescriptor.NonOffloaded<>(jobInformationOrBlobKey.left());
	} else {
		return new TaskDeploymentDescriptor.Offloaded<>(jobInformationOrBlobKey.right());
	}
}
 
Example #15
Source File: TaskDeploymentDescriptor.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public TaskDeploymentDescriptor(
	JobID jobId,
	MaybeOffloaded<JobInformation> serializedJobInformation,
	MaybeOffloaded<TaskInformation> serializedTaskInformation,
	ExecutionAttemptID executionAttemptId,
	AllocationID allocationId,
	int subtaskIndex,
	int attemptNumber,
	int targetSlotNumber,
	@Nullable JobManagerTaskRestore taskRestore,
	Collection<ResultPartitionDeploymentDescriptor> resultPartitionDeploymentDescriptors,
	Collection<InputGateDeploymentDescriptor> inputGateDeploymentDescriptors) {

	this.jobId = Preconditions.checkNotNull(jobId);

	this.serializedJobInformation = Preconditions.checkNotNull(serializedJobInformation);
	this.serializedTaskInformation = Preconditions.checkNotNull(serializedTaskInformation);

	this.executionId = Preconditions.checkNotNull(executionAttemptId);
	this.allocationId = Preconditions.checkNotNull(allocationId);

	Preconditions.checkArgument(0 <= subtaskIndex, "The subtask index must be positive.");
	this.subtaskIndex = subtaskIndex;

	Preconditions.checkArgument(0 <= attemptNumber, "The attempt number must be positive.");
	this.attemptNumber = attemptNumber;

	Preconditions.checkArgument(0 <= targetSlotNumber, "The target slot number must be positive.");
	this.targetSlotNumber = targetSlotNumber;

	this.taskRestore = taskRestore;

	this.producedPartitions = Preconditions.checkNotNull(resultPartitionDeploymentDescriptors);
	this.inputGates = Preconditions.checkNotNull(inputGateDeploymentDescriptors);
}
 
Example #16
Source File: TaskDeploymentDescriptorTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Nonnull
private TaskDeploymentDescriptor createTaskDeploymentDescriptor(TaskDeploymentDescriptor.MaybeOffloaded<JobInformation> jobInformation, TaskDeploymentDescriptor.MaybeOffloaded<TaskInformation> taskInformation) {
	return new TaskDeploymentDescriptor(
		jobID,
		jobInformation,
		taskInformation,
		execId,
		allocationId,
		indexInSubtaskGroup,
		attemptNumber,
		targetSlotNumber,
		taskRestore,
		producedResults,
		inputGates);
}
 
Example #17
Source File: TestTaskBuilder.java    From flink with Apache License 2.0 4 votes vote down vote up
public Task build() throws Exception {
	final JobVertexID jobVertexId = new JobVertexID();

	final SerializedValue<ExecutionConfig> serializedExecutionConfig = new SerializedValue<>(executionConfig);

	final JobInformation jobInformation = new JobInformation(
		jobId,
		"Test Job",
		serializedExecutionConfig,
		new Configuration(),
		requiredJarFileBlobKeys,
		Collections.emptyList());

	final TaskInformation taskInformation = new TaskInformation(
		jobVertexId,
		"Test Task",
		1,
		1,
		invokable.getName(),
		taskConfig);

	final TaskMetricGroup taskMetricGroup = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();

	return new Task(
		jobInformation,
		taskInformation,
		executionAttemptId,
		allocationID,
		0,
		0,
		resultPartitions,
		inputGates,
		0,
		MemoryManagerBuilder.newBuilder().setMemorySize(1024 * 1024).build(),
		mock(IOManager.class),
		shuffleEnvironment,
		kvStateService,
		new BroadcastVariableManager(),
		new TaskEventDispatcher(),
		externalResourceInfoProvider,
		new TestTaskStateManager(),
		taskManagerActions,
		new MockInputSplitProvider(),
		new TestCheckpointResponder(),
		new NoOpTaskOperatorEventGateway(),
		new TestGlobalAggregateManager(),
		classLoaderHandle,
		mock(FileCache.class),
		new TestingTaskManagerRuntimeInfo(taskManagerConfig),
		taskMetricGroup,
		consumableNotifier,
		partitionProducerStateChecker,
		executor);
}
 
Example #18
Source File: TaskAsyncCallTest.java    From flink with Apache License 2.0 4 votes vote down vote up
private Task createTask(Class<? extends AbstractInvokable> invokableClass) throws Exception {
	final TestingClassLoaderLease classLoaderHandle = TestingClassLoaderLease.newBuilder()
		.setGetOrResolveClassLoaderFunction((permanentBlobKeys, urls) -> new TestUserCodeClassLoader())
		.build();

	ResultPartitionConsumableNotifier consumableNotifier = new NoOpResultPartitionConsumableNotifier();
	PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
	Executor executor = mock(Executor.class);
	TaskMetricGroup taskMetricGroup = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();

	JobInformation jobInformation = new JobInformation(
		new JobID(),
		"Job Name",
		new SerializedValue<>(new ExecutionConfig()),
		new Configuration(),
		Collections.emptyList(),
		Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"Test Task",
		1,
		1,
		invokableClass.getName(),
		new Configuration());

	return new Task(
		jobInformation,
		taskInformation,
		new ExecutionAttemptID(),
		new AllocationID(),
		0,
		0,
		Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
		Collections.<InputGateDeploymentDescriptor>emptyList(),
		0,
		mock(MemoryManager.class),
		mock(IOManager.class),
		shuffleEnvironment,
		new KvStateService(new KvStateRegistry(), null, null),
		mock(BroadcastVariableManager.class),
		new TaskEventDispatcher(),
		ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES,
		new TestTaskStateManager(),
		mock(TaskManagerActions.class),
		mock(InputSplitProvider.class),
		mock(CheckpointResponder.class),
		new NoOpTaskOperatorEventGateway(),
		new TestGlobalAggregateManager(),
		classLoaderHandle,
		mock(FileCache.class),
		new TestingTaskManagerRuntimeInfo(),
		taskMetricGroup,
		consumableNotifier,
		partitionProducerStateChecker,
		executor);
}
 
Example #19
Source File: TaskExecutorSubmissionTest.java    From flink with Apache License 2.0 4 votes vote down vote up
static TaskDeploymentDescriptor createTaskDeploymentDescriptor(
		JobID jobId,
		String jobName,
		ExecutionAttemptID executionAttemptId,
		SerializedValue<ExecutionConfig> serializedExecutionConfig,
		String taskName,
		int maxNumberOfSubtasks,
		int subtaskIndex,
		int numberOfSubtasks,
		int attemptNumber,
		Configuration jobConfiguration,
		Configuration taskConfiguration,
		String invokableClassName,
		List<ResultPartitionDeploymentDescriptor> producedPartitions,
		List<InputGateDeploymentDescriptor> inputGates,
		Collection<PermanentBlobKey> requiredJarFiles,
		Collection<URL> requiredClasspaths,
		int targetSlotNumber) throws IOException {

	JobInformation jobInformation = new JobInformation(
		jobId,
		jobName,
		serializedExecutionConfig,
		jobConfiguration,
		requiredJarFiles,
		requiredClasspaths);

	TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		taskName,
		numberOfSubtasks,
		maxNumberOfSubtasks,
		invokableClassName,
		taskConfiguration);

	SerializedValue<JobInformation> serializedJobInformation = new SerializedValue<>(jobInformation);
	SerializedValue<TaskInformation> serializedJobVertexInformation = new SerializedValue<>(taskInformation);

	return new TaskDeploymentDescriptor(
		jobId,
		new TaskDeploymentDescriptor.NonOffloaded<>(serializedJobInformation),
		new TaskDeploymentDescriptor.NonOffloaded<>(serializedJobVertexInformation),
		executionAttemptId,
		new AllocationID(),
		subtaskIndex,
		attemptNumber,
		targetSlotNumber,
		null,
		producedPartitions,
		inputGates);
}
 
Example #20
Source File: JvmExitOnFatalErrorTest.java    From flink with Apache License 2.0 4 votes vote down vote up
public static void main(String[] args) throws Exception {

			System.err.println("creating task");

			// we suppress process exits via errors here to not
			// have a test that exits accidentally due to a programming error
			try {
				final Configuration taskManagerConfig = new Configuration();
				taskManagerConfig.setBoolean(TaskManagerOptions.KILL_ON_OUT_OF_MEMORY, true);

				final JobID jid = new JobID();
				final AllocationID allocationID = new AllocationID();
				final JobVertexID jobVertexId = new JobVertexID();
				final ExecutionAttemptID executionAttemptID = new ExecutionAttemptID();
				final AllocationID slotAllocationId = new AllocationID();

				final SerializedValue<ExecutionConfig> execConfig = new SerializedValue<>(new ExecutionConfig());

				final JobInformation jobInformation = new JobInformation(
						jid, "Test Job", execConfig, new Configuration(),
						Collections.emptyList(), Collections.emptyList());

				final TaskInformation taskInformation = new TaskInformation(
						jobVertexId, "Test Task", 1, 1, OomInvokable.class.getName(), new Configuration());

				final MemoryManager memoryManager = MemoryManagerBuilder.newBuilder().setMemorySize(1024 * 1024).build();
				final IOManager ioManager = new IOManagerAsync();

				final ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

				final Configuration copiedConf = new Configuration(taskManagerConfig);
				final TaskManagerRuntimeInfo tmInfo = TaskManagerConfiguration
					.fromConfiguration(
						taskManagerConfig,
						TaskExecutorResourceUtils.resourceSpecFromConfigForLocalExecution(copiedConf),
						InetAddress.getLoopbackAddress().getHostAddress());

				final Executor executor = Executors.newCachedThreadPool();

				final TaskLocalStateStore localStateStore =
					new TaskLocalStateStoreImpl(
						jid,
						allocationID,
						jobVertexId,
						0,
						TestLocalRecoveryConfig.disabled(),
						executor);

				final TaskStateManager slotStateManager =
					new TaskStateManagerImpl(
						jid,
						executionAttemptID,
						localStateStore,
						null,
						mock(CheckpointResponder.class));

				Task task = new Task(
						jobInformation,
						taskInformation,
						executionAttemptID,
						slotAllocationId,
						0,       // subtaskIndex
						0,       // attemptNumber
						Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
						Collections.<InputGateDeploymentDescriptor>emptyList(),
						0,       // targetSlotNumber
						memoryManager,
						ioManager,
						shuffleEnvironment,
						new KvStateService(new KvStateRegistry(), null, null),
						new BroadcastVariableManager(),
						new TaskEventDispatcher(),
						ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES,
						slotStateManager,
						new NoOpTaskManagerActions(),
						new NoOpInputSplitProvider(),
						NoOpCheckpointResponder.INSTANCE,
						new NoOpTaskOperatorEventGateway(),
						new TestGlobalAggregateManager(),
						TestingClassLoaderLease.newBuilder().build(),
						new FileCache(tmInfo.getTmpDirectories(), VoidPermanentBlobService.INSTANCE),
						tmInfo,
						UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
						new NoOpResultPartitionConsumableNotifier(),
						new NoOpPartitionProducerStateChecker(),
						executor);

				System.err.println("starting task thread");

				task.startTaskThread();
			}
			catch (Throwable t) {
				System.err.println("ERROR STARTING TASK");
				t.printStackTrace();
			}

			System.err.println("parking the main thread");
			CommonTestUtils.blockForeverNonInterruptibly();
		}
 
Example #21
Source File: StreamTaskTerminationTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * FLINK-6833
 *
 * <p>Tests that a finished stream task cannot be failed by an asynchronous checkpointing operation after
 * the stream task has stopped running.
 */
@Test
public void testConcurrentAsyncCheckpointCannotFailFinishedStreamTask() throws Exception {
	final Configuration taskConfiguration = new Configuration();
	final StreamConfig streamConfig = new StreamConfig(taskConfiguration);
	final NoOpStreamOperator<Long> noOpStreamOperator = new NoOpStreamOperator<>();

	final StateBackend blockingStateBackend = new BlockingStateBackend();

	streamConfig.setStreamOperator(noOpStreamOperator);
	streamConfig.setOperatorID(new OperatorID());
	streamConfig.setStateBackend(blockingStateBackend);

	final long checkpointId = 0L;
	final long checkpointTimestamp = 0L;

	final JobInformation jobInformation = new JobInformation(
		new JobID(),
		"Test Job",
		new SerializedValue<>(new ExecutionConfig()),
		new Configuration(),
		Collections.emptyList(),
		Collections.emptyList());

	final TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"Test Task",
		1,
		1,
		BlockingStreamTask.class.getName(),
		taskConfiguration);

	final TaskManagerRuntimeInfo taskManagerRuntimeInfo = new TestingTaskManagerRuntimeInfo();

	final ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

	final Task task = new Task(
		jobInformation,
		taskInformation,
		new ExecutionAttemptID(),
		new AllocationID(),
		0,
		0,
		Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
		Collections.<InputGateDeploymentDescriptor>emptyList(),
		0,
		MemoryManagerBuilder.newBuilder().setMemorySize(32L * 1024L).build(),
		new IOManagerAsync(),
		shuffleEnvironment,
		new KvStateService(new KvStateRegistry(), null, null),
		mock(BroadcastVariableManager.class),
		new TaskEventDispatcher(),
		ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES,
		new TestTaskStateManager(),
		mock(TaskManagerActions.class),
		mock(InputSplitProvider.class),
		mock(CheckpointResponder.class),
		new NoOpTaskOperatorEventGateway(),
		new TestGlobalAggregateManager(),
		TestingClassLoaderLease.newBuilder().build(),
		mock(FileCache.class),
		taskManagerRuntimeInfo,
		UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
		new NoOpResultPartitionConsumableNotifier(),
		mock(PartitionProducerStateChecker.class),
		Executors.directExecutor());

	CompletableFuture<Void> taskRun = CompletableFuture.runAsync(
		() -> task.run(),
		TestingUtils.defaultExecutor());

	// wait until the stream task started running
	RUN_LATCH.await();

	// trigger a checkpoint
	task.triggerCheckpointBarrier(checkpointId, checkpointTimestamp, CheckpointOptions.forCheckpointWithDefaultLocation(), false);

	// wait until the task has completed execution
	taskRun.get();

	// check that no failure occurred
	if (task.getFailureCause() != null) {
		throw new Exception("Task failed", task.getFailureCause());
	}

	// check that we have entered the finished state
	assertEquals(ExecutionState.FINISHED, task.getExecutionState());
}
 
Example #22
Source File: TaskDeploymentDescriptorBuilder.java    From flink with Apache License 2.0 4 votes vote down vote up
public TaskDeploymentDescriptorBuilder setSerializedJobInformation(
		MaybeOffloaded<JobInformation> serializedJobInformation) {
	this.serializedJobInformation = serializedJobInformation;
	return this;
}
 
Example #23
Source File: SynchronousCheckpointITCase.java    From flink with Apache License 2.0 4 votes vote down vote up
private Task createTask(Class<? extends AbstractInvokable> invokableClass) throws Exception {

		ResultPartitionConsumableNotifier consumableNotifier = new NoOpResultPartitionConsumableNotifier();
		PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
		Executor executor = mock(Executor.class);
		ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

		TaskMetricGroup taskMetricGroup = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();

		JobInformation jobInformation = new JobInformation(
				new JobID(),
				"Job Name",
				new SerializedValue<>(new ExecutionConfig()),
				new Configuration(),
				Collections.emptyList(),
				Collections.emptyList());

		TaskInformation taskInformation = new TaskInformation(
				new JobVertexID(),
				"Test Task",
				1,
				1,
				invokableClass.getName(),
				new Configuration());

		return new Task(
				jobInformation,
				taskInformation,
				new ExecutionAttemptID(),
				new AllocationID(),
				0,
				0,
				Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
				Collections.<InputGateDeploymentDescriptor>emptyList(),
				0,
				mock(MemoryManager.class),
				mock(IOManager.class),
				shuffleEnvironment,
				new KvStateService(new KvStateRegistry(), null, null),
				mock(BroadcastVariableManager.class),
				new TaskEventDispatcher(),
				ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES,
				new TestTaskStateManager(),
				mock(TaskManagerActions.class),
				mock(InputSplitProvider.class),
				mock(CheckpointResponder.class),
				new NoOpTaskOperatorEventGateway(),
				new TestGlobalAggregateManager(),
				TestingClassLoaderLease.newBuilder().build(),
				mock(FileCache.class),
				new TestingTaskManagerRuntimeInfo(),
				taskMetricGroup,
				consumableNotifier,
				partitionProducerStateChecker,
				executor);
	}
 
Example #24
Source File: TaskCheckpointingBehaviourTest.java    From flink with Apache License 2.0 4 votes vote down vote up
private static Task createTask(
	StreamOperator<?> op,
	StateBackend backend,
	CheckpointResponder checkpointResponder) throws IOException {

	Configuration taskConfig = new Configuration();
	StreamConfig cfg = new StreamConfig(taskConfig);
	cfg.setStreamOperator(op);
	cfg.setOperatorID(new OperatorID());
	cfg.setStateBackend(backend);

	ExecutionConfig executionConfig = new ExecutionConfig();

	JobInformation jobInformation = new JobInformation(
			new JobID(),
			"test job name",
			new SerializedValue<>(executionConfig),
			new Configuration(),
			Collections.emptyList(),
			Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
			new JobVertexID(),
			"test task name",
			1,
			11,
			TestStreamTask.class.getName(),
			taskConfig);

	ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

	return new Task(
			jobInformation,
			taskInformation,
			new ExecutionAttemptID(),
			new AllocationID(),
			0,
			0,
			Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
			Collections.<InputGateDeploymentDescriptor>emptyList(),
			0,
			mock(MemoryManager.class),
			mock(IOManager.class),
			shuffleEnvironment,
			new KvStateService(new KvStateRegistry(), null, null),
			mock(BroadcastVariableManager.class),
			new TaskEventDispatcher(),
			ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES,
			new TestTaskStateManager(),
			mock(TaskManagerActions.class),
			mock(InputSplitProvider.class),
			checkpointResponder,
			new NoOpTaskOperatorEventGateway(),
			new TestGlobalAggregateManager(),
			TestingClassLoaderLease.newBuilder().build(),
			new FileCache(new String[] { EnvironmentInformation.getTemporaryFileDirectory() },
				VoidPermanentBlobService.INSTANCE),
			new TestingTaskManagerRuntimeInfo(),
			UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
			new NoOpResultPartitionConsumableNotifier(),
			mock(PartitionProducerStateChecker.class),
			Executors.directExecutor());
}
 
Example #25
Source File: TaskCheckpointingBehaviourTest.java    From flink with Apache License 2.0 4 votes vote down vote up
private static Task createTask(
	StreamOperator<?> op,
	StateBackend backend,
	CheckpointResponder checkpointResponder) throws IOException {

	Configuration taskConfig = new Configuration();
	StreamConfig cfg = new StreamConfig(taskConfig);
	cfg.setStreamOperator(op);
	cfg.setOperatorID(new OperatorID());
	cfg.setStateBackend(backend);

	ExecutionConfig executionConfig = new ExecutionConfig();

	JobInformation jobInformation = new JobInformation(
			new JobID(),
			"test job name",
			new SerializedValue<>(executionConfig),
			new Configuration(),
			Collections.emptyList(),
			Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
			new JobVertexID(),
			"test task name",
			1,
			11,
			TestStreamTask.class.getName(),
			taskConfig);

	ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	return new Task(
			jobInformation,
			taskInformation,
			new ExecutionAttemptID(),
			new AllocationID(),
			0,
			0,
			Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
			Collections.<InputGateDeploymentDescriptor>emptyList(),
			0,
			mock(MemoryManager.class),
			mock(IOManager.class),
			shuffleEnvironment,
			new KvStateService(new KvStateRegistry(), null, null),
			mock(BroadcastVariableManager.class),
			new TaskEventDispatcher(),
			new TestTaskStateManager(),
			mock(TaskManagerActions.class),
			mock(InputSplitProvider.class),
			checkpointResponder,
			new TestGlobalAggregateManager(),
			blobService,
			new BlobLibraryCacheManager(
				blobService.getPermanentBlobService(),
				FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
				new String[0]),
			new FileCache(new String[] { EnvironmentInformation.getTemporaryFileDirectory() },
				blobService.getPermanentBlobService()),
			new TestingTaskManagerRuntimeInfo(),
			UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
			new NoOpResultPartitionConsumableNotifier(),
			mock(PartitionProducerStateChecker.class),
			Executors.directExecutor());
}
 
Example #26
Source File: StreamTaskTest.java    From flink with Apache License 2.0 4 votes vote down vote up
public static Task createTask(
		Class<? extends AbstractInvokable> invokable,
		StreamConfig taskConfig,
		Configuration taskManagerConfig,
		TestTaskStateManager taskStateManager,
		TaskManagerActions taskManagerActions) throws Exception {

	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	LibraryCacheManager libCache = mock(LibraryCacheManager.class);
	when(libCache.getClassLoader(any(JobID.class))).thenReturn(StreamTaskTest.class.getClassLoader());

	ResultPartitionConsumableNotifier consumableNotifier = new NoOpResultPartitionConsumableNotifier();
	PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
	Executor executor = mock(Executor.class);

	ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

	JobInformation jobInformation = new JobInformation(
		new JobID(),
		"Job Name",
		new SerializedValue<>(new ExecutionConfig()),
		new Configuration(),
		Collections.emptyList(),
		Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"Test Task",
		1,
		1,
		invokable.getName(),
		taskConfig.getConfiguration());

	return new Task(
		jobInformation,
		taskInformation,
		new ExecutionAttemptID(),
		new AllocationID(),
		0,
		0,
		Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
		Collections.<InputGateDeploymentDescriptor>emptyList(),
		0,
		mock(MemoryManager.class),
		mock(IOManager.class),
		shuffleEnvironment,
		new KvStateService(new KvStateRegistry(), null, null),
		mock(BroadcastVariableManager.class),
		new TaskEventDispatcher(),
		taskStateManager,
		taskManagerActions,
		mock(InputSplitProvider.class),
		mock(CheckpointResponder.class),
		new TestGlobalAggregateManager(),
		blobService,
		libCache,
		mock(FileCache.class),
		new TestingTaskManagerRuntimeInfo(taskManagerConfig, new String[] {System.getProperty("java.io.tmpdir")}),
		UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
		consumableNotifier,
		partitionProducerStateChecker,
		executor);
}
 
Example #27
Source File: SynchronousCheckpointITCase.java    From flink with Apache License 2.0 4 votes vote down vote up
private Task createTask(Class<? extends AbstractInvokable> invokableClass) throws Exception {
	BlobCacheService blobService =
			new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	LibraryCacheManager libCache = mock(LibraryCacheManager.class);
	when(libCache.getClassLoader(any(JobID.class))).thenReturn(ClassLoader.getSystemClassLoader());

	ResultPartitionConsumableNotifier consumableNotifier = new NoOpResultPartitionConsumableNotifier();
	PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
	Executor executor = mock(Executor.class);
	ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

	TaskMetricGroup taskMetricGroup = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();

	JobInformation jobInformation = new JobInformation(
			new JobID(),
			"Job Name",
			new SerializedValue<>(new ExecutionConfig()),
			new Configuration(),
			Collections.emptyList(),
			Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
			new JobVertexID(),
			"Test Task",
			1,
			1,
			invokableClass.getName(),
			new Configuration());

	return new Task(
			jobInformation,
			taskInformation,
			new ExecutionAttemptID(),
			new AllocationID(),
			0,
			0,
			Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
			Collections.<InputGateDeploymentDescriptor>emptyList(),
			0,
			mock(MemoryManager.class),
			mock(IOManager.class),
			shuffleEnvironment,
			new KvStateService(new KvStateRegistry(), null, null),
			mock(BroadcastVariableManager.class),
			new TaskEventDispatcher(),
			new TestTaskStateManager(),
			mock(TaskManagerActions.class),
			mock(InputSplitProvider.class),
			mock(CheckpointResponder.class),
			new TestGlobalAggregateManager(),
			blobService,
			libCache,
			mock(FileCache.class),
			new TestingTaskManagerRuntimeInfo(),
			taskMetricGroup,
			consumableNotifier,
			partitionProducerStateChecker,
			executor);
}
 
Example #28
Source File: StreamTaskTerminationTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * FLINK-6833
 *
 * <p>Tests that a finished stream task cannot be failed by an asynchronous checkpointing operation after
 * the stream task has stopped running.
 */
@Test
public void testConcurrentAsyncCheckpointCannotFailFinishedStreamTask() throws Exception {
	final Configuration taskConfiguration = new Configuration();
	final StreamConfig streamConfig = new StreamConfig(taskConfiguration);
	final NoOpStreamOperator<Long> noOpStreamOperator = new NoOpStreamOperator<>();

	final StateBackend blockingStateBackend = new BlockingStateBackend();

	streamConfig.setStreamOperator(noOpStreamOperator);
	streamConfig.setOperatorID(new OperatorID());
	streamConfig.setStateBackend(blockingStateBackend);

	final long checkpointId = 0L;
	final long checkpointTimestamp = 0L;

	final JobInformation jobInformation = new JobInformation(
		new JobID(),
		"Test Job",
		new SerializedValue<>(new ExecutionConfig()),
		new Configuration(),
		Collections.emptyList(),
		Collections.emptyList());

	final TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"Test Task",
		1,
		1,
		BlockingStreamTask.class.getName(),
		taskConfiguration);

	final TaskManagerRuntimeInfo taskManagerRuntimeInfo = new TestingTaskManagerRuntimeInfo();

	final ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	final Task task = new Task(
		jobInformation,
		taskInformation,
		new ExecutionAttemptID(),
		new AllocationID(),
		0,
		0,
		Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
		Collections.<InputGateDeploymentDescriptor>emptyList(),
		0,
		new MemoryManager(32L * 1024L, 1),
		new IOManagerAsync(),
		shuffleEnvironment,
		new KvStateService(new KvStateRegistry(), null, null),
		mock(BroadcastVariableManager.class),
		new TaskEventDispatcher(),
		new TestTaskStateManager(),
		mock(TaskManagerActions.class),
		mock(InputSplitProvider.class),
		mock(CheckpointResponder.class),
		new TestGlobalAggregateManager(),
		blobService,
		new BlobLibraryCacheManager(
			blobService.getPermanentBlobService(),
			FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
			new String[0]),
		mock(FileCache.class),
		taskManagerRuntimeInfo,
		UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
		new NoOpResultPartitionConsumableNotifier(),
		mock(PartitionProducerStateChecker.class),
		Executors.directExecutor());

	CompletableFuture<Void> taskRun = CompletableFuture.runAsync(
		() -> task.run(),
		TestingUtils.defaultExecutor());

	// wait until the stream task started running
	RUN_LATCH.await();

	// trigger a checkpoint
	task.triggerCheckpointBarrier(checkpointId, checkpointTimestamp, CheckpointOptions.forCheckpointWithDefaultLocation(), false);

	// wait until the task has completed execution
	taskRun.get();

	// check that no failure occurred
	if (task.getFailureCause() != null) {
		throw new Exception("Task failed", task.getFailureCause());
	}

	// check that we have entered the finished state
	assertEquals(ExecutionState.FINISHED, task.getExecutionState());
}
 
Example #29
Source File: JvmExitOnFatalErrorTest.java    From flink with Apache License 2.0 4 votes vote down vote up
public static void main(String[] args) throws Exception {

			System.err.println("creating task");

			// we suppress process exits via errors here to not
			// have a test that exits accidentally due to a programming error
			try {
				final Configuration taskManagerConfig = new Configuration();
				taskManagerConfig.setBoolean(TaskManagerOptions.KILL_ON_OUT_OF_MEMORY, true);

				final JobID jid = new JobID();
				final AllocationID allocationID = new AllocationID();
				final JobVertexID jobVertexId = new JobVertexID();
				final ExecutionAttemptID executionAttemptID = new ExecutionAttemptID();
				final AllocationID slotAllocationId = new AllocationID();

				final SerializedValue<ExecutionConfig> execConfig = new SerializedValue<>(new ExecutionConfig());

				final JobInformation jobInformation = new JobInformation(
						jid, "Test Job", execConfig, new Configuration(),
						Collections.emptyList(), Collections.emptyList());

				final TaskInformation taskInformation = new TaskInformation(
						jobVertexId, "Test Task", 1, 1, OomInvokable.class.getName(), new Configuration());

				final MemoryManager memoryManager = new MemoryManager(1024 * 1024, 1);
				final IOManager ioManager = new IOManagerAsync();

				final ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

				final TaskManagerRuntimeInfo tmInfo = TaskManagerConfiguration.fromConfiguration(taskManagerConfig);

				final Executor executor = Executors.newCachedThreadPool();

				BlobCacheService blobService =
					new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

				final TaskLocalStateStore localStateStore =
					new TaskLocalStateStoreImpl(
						jid,
						allocationID,
						jobVertexId,
						0,
						TestLocalRecoveryConfig.disabled(),
						executor);

				final TaskStateManager slotStateManager =
					new TaskStateManagerImpl(
						jid,
						executionAttemptID,
						localStateStore,
						null,
						mock(CheckpointResponder.class));

				Task task = new Task(
						jobInformation,
						taskInformation,
						executionAttemptID,
						slotAllocationId,
						0,       // subtaskIndex
						0,       // attemptNumber
						Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
						Collections.<InputGateDeploymentDescriptor>emptyList(),
						0,       // targetSlotNumber
						memoryManager,
						ioManager,
						shuffleEnvironment,
						new KvStateService(new KvStateRegistry(), null, null),
						new BroadcastVariableManager(),
						new TaskEventDispatcher(),
						slotStateManager,
						new NoOpTaskManagerActions(),
						new NoOpInputSplitProvider(),
						new NoOpCheckpointResponder(),
						new TestGlobalAggregateManager(),
						blobService,
						new BlobLibraryCacheManager(
							blobService.getPermanentBlobService(),
							FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
							new String[0]),
						new FileCache(tmInfo.getTmpDirectories(), blobService.getPermanentBlobService()),
						tmInfo,
						UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
						new NoOpResultPartitionConsumableNotifier(),
						new NoOpPartitionProducerStateChecker(),
						executor);

				System.err.println("starting task thread");

				task.startTaskThread();
			}
			catch (Throwable t) {
				System.err.println("ERROR STARTING TASK");
				t.printStackTrace();
			}

			System.err.println("parking the main thread");
			CommonTestUtils.blockForeverNonInterruptibly();
		}
 
Example #30
Source File: TaskExecutorTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Tests that we can submit a task to the TaskManager given that we've allocated a slot there.
 */
@Test(timeout = 10000L)
public void testTaskSubmission() throws Exception {
	final AllocationID allocationId = new AllocationID();
	final JobMasterId jobMasterId = JobMasterId.generate();
	final JobVertexID jobVertexId = new JobVertexID();

	JobInformation jobInformation = new JobInformation(
			jobId,
			testName.getMethodName(),
			new SerializedValue<>(new ExecutionConfig()),
			new Configuration(),
			Collections.emptyList(),
			Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
			jobVertexId,
			"test task",
			1,
			1,
			TestInvokable.class.getName(),
			new Configuration());

	SerializedValue<JobInformation> serializedJobInformation = new SerializedValue<>(jobInformation);
	SerializedValue<TaskInformation> serializedJobVertexInformation = new SerializedValue<>(taskInformation);

	final TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(
			jobId,
			new TaskDeploymentDescriptor.NonOffloaded<>(serializedJobInformation),
			new TaskDeploymentDescriptor.NonOffloaded<>(serializedJobVertexInformation),
			new ExecutionAttemptID(),
			allocationId,
			0,
			0,
			0,
			null,
			Collections.emptyList(),
			Collections.emptyList());

	final LibraryCacheManager libraryCacheManager = mock(LibraryCacheManager.class);
	when(libraryCacheManager.getClassLoader(any(JobID.class))).thenReturn(ClassLoader.getSystemClassLoader());

	final JobMasterGateway jobMasterGateway = mock(JobMasterGateway.class);
	when(jobMasterGateway.getFencingToken()).thenReturn(jobMasterId);

	final OneShotLatch taskInTerminalState = new OneShotLatch();
	final TaskManagerActions taskManagerActions = new NoOpTaskManagerActions() {
		@Override
		public void updateTaskExecutionState(TaskExecutionState taskExecutionState) {
			if (taskExecutionState.getExecutionState().isTerminal()) {
				taskInTerminalState.trigger();
			}
		}
	};
	final JobManagerConnection jobManagerConnection = new JobManagerConnection(
		jobId,
		ResourceID.generate(),
		jobMasterGateway,
		taskManagerActions,
		mock(CheckpointResponder.class),
		new TestGlobalAggregateManager(),
		libraryCacheManager,
		new NoOpResultPartitionConsumableNotifier(),
		mock(PartitionProducerStateChecker.class));

	final JobManagerTable jobManagerTable = new JobManagerTable();
	jobManagerTable.put(jobId, jobManagerConnection);

	final TaskSlotTable taskSlotTable = mock(TaskSlotTable.class);
	when(taskSlotTable.tryMarkSlotActive(eq(jobId), eq(allocationId))).thenReturn(true);
	when(taskSlotTable.addTask(any(Task.class))).thenReturn(true);

	final TaskExecutorLocalStateStoresManager localStateStoresManager = createTaskExecutorLocalStateStoresManager();

	final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder()
		.setShuffleEnvironment(nettyShuffleEnvironment)
		.setTaskSlotTable(taskSlotTable)
		.setJobManagerTable(jobManagerTable)
		.setTaskStateManager(localStateStoresManager)
		.build();

	TaskExecutor taskManager = createTaskExecutor(taskManagerServices);

	try {
		taskManager.start();

		final TaskExecutorGateway tmGateway = taskManager.getSelfGateway(TaskExecutorGateway.class);

		tmGateway.submitTask(tdd, jobMasterId, timeout);

		CompletableFuture<Boolean> completionFuture = TestInvokable.COMPLETABLE_FUTURE;

		completionFuture.get();

		taskInTerminalState.await();
	} finally {
		RpcUtils.terminateRpcEndpoint(taskManager, timeout);
	}
}