org.apache.flink.runtime.executiongraph.TaskInformation Java Examples

The following examples show how to use org.apache.flink.runtime.executiongraph.TaskInformation. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: flink   Author: flink-tpc-ds   File: TaskDeploymentDescriptorFactory.java    License: Apache License 2.0 6 votes vote down vote up
private TaskDeploymentDescriptorFactory(
		ExecutionAttemptID executionId,
		int attemptNumber,
		MaybeOffloaded<JobInformation> serializedJobInformation,
		MaybeOffloaded<TaskInformation> taskInfo,
		JobID jobID,
		boolean allowUnknownPartitions,
		int subtaskIndex,
		ExecutionEdge[][] inputEdges) {
	this.executionId = executionId;
	this.attemptNumber = attemptNumber;
	this.serializedJobInformation = serializedJobInformation;
	this.taskInfo = taskInfo;
	this.jobID = jobID;
	this.allowUnknownPartitions = allowUnknownPartitions;
	this.subtaskIndex = subtaskIndex;
	this.inputEdges = inputEdges;
}
 
Example #2
Source Project: flink   Author: apache   File: TaskDeploymentDescriptorFactory.java    License: Apache License 2.0 6 votes vote down vote up
private TaskDeploymentDescriptorFactory(
		ExecutionAttemptID executionId,
		int attemptNumber,
		MaybeOffloaded<JobInformation> serializedJobInformation,
		MaybeOffloaded<TaskInformation> taskInfo,
		JobID jobID,
		boolean allowUnknownPartitions,
		int subtaskIndex,
		ExecutionEdge[][] inputEdges) {
	this.executionId = executionId;
	this.attemptNumber = attemptNumber;
	this.serializedJobInformation = serializedJobInformation;
	this.taskInfo = taskInfo;
	this.jobID = jobID;
	this.allowUnknownPartitions = allowUnknownPartitions;
	this.subtaskIndex = subtaskIndex;
	this.inputEdges = inputEdges;
}
 
Example #3
Source Project: flink   Author: apache   File: TaskDeploymentDescriptorBuilder.java    License: Apache License 2.0 6 votes vote down vote up
private TaskDeploymentDescriptorBuilder(JobID jobId, String invokableClassName) throws IOException {
	TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"test task",
		1,
		1,
		invokableClassName,
		new Configuration());

	this.jobId = jobId;
	this.serializedJobInformation =
		new NonOffloaded<>(new SerializedValue<>(new DummyJobInformation(jobId, "DummyJob")));
	this.serializedTaskInformation = new NonOffloaded<>(new SerializedValue<>(taskInformation));
	this.executionId = new ExecutionAttemptID();
	this.allocationId = new AllocationID();
	this.subtaskIndex = 0;
	this.attemptNumber = 0;
	this.producedPartitions = Collections.emptyList();
	this.inputGates = Collections.emptyList();
	this.targetSlotNumber = 0;
	this.taskRestore = null;
}
 
Example #4
Source Project: Flink-CEPplus   Author: ljygz   File: TaskDeploymentDescriptor.java    License: Apache License 2.0 5 votes vote down vote up
public TaskDeploymentDescriptor(
	JobID jobId,
	MaybeOffloaded<JobInformation> serializedJobInformation,
	MaybeOffloaded<TaskInformation> serializedTaskInformation,
	ExecutionAttemptID executionAttemptId,
	AllocationID allocationId,
	int subtaskIndex,
	int attemptNumber,
	int targetSlotNumber,
	@Nullable JobManagerTaskRestore taskRestore,
	Collection<ResultPartitionDeploymentDescriptor> resultPartitionDeploymentDescriptors,
	Collection<InputGateDeploymentDescriptor> inputGateDeploymentDescriptors) {

	this.jobId = Preconditions.checkNotNull(jobId);

	this.serializedJobInformation = Preconditions.checkNotNull(serializedJobInformation);
	this.serializedTaskInformation = Preconditions.checkNotNull(serializedTaskInformation);

	this.executionId = Preconditions.checkNotNull(executionAttemptId);
	this.allocationId = Preconditions.checkNotNull(allocationId);

	Preconditions.checkArgument(0 <= subtaskIndex, "The subtask index must be positive.");
	this.subtaskIndex = subtaskIndex;

	Preconditions.checkArgument(0 <= attemptNumber, "The attempt number must be positive.");
	this.attemptNumber = attemptNumber;

	Preconditions.checkArgument(0 <= targetSlotNumber, "The target slot number must be positive.");
	this.targetSlotNumber = targetSlotNumber;

	this.taskRestore = taskRestore;

	this.producedPartitions = Preconditions.checkNotNull(resultPartitionDeploymentDescriptors);
	this.inputGates = Preconditions.checkNotNull(inputGateDeploymentDescriptors);
}
 
Example #5
Source Project: Flink-CEPplus   Author: ljygz   File: TaskDeploymentDescriptor.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Return the sub task's serialized task information.
 *
 * @return serialized task information (may be <tt>null</tt> before a call to {@link
 * #loadBigData(PermanentBlobService)}).
 */
@Nullable
public SerializedValue<TaskInformation> getSerializedTaskInformation() {
	if (serializedTaskInformation instanceof NonOffloaded) {
		NonOffloaded<TaskInformation> taskInformation =
			(NonOffloaded<TaskInformation>) serializedTaskInformation;
		return taskInformation.serializedValue;
	} else {
		throw new IllegalStateException(
			"Trying to work with offloaded serialized job information.");
	}
}
 
Example #6
Source Project: Flink-CEPplus   Author: ljygz   File: TaskDeploymentDescriptorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Nonnull
private TaskDeploymentDescriptor createTaskDeploymentDescriptor(TaskDeploymentDescriptor.MaybeOffloaded<JobInformation> jobInformation, TaskDeploymentDescriptor.MaybeOffloaded<TaskInformation> taskInformation) {
	return new TaskDeploymentDescriptor(
		jobID,
		jobInformation,
		taskInformation,
		execId,
		allocationId,
		indexInSubtaskGroup,
		attemptNumber,
		targetSlotNumber,
		taskRestore,
		producedResults,
		inputGates);
}
 
Example #7
Source Project: flink   Author: flink-tpc-ds   File: TaskDeploymentDescriptorFactory.java    License: Apache License 2.0 5 votes vote down vote up
private static MaybeOffloaded<TaskInformation> getSerializedTaskInformation(
	Either<SerializedValue<TaskInformation>,
		PermanentBlobKey> taskInfo) {
	return taskInfo.isLeft() ?
		new TaskDeploymentDescriptor.NonOffloaded<>(taskInfo.left()) :
		new TaskDeploymentDescriptor.Offloaded<>(taskInfo.right());
}
 
Example #8
Source Project: flink   Author: flink-tpc-ds   File: TaskDeploymentDescriptor.java    License: Apache License 2.0 5 votes vote down vote up
public TaskDeploymentDescriptor(
	JobID jobId,
	MaybeOffloaded<JobInformation> serializedJobInformation,
	MaybeOffloaded<TaskInformation> serializedTaskInformation,
	ExecutionAttemptID executionAttemptId,
	AllocationID allocationId,
	int subtaskIndex,
	int attemptNumber,
	int targetSlotNumber,
	@Nullable JobManagerTaskRestore taskRestore,
	Collection<ResultPartitionDeploymentDescriptor> resultPartitionDeploymentDescriptors,
	Collection<InputGateDeploymentDescriptor> inputGateDeploymentDescriptors) {

	this.jobId = Preconditions.checkNotNull(jobId);

	this.serializedJobInformation = Preconditions.checkNotNull(serializedJobInformation);
	this.serializedTaskInformation = Preconditions.checkNotNull(serializedTaskInformation);

	this.executionId = Preconditions.checkNotNull(executionAttemptId);
	this.allocationId = Preconditions.checkNotNull(allocationId);

	Preconditions.checkArgument(0 <= subtaskIndex, "The subtask index must be positive.");
	this.subtaskIndex = subtaskIndex;

	Preconditions.checkArgument(0 <= attemptNumber, "The attempt number must be positive.");
	this.attemptNumber = attemptNumber;

	Preconditions.checkArgument(0 <= targetSlotNumber, "The target slot number must be positive.");
	this.targetSlotNumber = targetSlotNumber;

	this.taskRestore = taskRestore;

	this.producedPartitions = Preconditions.checkNotNull(resultPartitionDeploymentDescriptors);
	this.inputGates = Preconditions.checkNotNull(inputGateDeploymentDescriptors);
}
 
Example #9
Source Project: flink   Author: flink-tpc-ds   File: TaskDeploymentDescriptor.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Return the sub task's serialized task information.
 *
 * @return serialized task information (may throw {@link IllegalStateException} if {@link
 * #loadBigData(PermanentBlobService)} is not called beforehand)).
 * @throws IllegalStateException If job information is offloaded to BLOB store.
 */
public SerializedValue<TaskInformation> getSerializedTaskInformation() {
	if (serializedTaskInformation instanceof NonOffloaded) {
		NonOffloaded<TaskInformation> taskInformation =
			(NonOffloaded<TaskInformation>) serializedTaskInformation;
		return taskInformation.serializedValue;
	} else {
		throw new IllegalStateException(
			"Trying to work with offloaded serialized job information.");
	}
}
 
Example #10
Source Project: flink   Author: flink-tpc-ds   File: TaskDeploymentDescriptorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Nonnull
private TaskDeploymentDescriptor createTaskDeploymentDescriptor(TaskDeploymentDescriptor.MaybeOffloaded<JobInformation> jobInformation, TaskDeploymentDescriptor.MaybeOffloaded<TaskInformation> taskInformation) {
	return new TaskDeploymentDescriptor(
		jobID,
		jobInformation,
		taskInformation,
		execId,
		allocationId,
		indexInSubtaskGroup,
		attemptNumber,
		targetSlotNumber,
		taskRestore,
		producedResults,
		inputGates);
}
 
Example #11
Source Project: flink   Author: apache   File: TaskDeploymentDescriptorFactory.java    License: Apache License 2.0 5 votes vote down vote up
private static MaybeOffloaded<TaskInformation> getSerializedTaskInformation(
	Either<SerializedValue<TaskInformation>,
		PermanentBlobKey> taskInfo) {
	return taskInfo.isLeft() ?
		new TaskDeploymentDescriptor.NonOffloaded<>(taskInfo.left()) :
		new TaskDeploymentDescriptor.Offloaded<>(taskInfo.right());
}
 
Example #12
Source Project: flink   Author: apache   File: TaskDeploymentDescriptor.java    License: Apache License 2.0 5 votes vote down vote up
public TaskDeploymentDescriptor(
	JobID jobId,
	MaybeOffloaded<JobInformation> serializedJobInformation,
	MaybeOffloaded<TaskInformation> serializedTaskInformation,
	ExecutionAttemptID executionAttemptId,
	AllocationID allocationId,
	int subtaskIndex,
	int attemptNumber,
	int targetSlotNumber,
	@Nullable JobManagerTaskRestore taskRestore,
	List<ResultPartitionDeploymentDescriptor> resultPartitionDeploymentDescriptors,
	List<InputGateDeploymentDescriptor> inputGateDeploymentDescriptors) {

	this.jobId = Preconditions.checkNotNull(jobId);

	this.serializedJobInformation = Preconditions.checkNotNull(serializedJobInformation);
	this.serializedTaskInformation = Preconditions.checkNotNull(serializedTaskInformation);

	this.executionId = Preconditions.checkNotNull(executionAttemptId);
	this.allocationId = Preconditions.checkNotNull(allocationId);

	Preconditions.checkArgument(0 <= subtaskIndex, "The subtask index must be positive.");
	this.subtaskIndex = subtaskIndex;

	Preconditions.checkArgument(0 <= attemptNumber, "The attempt number must be positive.");
	this.attemptNumber = attemptNumber;

	Preconditions.checkArgument(0 <= targetSlotNumber, "The target slot number must be positive.");
	this.targetSlotNumber = targetSlotNumber;

	this.taskRestore = taskRestore;

	this.producedPartitions = Preconditions.checkNotNull(resultPartitionDeploymentDescriptors);
	this.inputGates = Preconditions.checkNotNull(inputGateDeploymentDescriptors);
}
 
Example #13
Source Project: flink   Author: apache   File: TaskDeploymentDescriptor.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Return the sub task's serialized task information.
 *
 * @return serialized task information (may throw {@link IllegalStateException} if {@link
 * #loadBigData(PermanentBlobService)} is not called beforehand)).
 * @throws IllegalStateException If job information is offloaded to BLOB store.
 */
public SerializedValue<TaskInformation> getSerializedTaskInformation() {
	if (serializedTaskInformation instanceof NonOffloaded) {
		NonOffloaded<TaskInformation> taskInformation =
			(NonOffloaded<TaskInformation>) serializedTaskInformation;
		return taskInformation.serializedValue;
	} else {
		throw new IllegalStateException(
			"Trying to work with offloaded serialized job information.");
	}
}
 
Example #14
Source Project: flink   Author: apache   File: TaskDeploymentDescriptorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Nonnull
private TaskDeploymentDescriptor createTaskDeploymentDescriptor(TaskDeploymentDescriptor.MaybeOffloaded<JobInformation> jobInformation, TaskDeploymentDescriptor.MaybeOffloaded<TaskInformation> taskInformation) {
	return new TaskDeploymentDescriptor(
		jobID,
		jobInformation,
		taskInformation,
		execId,
		allocationId,
		indexInSubtaskGroup,
		attemptNumber,
		targetSlotNumber,
		taskRestore,
		producedResults,
		inputGates);
}
 
Example #15
Source Project: Flink-CEPplus   Author: ljygz   File: TaskAsyncCallTest.java    License: Apache License 2.0 4 votes vote down vote up
private Task createTask(Class<? extends AbstractInvokable> invokableClass) throws Exception {
	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	LibraryCacheManager libCache = mock(LibraryCacheManager.class);
	when(libCache.getClassLoader(any(JobID.class))).thenReturn(new TestUserCodeClassLoader());

	ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
	ResultPartitionConsumableNotifier consumableNotifier = new NoOpResultPartitionConsumableNotifier();
	PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
	Executor executor = mock(Executor.class);
	TaskEventDispatcher taskEventDispatcher = mock(TaskEventDispatcher.class);
	NetworkEnvironment networkEnvironment = mock(NetworkEnvironment.class);
	when(networkEnvironment.getResultPartitionManager()).thenReturn(partitionManager);
	when(networkEnvironment.getDefaultIOMode()).thenReturn(IOManager.IOMode.SYNC);
	when(networkEnvironment.createKvStateTaskRegistry(any(JobID.class), any(JobVertexID.class)))
			.thenReturn(mock(TaskKvStateRegistry.class));
	when(networkEnvironment.getTaskEventDispatcher()).thenReturn(taskEventDispatcher);

	TaskMetricGroup taskMetricGroup = mock(TaskMetricGroup.class);
	when(taskMetricGroup.getIOMetricGroup()).thenReturn(mock(TaskIOMetricGroup.class));

	JobInformation jobInformation = new JobInformation(
		new JobID(),
		"Job Name",
		new SerializedValue<>(new ExecutionConfig()),
		new Configuration(),
		Collections.emptyList(),
		Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"Test Task",
		1,
		1,
		invokableClass.getName(),
		new Configuration());

	return new Task(
		jobInformation,
		taskInformation,
		new ExecutionAttemptID(),
		new AllocationID(),
		0,
		0,
		Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
		Collections.<InputGateDeploymentDescriptor>emptyList(),
		0,
		mock(MemoryManager.class),
		mock(IOManager.class),
		networkEnvironment,
		mock(BroadcastVariableManager.class),
		new TestTaskStateManager(),
		mock(TaskManagerActions.class),
		mock(InputSplitProvider.class),
		mock(CheckpointResponder.class),
		new TestGlobalAggregateManager(),
		blobService,
		libCache,
		mock(FileCache.class),
		new TestingTaskManagerRuntimeInfo(),
		taskMetricGroup,
		consumableNotifier,
		partitionProducerStateChecker,
		executor);
}
 
Example #16
Source Project: Flink-CEPplus   Author: ljygz   File: TaskManagerTest.java    License: Apache License 2.0 4 votes vote down vote up
private static TaskDeploymentDescriptor createTaskDeploymentDescriptor(
	JobID jobId,
	String jobName,
	JobVertexID jobVertexId,
	ExecutionAttemptID executionAttemptId,
	SerializedValue<ExecutionConfig> serializedExecutionConfig,
	String taskName,
	int numberOfKeyGroups,
	int subtaskIndex,
	int parallelism,
	int attemptNumber,
	Configuration jobConfiguration,
	Configuration taskConfiguration,
	String invokableClassName,
	Collection<ResultPartitionDeploymentDescriptor> producedPartitions,
	Collection<InputGateDeploymentDescriptor> inputGates,
	Collection<PermanentBlobKey> requiredJarFiles,
	Collection<URL> requiredClasspaths,
	int targetSlotNumber) throws IOException {

	JobInformation jobInformation = new JobInformation(
		jobId,
		jobName,
		serializedExecutionConfig,
		jobConfiguration,
		requiredJarFiles,
		requiredClasspaths);

	TaskInformation taskInformation = new TaskInformation(
		jobVertexId,
		taskName,
		parallelism,
		numberOfKeyGroups,
		invokableClassName,
		taskConfiguration);

	SerializedValue<JobInformation> serializedJobInformation = new SerializedValue<>(jobInformation);
	SerializedValue<TaskInformation> serializedJobVertexInformation = new SerializedValue<>(taskInformation);

	return new TaskDeploymentDescriptor(
		jobId,
		new TaskDeploymentDescriptor.NonOffloaded<>(serializedJobInformation),
		new TaskDeploymentDescriptor.NonOffloaded<>(serializedJobVertexInformation),
		executionAttemptId,
		new AllocationID(),
		subtaskIndex,
		attemptNumber,
		targetSlotNumber,
		null,
		producedPartitions,
		inputGates);

}
 
Example #17
Source Project: Flink-CEPplus   Author: ljygz   File: TaskTest.java    License: Apache License 2.0 4 votes vote down vote up
private Task build() throws Exception {
	final JobID jobId = new JobID();
	final JobVertexID jobVertexId = new JobVertexID();
	final ExecutionAttemptID executionAttemptId = new ExecutionAttemptID();

	final SerializedValue<ExecutionConfig> serializedExecutionConfig = new SerializedValue<>(executionConfig);

	final JobInformation jobInformation = new JobInformation(
		jobId,
		"Test Job",
		serializedExecutionConfig,
		new Configuration(),
		requiredJarFileBlobKeys,
		Collections.emptyList());

	final TaskInformation taskInformation = new TaskInformation(
		jobVertexId,
		"Test Task",
		1,
		1,
		invokable.getName(),
		new Configuration());

	final BlobCacheService blobCacheService = new BlobCacheService(
		mock(PermanentBlobCache.class),
		mock(TransientBlobCache.class));

	final TaskMetricGroup taskMetricGroup = mock(TaskMetricGroup.class);
	when(taskMetricGroup.getIOMetricGroup()).thenReturn(mock(TaskIOMetricGroup.class));

	return new Task(
		jobInformation,
		taskInformation,
		executionAttemptId,
		new AllocationID(),
		0,
		0,
		Collections.emptyList(),
		Collections.emptyList(),
		0,
		mock(MemoryManager.class),
		mock(IOManager.class),
		networkEnvironment,
		mock(BroadcastVariableManager.class),
		new TestTaskStateManager(),
		taskManagerActions,
		new MockInputSplitProvider(),
		new TestCheckpointResponder(),
		new TestGlobalAggregateManager(),
		blobCacheService,
		libraryCacheManager,
		mock(FileCache.class),
		new TestingTaskManagerRuntimeInfo(taskManagerConfig),
		taskMetricGroup,
		consumableNotifier,
		partitionProducerStateChecker,
		executor);
}
 
Example #18
Source Project: Flink-CEPplus   Author: ljygz   File: JvmExitOnFatalErrorTest.java    License: Apache License 2.0 4 votes vote down vote up
public static void main(String[] args) throws Exception {

			System.err.println("creating task");

			// we suppress process exits via errors here to not
			// have a test that exits accidentally due to a programming error
			try {
				final Configuration taskManagerConfig = new Configuration();
				taskManagerConfig.setBoolean(TaskManagerOptions.KILL_ON_OUT_OF_MEMORY, true);

				final JobID jid = new JobID();
				final AllocationID allocationID = new AllocationID();
				final JobVertexID jobVertexId = new JobVertexID();
				final ExecutionAttemptID executionAttemptID = new ExecutionAttemptID();
				final AllocationID slotAllocationId = new AllocationID();

				final SerializedValue<ExecutionConfig> execConfig = new SerializedValue<>(new ExecutionConfig());

				final JobInformation jobInformation = new JobInformation(
						jid, "Test Job", execConfig, new Configuration(),
						Collections.emptyList(), Collections.emptyList());

				final TaskInformation taskInformation = new TaskInformation(
						jobVertexId, "Test Task", 1, 1, OomInvokable.class.getName(), new Configuration());

				final MemoryManager memoryManager = new MemoryManager(1024 * 1024, 1);
				final IOManager ioManager = new IOManagerAsync();

				final NetworkEnvironment networkEnvironment = mock(NetworkEnvironment.class);
				when(networkEnvironment.createKvStateTaskRegistry(jid, jobVertexId)).thenReturn(mock(TaskKvStateRegistry.class));
				TaskEventDispatcher taskEventDispatcher = mock(TaskEventDispatcher.class);
				when(networkEnvironment.getTaskEventDispatcher()).thenReturn(taskEventDispatcher);

				final TaskManagerRuntimeInfo tmInfo = TaskManagerConfiguration.fromConfiguration(taskManagerConfig);

				final Executor executor = Executors.newCachedThreadPool();

				BlobCacheService blobService =
					new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

				final TaskLocalStateStore localStateStore =
					new TaskLocalStateStoreImpl(
						jid,
						allocationID,
						jobVertexId,
						0,
						TestLocalRecoveryConfig.disabled(),
						executor);

				final TaskStateManager slotStateManager =
					new TaskStateManagerImpl(
						jid,
						executionAttemptID,
						localStateStore,
						null,
						mock(CheckpointResponder.class));

				Task task = new Task(
						jobInformation,
						taskInformation,
						executionAttemptID,
						slotAllocationId,
						0,       // subtaskIndex
						0,       // attemptNumber
						Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
						Collections.<InputGateDeploymentDescriptor>emptyList(),
						0,       // targetSlotNumber
						memoryManager,
						ioManager,
						networkEnvironment,
						new BroadcastVariableManager(),
						slotStateManager,
						new NoOpTaskManagerActions(),
						new NoOpInputSplitProvider(),
						new NoOpCheckpointResponder(),
						new TestGlobalAggregateManager(),
						blobService,
						new BlobLibraryCacheManager(
							blobService.getPermanentBlobService(),
							FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
							new String[0]),
						new FileCache(tmInfo.getTmpDirectories(), blobService.getPermanentBlobService()),
						tmInfo,
						UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
						new NoOpResultPartitionConsumableNotifier(),
						new NoOpPartitionProducerStateChecker(),
						executor);

				System.err.println("starting task thread");

				task.startTaskThread();
			}
			catch (Throwable t) {
				System.err.println("ERROR STARTING TASK");
				t.printStackTrace();
			}

			System.err.println("parking the main thread");
			CommonTestUtils.blockForeverNonInterruptibly();
		}
 
Example #19
Source Project: Flink-CEPplus   Author: ljygz   File: StreamTaskTerminationTest.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * FLINK-6833
 *
 * <p>Tests that a finished stream task cannot be failed by an asynchronous checkpointing operation after
 * the stream task has stopped running.
 */
@Test
public void testConcurrentAsyncCheckpointCannotFailFinishedStreamTask() throws Exception {
	final Configuration taskConfiguration = new Configuration();
	final StreamConfig streamConfig = new StreamConfig(taskConfiguration);
	final NoOpStreamOperator<Long> noOpStreamOperator = new NoOpStreamOperator<>();

	final StateBackend blockingStateBackend = new BlockingStateBackend();

	streamConfig.setStreamOperator(noOpStreamOperator);
	streamConfig.setOperatorID(new OperatorID());
	streamConfig.setStateBackend(blockingStateBackend);

	final long checkpointId = 0L;
	final long checkpointTimestamp = 0L;

	final JobInformation jobInformation = new JobInformation(
		new JobID(),
		"Test Job",
		new SerializedValue<>(new ExecutionConfig()),
		new Configuration(),
		Collections.emptyList(),
		Collections.emptyList());

	final TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"Test Task",
		1,
		1,
		BlockingStreamTask.class.getName(),
		taskConfiguration);

	final TaskManagerRuntimeInfo taskManagerRuntimeInfo = new TestingTaskManagerRuntimeInfo();

	TaskEventDispatcher taskEventDispatcher = new TaskEventDispatcher();
	final NetworkEnvironment networkEnv = mock(NetworkEnvironment.class);
	when(networkEnv.createKvStateTaskRegistry(any(JobID.class), any(JobVertexID.class))).thenReturn(mock(TaskKvStateRegistry.class));
	when(networkEnv.getTaskEventDispatcher()).thenReturn(taskEventDispatcher);

	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	final Task task = new Task(
		jobInformation,
		taskInformation,
		new ExecutionAttemptID(),
		new AllocationID(),
		0,
		0,
		Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
		Collections.<InputGateDeploymentDescriptor>emptyList(),
		0,
		new MemoryManager(32L * 1024L, 1),
		new IOManagerAsync(),
		networkEnv,
		mock(BroadcastVariableManager.class),
		new TestTaskStateManager(),
		mock(TaskManagerActions.class),
		mock(InputSplitProvider.class),
		mock(CheckpointResponder.class),
		new TestGlobalAggregateManager(),
		blobService,
		new BlobLibraryCacheManager(
			blobService.getPermanentBlobService(),
			FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
			new String[0]),
		mock(FileCache.class),
		taskManagerRuntimeInfo,
		UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
		new NoOpResultPartitionConsumableNotifier(),
		mock(PartitionProducerStateChecker.class),
		Executors.directExecutor());

	CompletableFuture<Void> taskRun = CompletableFuture.runAsync(
		() -> task.run(),
		TestingUtils.defaultExecutor());

	// wait until the stream task started running
	RUN_LATCH.await();

	// trigger a checkpoint
	task.triggerCheckpointBarrier(checkpointId, checkpointTimestamp, CheckpointOptions.forCheckpointWithDefaultLocation());

	// wait until the task has completed execution
	taskRun.get();

	// check that no failure occurred
	if (task.getFailureCause() != null) {
		throw new Exception("Task failed", task.getFailureCause());
	}

	// check that we have entered the finished state
	assertEquals(ExecutionState.FINISHED, task.getExecutionState());
}
 
Example #20
Source Project: Flink-CEPplus   Author: ljygz   File: StreamTaskTest.java    License: Apache License 2.0 4 votes vote down vote up
public static Task createTask(
		Class<? extends AbstractInvokable> invokable,
		StreamConfig taskConfig,
		Configuration taskManagerConfig,
		TestTaskStateManager taskStateManager,
		TaskManagerActions taskManagerActions) throws Exception {

	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	LibraryCacheManager libCache = mock(LibraryCacheManager.class);
	when(libCache.getClassLoader(any(JobID.class))).thenReturn(StreamTaskTest.class.getClassLoader());

	ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
	ResultPartitionConsumableNotifier consumableNotifier = new NoOpResultPartitionConsumableNotifier();
	PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
	Executor executor = mock(Executor.class);
	TaskEventDispatcher taskEventDispatcher = new TaskEventDispatcher();

	NetworkEnvironment network = mock(NetworkEnvironment.class);
	when(network.getResultPartitionManager()).thenReturn(partitionManager);
	when(network.getDefaultIOMode()).thenReturn(IOManager.IOMode.SYNC);
	when(network.createKvStateTaskRegistry(any(JobID.class), any(JobVertexID.class)))
			.thenReturn(mock(TaskKvStateRegistry.class));
	when(network.getTaskEventDispatcher()).thenReturn(taskEventDispatcher);

	JobInformation jobInformation = new JobInformation(
		new JobID(),
		"Job Name",
		new SerializedValue<>(new ExecutionConfig()),
		new Configuration(),
		Collections.emptyList(),
		Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"Test Task",
		1,
		1,
		invokable.getName(),
		taskConfig.getConfiguration());

	return new Task(
		jobInformation,
		taskInformation,
		new ExecutionAttemptID(),
		new AllocationID(),
		0,
		0,
		Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
		Collections.<InputGateDeploymentDescriptor>emptyList(),
		0,
		mock(MemoryManager.class),
		mock(IOManager.class),
		network,
		mock(BroadcastVariableManager.class),
		taskStateManager,
		taskManagerActions,
		mock(InputSplitProvider.class),
		mock(CheckpointResponder.class),
		new TestGlobalAggregateManager(),
		blobService,
		libCache,
		mock(FileCache.class),
		new TestingTaskManagerRuntimeInfo(taskManagerConfig, new String[] {System.getProperty("java.io.tmpdir")}),
		UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
		consumableNotifier,
		partitionProducerStateChecker,
		executor);
}
 
Example #21
Source Project: Flink-CEPplus   Author: ljygz   File: TaskCheckpointingBehaviourTest.java    License: Apache License 2.0 4 votes vote down vote up
private static Task createTask(
	StreamOperator<?> op,
	StateBackend backend,
	CheckpointResponder checkpointResponder,
	boolean failOnCheckpointErrors) throws IOException {

	Configuration taskConfig = new Configuration();
	StreamConfig cfg = new StreamConfig(taskConfig);
	cfg.setStreamOperator(op);
	cfg.setOperatorID(new OperatorID());
	cfg.setStateBackend(backend);

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setFailTaskOnCheckpointError(failOnCheckpointErrors);

	JobInformation jobInformation = new JobInformation(
			new JobID(),
			"test job name",
			new SerializedValue<>(executionConfig),
			new Configuration(),
			Collections.emptyList(),
			Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
			new JobVertexID(),
			"test task name",
			1,
			11,
			TestStreamTask.class.getName(),
			taskConfig);

	TaskKvStateRegistry mockKvRegistry = mock(TaskKvStateRegistry.class);
	TaskEventDispatcher taskEventDispatcher = new TaskEventDispatcher();
	NetworkEnvironment network = mock(NetworkEnvironment.class);
	when(network.createKvStateTaskRegistry(any(JobID.class), any(JobVertexID.class))).thenReturn(mockKvRegistry);
	when(network.getTaskEventDispatcher()).thenReturn(taskEventDispatcher);

	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	return new Task(
			jobInformation,
			taskInformation,
			new ExecutionAttemptID(),
			new AllocationID(),
			0,
			0,
			Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
			Collections.<InputGateDeploymentDescriptor>emptyList(),
			0,
			mock(MemoryManager.class),
			mock(IOManager.class),
			network,
			mock(BroadcastVariableManager.class),
			new TestTaskStateManager(),
			mock(TaskManagerActions.class),
			mock(InputSplitProvider.class),
			checkpointResponder,
			new TestGlobalAggregateManager(),
			blobService,
			new BlobLibraryCacheManager(
				blobService.getPermanentBlobService(),
				FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
				new String[0]),
			new FileCache(new String[] { EnvironmentInformation.getTemporaryFileDirectory() },
				blobService.getPermanentBlobService()),
			new TestingTaskManagerRuntimeInfo(),
			UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
			new NoOpResultPartitionConsumableNotifier(),
			mock(PartitionProducerStateChecker.class),
			Executors.directExecutor());
}
 
Example #22
Source Project: flink   Author: flink-tpc-ds   File: TestTaskBuilder.java    License: Apache License 2.0 4 votes vote down vote up
public Task build() throws Exception {
	final JobID jobId = new JobID();
	final JobVertexID jobVertexId = new JobVertexID();
	final ExecutionAttemptID executionAttemptId = new ExecutionAttemptID();

	final SerializedValue<ExecutionConfig> serializedExecutionConfig = new SerializedValue<>(executionConfig);

	final JobInformation jobInformation = new JobInformation(
		jobId,
		"Test Job",
		serializedExecutionConfig,
		new Configuration(),
		requiredJarFileBlobKeys,
		Collections.emptyList());

	final TaskInformation taskInformation = new TaskInformation(
		jobVertexId,
		"Test Task",
		1,
		1,
		invokable.getName(),
		new Configuration());

	final BlobCacheService blobCacheService = new BlobCacheService(
		mock(PermanentBlobCache.class),
		mock(TransientBlobCache.class));

	final TaskMetricGroup taskMetricGroup = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();

	return new Task(
		jobInformation,
		taskInformation,
		executionAttemptId,
		new AllocationID(),
		0,
		0,
		resultPartitions,
		inputGates,
		0,
		new MemoryManager(1024 * 1024, 1),
		mock(IOManager.class),
		shuffleEnvironment,
		kvStateService,
		new BroadcastVariableManager(),
		new TaskEventDispatcher(),
		new TestTaskStateManager(),
		taskManagerActions,
		new MockInputSplitProvider(),
		new TestCheckpointResponder(),
		new TestGlobalAggregateManager(),
		blobCacheService,
		libraryCacheManager,
		mock(FileCache.class),
		new TestingTaskManagerRuntimeInfo(taskManagerConfig),
		taskMetricGroup,
		consumableNotifier,
		partitionProducerStateChecker,
		executor);
}
 
Example #23
Source Project: flink   Author: flink-tpc-ds   File: TaskAsyncCallTest.java    License: Apache License 2.0 4 votes vote down vote up
private Task createTask(Class<? extends AbstractInvokable> invokableClass) throws Exception {
	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	LibraryCacheManager libCache = mock(LibraryCacheManager.class);
	when(libCache.getClassLoader(any(JobID.class))).thenReturn(new TestUserCodeClassLoader());

	ResultPartitionConsumableNotifier consumableNotifier = new NoOpResultPartitionConsumableNotifier();
	PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
	Executor executor = mock(Executor.class);
	TaskMetricGroup taskMetricGroup = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();

	JobInformation jobInformation = new JobInformation(
		new JobID(),
		"Job Name",
		new SerializedValue<>(new ExecutionConfig()),
		new Configuration(),
		Collections.emptyList(),
		Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"Test Task",
		1,
		1,
		invokableClass.getName(),
		new Configuration());

	return new Task(
		jobInformation,
		taskInformation,
		new ExecutionAttemptID(),
		new AllocationID(),
		0,
		0,
		Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
		Collections.<InputGateDeploymentDescriptor>emptyList(),
		0,
		mock(MemoryManager.class),
		mock(IOManager.class),
		shuffleEnvironment,
		new KvStateService(new KvStateRegistry(), null, null),
		mock(BroadcastVariableManager.class),
		new TaskEventDispatcher(),
		new TestTaskStateManager(),
		mock(TaskManagerActions.class),
		mock(InputSplitProvider.class),
		mock(CheckpointResponder.class),
		new TestGlobalAggregateManager(),
		blobService,
		libCache,
		mock(FileCache.class),
		new TestingTaskManagerRuntimeInfo(),
		taskMetricGroup,
		consumableNotifier,
		partitionProducerStateChecker,
		executor);
}
 
Example #24
Source Project: flink   Author: flink-tpc-ds   File: TaskExecutorSubmissionTest.java    License: Apache License 2.0 4 votes vote down vote up
static TaskDeploymentDescriptor createTaskDeploymentDescriptor(
		JobID jobId,
		String jobName,
		ExecutionAttemptID executionAttemptId,
		SerializedValue<ExecutionConfig> serializedExecutionConfig,
		String taskName,
		int maxNumberOfSubtasks,
		int subtaskIndex,
		int numberOfSubtasks,
		int attemptNumber,
		Configuration jobConfiguration,
		Configuration taskConfiguration,
		String invokableClassName,
		Collection<ResultPartitionDeploymentDescriptor> producedPartitions,
		Collection<InputGateDeploymentDescriptor> inputGates,
		Collection<PermanentBlobKey> requiredJarFiles,
		Collection<URL> requiredClasspaths,
		int targetSlotNumber) throws IOException {

	JobInformation jobInformation = new JobInformation(
		jobId,
		jobName,
		serializedExecutionConfig,
		jobConfiguration,
		requiredJarFiles,
		requiredClasspaths);

	TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		taskName,
		numberOfSubtasks,
		maxNumberOfSubtasks,
		invokableClassName,
		taskConfiguration);

	SerializedValue<JobInformation> serializedJobInformation = new SerializedValue<>(jobInformation);
	SerializedValue<TaskInformation> serializedJobVertexInformation = new SerializedValue<>(taskInformation);

	return new TaskDeploymentDescriptor(
		jobId,
		new TaskDeploymentDescriptor.NonOffloaded<>(serializedJobInformation),
		new TaskDeploymentDescriptor.NonOffloaded<>(serializedJobVertexInformation),
		executionAttemptId,
		new AllocationID(),
		subtaskIndex,
		attemptNumber,
		targetSlotNumber,
		null,
		producedPartitions,
		inputGates);
}
 
Example #25
Source Project: flink   Author: flink-tpc-ds   File: TaskExecutorTest.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Tests that we can submit a task to the TaskManager given that we've allocated a slot there.
 */
@Test(timeout = 10000L)
public void testTaskSubmission() throws Exception {
	final AllocationID allocationId = new AllocationID();
	final JobMasterId jobMasterId = JobMasterId.generate();
	final JobVertexID jobVertexId = new JobVertexID();

	JobInformation jobInformation = new JobInformation(
			jobId,
			testName.getMethodName(),
			new SerializedValue<>(new ExecutionConfig()),
			new Configuration(),
			Collections.emptyList(),
			Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
			jobVertexId,
			"test task",
			1,
			1,
			TestInvokable.class.getName(),
			new Configuration());

	SerializedValue<JobInformation> serializedJobInformation = new SerializedValue<>(jobInformation);
	SerializedValue<TaskInformation> serializedJobVertexInformation = new SerializedValue<>(taskInformation);

	final TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(
			jobId,
			new TaskDeploymentDescriptor.NonOffloaded<>(serializedJobInformation),
			new TaskDeploymentDescriptor.NonOffloaded<>(serializedJobVertexInformation),
			new ExecutionAttemptID(),
			allocationId,
			0,
			0,
			0,
			null,
			Collections.emptyList(),
			Collections.emptyList());

	final LibraryCacheManager libraryCacheManager = mock(LibraryCacheManager.class);
	when(libraryCacheManager.getClassLoader(any(JobID.class))).thenReturn(ClassLoader.getSystemClassLoader());

	final JobMasterGateway jobMasterGateway = mock(JobMasterGateway.class);
	when(jobMasterGateway.getFencingToken()).thenReturn(jobMasterId);

	final OneShotLatch taskInTerminalState = new OneShotLatch();
	final TaskManagerActions taskManagerActions = new NoOpTaskManagerActions() {
		@Override
		public void updateTaskExecutionState(TaskExecutionState taskExecutionState) {
			if (taskExecutionState.getExecutionState().isTerminal()) {
				taskInTerminalState.trigger();
			}
		}
	};
	final JobManagerConnection jobManagerConnection = new JobManagerConnection(
		jobId,
		ResourceID.generate(),
		jobMasterGateway,
		taskManagerActions,
		mock(CheckpointResponder.class),
		new TestGlobalAggregateManager(),
		libraryCacheManager,
		new NoOpResultPartitionConsumableNotifier(),
		mock(PartitionProducerStateChecker.class));

	final JobManagerTable jobManagerTable = new JobManagerTable();
	jobManagerTable.put(jobId, jobManagerConnection);

	final TaskSlotTable taskSlotTable = mock(TaskSlotTable.class);
	when(taskSlotTable.tryMarkSlotActive(eq(jobId), eq(allocationId))).thenReturn(true);
	when(taskSlotTable.addTask(any(Task.class))).thenReturn(true);

	final TaskExecutorLocalStateStoresManager localStateStoresManager = createTaskExecutorLocalStateStoresManager();

	final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder()
		.setShuffleEnvironment(nettyShuffleEnvironment)
		.setTaskSlotTable(taskSlotTable)
		.setJobManagerTable(jobManagerTable)
		.setTaskStateManager(localStateStoresManager)
		.build();

	TaskExecutor taskManager = createTaskExecutor(taskManagerServices);

	try {
		taskManager.start();

		final TaskExecutorGateway tmGateway = taskManager.getSelfGateway(TaskExecutorGateway.class);

		tmGateway.submitTask(tdd, jobMasterId, timeout);

		CompletableFuture<Boolean> completionFuture = TestInvokable.COMPLETABLE_FUTURE;

		completionFuture.get();

		taskInTerminalState.await();
	} finally {
		RpcUtils.terminateRpcEndpoint(taskManager, timeout);
	}
}
 
Example #26
Source Project: flink   Author: flink-tpc-ds   File: JvmExitOnFatalErrorTest.java    License: Apache License 2.0 4 votes vote down vote up
public static void main(String[] args) throws Exception {

			System.err.println("creating task");

			// we suppress process exits via errors here to not
			// have a test that exits accidentally due to a programming error
			try {
				final Configuration taskManagerConfig = new Configuration();
				taskManagerConfig.setBoolean(TaskManagerOptions.KILL_ON_OUT_OF_MEMORY, true);

				final JobID jid = new JobID();
				final AllocationID allocationID = new AllocationID();
				final JobVertexID jobVertexId = new JobVertexID();
				final ExecutionAttemptID executionAttemptID = new ExecutionAttemptID();
				final AllocationID slotAllocationId = new AllocationID();

				final SerializedValue<ExecutionConfig> execConfig = new SerializedValue<>(new ExecutionConfig());

				final JobInformation jobInformation = new JobInformation(
						jid, "Test Job", execConfig, new Configuration(),
						Collections.emptyList(), Collections.emptyList());

				final TaskInformation taskInformation = new TaskInformation(
						jobVertexId, "Test Task", 1, 1, OomInvokable.class.getName(), new Configuration());

				final MemoryManager memoryManager = new MemoryManager(1024 * 1024, 1);
				final IOManager ioManager = new IOManagerAsync();

				final ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

				final TaskManagerRuntimeInfo tmInfo = TaskManagerConfiguration.fromConfiguration(taskManagerConfig);

				final Executor executor = Executors.newCachedThreadPool();

				BlobCacheService blobService =
					new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

				final TaskLocalStateStore localStateStore =
					new TaskLocalStateStoreImpl(
						jid,
						allocationID,
						jobVertexId,
						0,
						TestLocalRecoveryConfig.disabled(),
						executor);

				final TaskStateManager slotStateManager =
					new TaskStateManagerImpl(
						jid,
						executionAttemptID,
						localStateStore,
						null,
						mock(CheckpointResponder.class));

				Task task = new Task(
						jobInformation,
						taskInformation,
						executionAttemptID,
						slotAllocationId,
						0,       // subtaskIndex
						0,       // attemptNumber
						Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
						Collections.<InputGateDeploymentDescriptor>emptyList(),
						0,       // targetSlotNumber
						memoryManager,
						ioManager,
						shuffleEnvironment,
						new KvStateService(new KvStateRegistry(), null, null),
						new BroadcastVariableManager(),
						new TaskEventDispatcher(),
						slotStateManager,
						new NoOpTaskManagerActions(),
						new NoOpInputSplitProvider(),
						new NoOpCheckpointResponder(),
						new TestGlobalAggregateManager(),
						blobService,
						new BlobLibraryCacheManager(
							blobService.getPermanentBlobService(),
							FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
							new String[0]),
						new FileCache(tmInfo.getTmpDirectories(), blobService.getPermanentBlobService()),
						tmInfo,
						UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
						new NoOpResultPartitionConsumableNotifier(),
						new NoOpPartitionProducerStateChecker(),
						executor);

				System.err.println("starting task thread");

				task.startTaskThread();
			}
			catch (Throwable t) {
				System.err.println("ERROR STARTING TASK");
				t.printStackTrace();
			}

			System.err.println("parking the main thread");
			CommonTestUtils.blockForeverNonInterruptibly();
		}
 
Example #27
Source Project: flink   Author: flink-tpc-ds   File: StreamTaskTerminationTest.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * FLINK-6833
 *
 * <p>Tests that a finished stream task cannot be failed by an asynchronous checkpointing operation after
 * the stream task has stopped running.
 */
@Test
public void testConcurrentAsyncCheckpointCannotFailFinishedStreamTask() throws Exception {
	final Configuration taskConfiguration = new Configuration();
	final StreamConfig streamConfig = new StreamConfig(taskConfiguration);
	final NoOpStreamOperator<Long> noOpStreamOperator = new NoOpStreamOperator<>();

	final StateBackend blockingStateBackend = new BlockingStateBackend();

	streamConfig.setStreamOperator(noOpStreamOperator);
	streamConfig.setOperatorID(new OperatorID());
	streamConfig.setStateBackend(blockingStateBackend);

	final long checkpointId = 0L;
	final long checkpointTimestamp = 0L;

	final JobInformation jobInformation = new JobInformation(
		new JobID(),
		"Test Job",
		new SerializedValue<>(new ExecutionConfig()),
		new Configuration(),
		Collections.emptyList(),
		Collections.emptyList());

	final TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"Test Task",
		1,
		1,
		BlockingStreamTask.class.getName(),
		taskConfiguration);

	final TaskManagerRuntimeInfo taskManagerRuntimeInfo = new TestingTaskManagerRuntimeInfo();

	final ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	final Task task = new Task(
		jobInformation,
		taskInformation,
		new ExecutionAttemptID(),
		new AllocationID(),
		0,
		0,
		Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
		Collections.<InputGateDeploymentDescriptor>emptyList(),
		0,
		new MemoryManager(32L * 1024L, 1),
		new IOManagerAsync(),
		shuffleEnvironment,
		new KvStateService(new KvStateRegistry(), null, null),
		mock(BroadcastVariableManager.class),
		new TaskEventDispatcher(),
		new TestTaskStateManager(),
		mock(TaskManagerActions.class),
		mock(InputSplitProvider.class),
		mock(CheckpointResponder.class),
		new TestGlobalAggregateManager(),
		blobService,
		new BlobLibraryCacheManager(
			blobService.getPermanentBlobService(),
			FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
			new String[0]),
		mock(FileCache.class),
		taskManagerRuntimeInfo,
		UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
		new NoOpResultPartitionConsumableNotifier(),
		mock(PartitionProducerStateChecker.class),
		Executors.directExecutor());

	CompletableFuture<Void> taskRun = CompletableFuture.runAsync(
		() -> task.run(),
		TestingUtils.defaultExecutor());

	// wait until the stream task started running
	RUN_LATCH.await();

	// trigger a checkpoint
	task.triggerCheckpointBarrier(checkpointId, checkpointTimestamp, CheckpointOptions.forCheckpointWithDefaultLocation(), false);

	// wait until the task has completed execution
	taskRun.get();

	// check that no failure occurred
	if (task.getFailureCause() != null) {
		throw new Exception("Task failed", task.getFailureCause());
	}

	// check that we have entered the finished state
	assertEquals(ExecutionState.FINISHED, task.getExecutionState());
}
 
Example #28
Source Project: flink   Author: flink-tpc-ds   File: SynchronousCheckpointITCase.java    License: Apache License 2.0 4 votes vote down vote up
private Task createTask(Class<? extends AbstractInvokable> invokableClass) throws Exception {
	BlobCacheService blobService =
			new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	LibraryCacheManager libCache = mock(LibraryCacheManager.class);
	when(libCache.getClassLoader(any(JobID.class))).thenReturn(ClassLoader.getSystemClassLoader());

	ResultPartitionConsumableNotifier consumableNotifier = new NoOpResultPartitionConsumableNotifier();
	PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
	Executor executor = mock(Executor.class);
	ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

	TaskMetricGroup taskMetricGroup = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();

	JobInformation jobInformation = new JobInformation(
			new JobID(),
			"Job Name",
			new SerializedValue<>(new ExecutionConfig()),
			new Configuration(),
			Collections.emptyList(),
			Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
			new JobVertexID(),
			"Test Task",
			1,
			1,
			invokableClass.getName(),
			new Configuration());

	return new Task(
			jobInformation,
			taskInformation,
			new ExecutionAttemptID(),
			new AllocationID(),
			0,
			0,
			Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
			Collections.<InputGateDeploymentDescriptor>emptyList(),
			0,
			mock(MemoryManager.class),
			mock(IOManager.class),
			shuffleEnvironment,
			new KvStateService(new KvStateRegistry(), null, null),
			mock(BroadcastVariableManager.class),
			new TaskEventDispatcher(),
			new TestTaskStateManager(),
			mock(TaskManagerActions.class),
			mock(InputSplitProvider.class),
			mock(CheckpointResponder.class),
			new TestGlobalAggregateManager(),
			blobService,
			libCache,
			mock(FileCache.class),
			new TestingTaskManagerRuntimeInfo(),
			taskMetricGroup,
			consumableNotifier,
			partitionProducerStateChecker,
			executor);
}
 
Example #29
Source Project: flink   Author: flink-tpc-ds   File: StreamTaskTest.java    License: Apache License 2.0 4 votes vote down vote up
public static Task createTask(
		Class<? extends AbstractInvokable> invokable,
		StreamConfig taskConfig,
		Configuration taskManagerConfig,
		TestTaskStateManager taskStateManager,
		TaskManagerActions taskManagerActions) throws Exception {

	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	LibraryCacheManager libCache = mock(LibraryCacheManager.class);
	when(libCache.getClassLoader(any(JobID.class))).thenReturn(StreamTaskTest.class.getClassLoader());

	ResultPartitionConsumableNotifier consumableNotifier = new NoOpResultPartitionConsumableNotifier();
	PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
	Executor executor = mock(Executor.class);

	ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

	JobInformation jobInformation = new JobInformation(
		new JobID(),
		"Job Name",
		new SerializedValue<>(new ExecutionConfig()),
		new Configuration(),
		Collections.emptyList(),
		Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"Test Task",
		1,
		1,
		invokable.getName(),
		taskConfig.getConfiguration());

	return new Task(
		jobInformation,
		taskInformation,
		new ExecutionAttemptID(),
		new AllocationID(),
		0,
		0,
		Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
		Collections.<InputGateDeploymentDescriptor>emptyList(),
		0,
		mock(MemoryManager.class),
		mock(IOManager.class),
		shuffleEnvironment,
		new KvStateService(new KvStateRegistry(), null, null),
		mock(BroadcastVariableManager.class),
		new TaskEventDispatcher(),
		taskStateManager,
		taskManagerActions,
		mock(InputSplitProvider.class),
		mock(CheckpointResponder.class),
		new TestGlobalAggregateManager(),
		blobService,
		libCache,
		mock(FileCache.class),
		new TestingTaskManagerRuntimeInfo(taskManagerConfig, new String[] {System.getProperty("java.io.tmpdir")}),
		UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
		consumableNotifier,
		partitionProducerStateChecker,
		executor);
}
 
Example #30
Source Project: flink   Author: flink-tpc-ds   File: TaskCheckpointingBehaviourTest.java    License: Apache License 2.0 4 votes vote down vote up
private static Task createTask(
	StreamOperator<?> op,
	StateBackend backend,
	CheckpointResponder checkpointResponder) throws IOException {

	Configuration taskConfig = new Configuration();
	StreamConfig cfg = new StreamConfig(taskConfig);
	cfg.setStreamOperator(op);
	cfg.setOperatorID(new OperatorID());
	cfg.setStateBackend(backend);

	ExecutionConfig executionConfig = new ExecutionConfig();

	JobInformation jobInformation = new JobInformation(
			new JobID(),
			"test job name",
			new SerializedValue<>(executionConfig),
			new Configuration(),
			Collections.emptyList(),
			Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
			new JobVertexID(),
			"test task name",
			1,
			11,
			TestStreamTask.class.getName(),
			taskConfig);

	ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	return new Task(
			jobInformation,
			taskInformation,
			new ExecutionAttemptID(),
			new AllocationID(),
			0,
			0,
			Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
			Collections.<InputGateDeploymentDescriptor>emptyList(),
			0,
			mock(MemoryManager.class),
			mock(IOManager.class),
			shuffleEnvironment,
			new KvStateService(new KvStateRegistry(), null, null),
			mock(BroadcastVariableManager.class),
			new TaskEventDispatcher(),
			new TestTaskStateManager(),
			mock(TaskManagerActions.class),
			mock(InputSplitProvider.class),
			checkpointResponder,
			new TestGlobalAggregateManager(),
			blobService,
			new BlobLibraryCacheManager(
				blobService.getPermanentBlobService(),
				FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
				new String[0]),
			new FileCache(new String[] { EnvironmentInformation.getTemporaryFileDirectory() },
				blobService.getPermanentBlobService()),
			new TestingTaskManagerRuntimeInfo(),
			UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
			new NoOpResultPartitionConsumableNotifier(),
			mock(PartitionProducerStateChecker.class),
			Executors.directExecutor());
}