Java Code Examples for org.apache.flink.runtime.taskmanager.CheckpointResponder

The following examples show how to use org.apache.flink.runtime.taskmanager.CheckpointResponder. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: JobManagerConnection.java    License: Apache License 2.0 6 votes vote down vote up
public JobManagerConnection(
			JobID jobID,
			ResourceID resourceID,
			JobMasterGateway jobMasterGateway,
			TaskManagerActions taskManagerActions,
			CheckpointResponder checkpointResponder,
			GlobalAggregateManager aggregateManager,
			LibraryCacheManager libraryCacheManager,
			ResultPartitionConsumableNotifier resultPartitionConsumableNotifier,
			PartitionProducerStateChecker partitionStateChecker) {
	this.jobID = Preconditions.checkNotNull(jobID);
	this.resourceID = Preconditions.checkNotNull(resourceID);
	this.jobMasterGateway = Preconditions.checkNotNull(jobMasterGateway);
	this.taskManagerActions = Preconditions.checkNotNull(taskManagerActions);
	this.checkpointResponder = Preconditions.checkNotNull(checkpointResponder);
	this.aggregateManager = Preconditions.checkNotNull(aggregateManager);
	this.libraryCacheManager = Preconditions.checkNotNull(libraryCacheManager);
	this.resultPartitionConsumableNotifier = Preconditions.checkNotNull(resultPartitionConsumableNotifier);
	this.partitionStateChecker = Preconditions.checkNotNull(partitionStateChecker);
}
 
Example 2
@Test
public void testBlockingNonInterruptibleCheckpoint() throws Exception {

	StateBackend lockingStateBackend = new BackendForTestStream(LockingOutputStream::new);

	Task task =
		createTask(new TestOperator(), lockingStateBackend, mock(CheckpointResponder.class), true);

	// start the task and wait until it is in "restore"
	task.startTaskThread();
	IN_CHECKPOINT_LATCH.await();

	// cancel the task and wait. unless cancellation properly closes
	// the streams, this will never terminate
	task.cancelExecution();
	task.getExecutingThread().join();

	assertEquals(ExecutionState.CANCELED, task.getExecutionState());
	assertNull(task.getFailureCause());
}
 
Example 3
Source Project: flink   Source File: JobManagerConnection.java    License: Apache License 2.0 6 votes vote down vote up
public JobManagerConnection(
			JobID jobID,
			ResourceID resourceID,
			JobMasterGateway jobMasterGateway,
			TaskManagerActions taskManagerActions,
			CheckpointResponder checkpointResponder,
			GlobalAggregateManager aggregateManager,
			LibraryCacheManager libraryCacheManager,
			ResultPartitionConsumableNotifier resultPartitionConsumableNotifier,
			PartitionProducerStateChecker partitionStateChecker) {
	this.jobID = Preconditions.checkNotNull(jobID);
	this.resourceID = Preconditions.checkNotNull(resourceID);
	this.jobMasterGateway = Preconditions.checkNotNull(jobMasterGateway);
	this.taskManagerActions = Preconditions.checkNotNull(taskManagerActions);
	this.checkpointResponder = Preconditions.checkNotNull(checkpointResponder);
	this.aggregateManager = Preconditions.checkNotNull(aggregateManager);
	this.libraryCacheManager = Preconditions.checkNotNull(libraryCacheManager);
	this.resultPartitionConsumableNotifier = Preconditions.checkNotNull(resultPartitionConsumableNotifier);
	this.partitionStateChecker = Preconditions.checkNotNull(partitionStateChecker);
}
 
Example 4
Source Project: flink   Source File: TaskSubmissionTestEnvironment.java    License: Apache License 2.0 6 votes vote down vote up
static JobManagerConnection createJobManagerConnection(JobID jobId, JobMasterGateway jobMasterGateway, RpcService testingRpcService, TaskManagerActions taskManagerActions, Time timeout) {
	final LibraryCacheManager libraryCacheManager = mock(LibraryCacheManager.class);
	when(libraryCacheManager.getClassLoader(any(JobID.class))).thenReturn(ClassLoader.getSystemClassLoader());

	final PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
	when(partitionProducerStateChecker.requestPartitionProducerState(any(), any(), any()))
		.thenReturn(CompletableFuture.completedFuture(ExecutionState.RUNNING));

	return new JobManagerConnection(
		jobId,
		ResourceID.generate(),
		jobMasterGateway,
		taskManagerActions,
		mock(CheckpointResponder.class),
		new TestGlobalAggregateManager(),
		libraryCacheManager,
		new RpcResultPartitionConsumableNotifier(jobMasterGateway, testingRpcService.getExecutor(), timeout),
		partitionProducerStateChecker);
}
 
Example 5
Source Project: flink   Source File: TaskCheckpointingBehaviourTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testBlockingNonInterruptibleCheckpoint() throws Exception {

	StateBackend lockingStateBackend = new BackendForTestStream(LockingOutputStream::new);

	Task task =
		createTask(new TestOperator(), lockingStateBackend, mock(CheckpointResponder.class));

	// start the task and wait until it is in "restore"
	task.startTaskThread();
	IN_CHECKPOINT_LATCH.await();

	// cancel the task and wait. unless cancellation properly closes
	// the streams, this will never terminate
	task.cancelExecution();
	task.getExecutingThread().join();

	assertEquals(ExecutionState.CANCELED, task.getExecutionState());
	assertNull(task.getFailureCause());
}
 
Example 6
Source Project: flink   Source File: DefaultJobTable.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public JobTable.Connection connect(
		ResourceID resourceId,
		JobMasterGateway jobMasterGateway,
		TaskManagerActions taskManagerActions,
		CheckpointResponder checkpointResponder,
		GlobalAggregateManager aggregateManager,
		ResultPartitionConsumableNotifier resultPartitionConsumableNotifier,
		PartitionProducerStateChecker partitionStateChecker) {
	verifyJobIsNotClosed();
	Preconditions.checkState(connection == null);

	connection = new EstablishedConnection(
		resourceId,
		jobMasterGateway,
		taskManagerActions,
		checkpointResponder,
		aggregateManager,
		resultPartitionConsumableNotifier,
		partitionStateChecker);
	resourceIdJobIdIndex.put(resourceId, jobId);

	return this;
}
 
Example 7
Source Project: flink   Source File: DefaultJobTable.java    License: Apache License 2.0 6 votes vote down vote up
private EstablishedConnection(
	ResourceID resourceID,
	JobMasterGateway jobMasterGateway,
	TaskManagerActions taskManagerActions,
	CheckpointResponder checkpointResponder,
	GlobalAggregateManager globalAggregateManager,
	ResultPartitionConsumableNotifier resultPartitionConsumableNotifier,
	PartitionProducerStateChecker partitionStateChecker) {
	this.resourceID = Preconditions.checkNotNull(resourceID);
	this.jobMasterGateway = Preconditions.checkNotNull(jobMasterGateway);
	this.taskManagerActions = Preconditions.checkNotNull(taskManagerActions);
	this.checkpointResponder = Preconditions.checkNotNull(checkpointResponder);
	this.globalAggregateManager = Preconditions.checkNotNull(globalAggregateManager);
	this.resultPartitionConsumableNotifier = Preconditions.checkNotNull(resultPartitionConsumableNotifier);
	this.partitionStateChecker = Preconditions.checkNotNull(partitionStateChecker);
}
 
Example 8
Source Project: flink   Source File: TaskCheckpointingBehaviourTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testBlockingNonInterruptibleCheckpoint() throws Exception {

	StateBackend lockingStateBackend = new BackendForTestStream(LockingOutputStream::new);

	Task task =
		createTask(new TestOperator(), lockingStateBackend, mock(CheckpointResponder.class));

	// start the task and wait until it is in "restore"
	task.startTaskThread();
	IN_CHECKPOINT_LATCH.await();

	// cancel the task and wait. unless cancellation properly closes
	// the streams, this will never terminate
	task.cancelExecution();
	task.getExecutingThread().join();

	assertEquals(ExecutionState.CANCELED, task.getExecutionState());
	assertNull(task.getFailureCause());
}
 
Example 9
Source Project: Flink-CEPplus   Source File: TaskExecutor.java    License: Apache License 2.0 5 votes vote down vote up
private JobManagerConnection associateWithJobManager(
		JobID jobID,
		ResourceID resourceID,
		JobMasterGateway jobMasterGateway) {
	checkNotNull(jobID);
	checkNotNull(resourceID);
	checkNotNull(jobMasterGateway);

	TaskManagerActions taskManagerActions = new TaskManagerActionsImpl(jobMasterGateway);

	CheckpointResponder checkpointResponder = new RpcCheckpointResponder(jobMasterGateway);
	GlobalAggregateManager aggregateManager = new RpcGlobalAggregateManager(jobMasterGateway);

	final LibraryCacheManager libraryCacheManager = new BlobLibraryCacheManager(
		blobCacheService.getPermanentBlobService(),
		taskManagerConfiguration.getClassLoaderResolveOrder(),
		taskManagerConfiguration.getAlwaysParentFirstLoaderPatterns());

	ResultPartitionConsumableNotifier resultPartitionConsumableNotifier = new RpcResultPartitionConsumableNotifier(
		jobMasterGateway,
		getRpcService().getExecutor(),
		taskManagerConfiguration.getTimeout());

	PartitionProducerStateChecker partitionStateChecker = new RpcPartitionStateChecker(jobMasterGateway);

	registerQueryableState(jobID, jobMasterGateway);

	return new JobManagerConnection(
		jobID,
		resourceID,
		jobMasterGateway,
		taskManagerActions,
		checkpointResponder,
		aggregateManager,
		libraryCacheManager,
		resultPartitionConsumableNotifier,
		partitionStateChecker);
}
 
Example 10
Source Project: Flink-CEPplus   Source File: TaskStateManagerImpl.java    License: Apache License 2.0 5 votes vote down vote up
public TaskStateManagerImpl(
	@Nonnull JobID jobId,
	@Nonnull ExecutionAttemptID executionAttemptID,
	@Nonnull TaskLocalStateStore localStateStore,
	@Nullable JobManagerTaskRestore jobManagerTaskRestore,
	@Nonnull CheckpointResponder checkpointResponder) {

	this.jobId = jobId;
	this.localStateStore = localStateStore;
	this.jobManagerTaskRestore = jobManagerTaskRestore;
	this.executionAttemptID = executionAttemptID;
	this.checkpointResponder = checkpointResponder;
}
 
Example 11
Source Project: Flink-CEPplus   Source File: TestTaskStateManager.java    License: Apache License 2.0 5 votes vote down vote up
public TestTaskStateManager(
	JobID jobId,
	ExecutionAttemptID executionAttemptID,
	CheckpointResponder checkpointResponder,
	LocalRecoveryConfig localRecoveryConfig) {
	this.jobId = jobId;
	this.executionAttemptID = executionAttemptID;
	this.checkpointResponder = checkpointResponder;
	this.localRecoveryDirectoryProvider = localRecoveryConfig;
	this.jobManagerTaskStateSnapshotsByCheckpointId = new HashMap<>();
	this.taskManagerTaskStateSnapshotsByCheckpointId = new HashMap<>();
	this.reportedCheckpointId = -1L;
}
 
Example 12
Source Project: Flink-CEPplus   Source File: TaskStateManagerImplTest.java    License: Apache License 2.0 5 votes vote down vote up
public static TaskStateManager taskStateManager(
	JobID jobID,
	ExecutionAttemptID executionAttemptID,
	CheckpointResponder checkpointResponderMock,
	JobManagerTaskRestore jobManagerTaskRestore,
	TaskLocalStateStore localStateStore) {

	return new TaskStateManagerImpl(
		jobID,
		executionAttemptID,
		localStateStore,
		jobManagerTaskRestore,
		checkpointResponderMock);
}
 
Example 13
Source Project: flink   Source File: TaskExecutor.java    License: Apache License 2.0 5 votes vote down vote up
private JobManagerConnection associateWithJobManager(
		JobID jobID,
		ResourceID resourceID,
		JobMasterGateway jobMasterGateway) {
	checkNotNull(jobID);
	checkNotNull(resourceID);
	checkNotNull(jobMasterGateway);

	TaskManagerActions taskManagerActions = new TaskManagerActionsImpl(jobMasterGateway);

	CheckpointResponder checkpointResponder = new RpcCheckpointResponder(jobMasterGateway);
	GlobalAggregateManager aggregateManager = new RpcGlobalAggregateManager(jobMasterGateway);

	final LibraryCacheManager libraryCacheManager = new BlobLibraryCacheManager(
		blobCacheService.getPermanentBlobService(),
		taskManagerConfiguration.getClassLoaderResolveOrder(),
		taskManagerConfiguration.getAlwaysParentFirstLoaderPatterns());

	ResultPartitionConsumableNotifier resultPartitionConsumableNotifier = new RpcResultPartitionConsumableNotifier(
		jobMasterGateway,
		getRpcService().getExecutor(),
		taskManagerConfiguration.getTimeout());

	PartitionProducerStateChecker partitionStateChecker = new RpcPartitionStateChecker(jobMasterGateway);

	registerQueryableState(jobID, jobMasterGateway);

	return new JobManagerConnection(
		jobID,
		resourceID,
		jobMasterGateway,
		taskManagerActions,
		checkpointResponder,
		aggregateManager,
		libraryCacheManager,
		resultPartitionConsumableNotifier,
		partitionStateChecker);
}
 
Example 14
Source Project: flink   Source File: TaskStateManagerImpl.java    License: Apache License 2.0 5 votes vote down vote up
public TaskStateManagerImpl(
	@Nonnull JobID jobId,
	@Nonnull ExecutionAttemptID executionAttemptID,
	@Nonnull TaskLocalStateStore localStateStore,
	@Nullable JobManagerTaskRestore jobManagerTaskRestore,
	@Nonnull CheckpointResponder checkpointResponder) {

	this.jobId = jobId;
	this.localStateStore = localStateStore;
	this.jobManagerTaskRestore = jobManagerTaskRestore;
	this.executionAttemptID = executionAttemptID;
	this.checkpointResponder = checkpointResponder;
}
 
Example 15
Source Project: flink   Source File: TestTaskStateManager.java    License: Apache License 2.0 5 votes vote down vote up
public TestTaskStateManager(
	JobID jobId,
	ExecutionAttemptID executionAttemptID,
	CheckpointResponder checkpointResponder,
	LocalRecoveryConfig localRecoveryConfig) {
	this.jobId = jobId;
	this.executionAttemptID = executionAttemptID;
	this.checkpointResponder = checkpointResponder;
	this.localRecoveryDirectoryProvider = localRecoveryConfig;
	this.jobManagerTaskStateSnapshotsByCheckpointId = new HashMap<>();
	this.taskManagerTaskStateSnapshotsByCheckpointId = new HashMap<>();
	this.reportedCheckpointId = -1L;
}
 
Example 16
Source Project: flink   Source File: TaskStateManagerImplTest.java    License: Apache License 2.0 5 votes vote down vote up
public static TaskStateManager taskStateManager(
	JobID jobID,
	ExecutionAttemptID executionAttemptID,
	CheckpointResponder checkpointResponderMock,
	JobManagerTaskRestore jobManagerTaskRestore,
	TaskLocalStateStore localStateStore) {

	return new TaskStateManagerImpl(
		jobID,
		executionAttemptID,
		localStateStore,
		jobManagerTaskRestore,
		checkpointResponderMock);
}
 
Example 17
Source Project: flink   Source File: TaskExecutor.java    License: Apache License 2.0 5 votes vote down vote up
private JobTable.Connection associateWithJobManager(
		JobTable.Job job,
		ResourceID resourceID,
		JobMasterGateway jobMasterGateway) {
	checkNotNull(resourceID);
	checkNotNull(jobMasterGateway);

	TaskManagerActions taskManagerActions = new TaskManagerActionsImpl(jobMasterGateway);

	CheckpointResponder checkpointResponder = new RpcCheckpointResponder(jobMasterGateway);
	GlobalAggregateManager aggregateManager = new RpcGlobalAggregateManager(jobMasterGateway);

	ResultPartitionConsumableNotifier resultPartitionConsumableNotifier = new RpcResultPartitionConsumableNotifier(
		jobMasterGateway,
		getRpcService().getExecutor(),
		taskManagerConfiguration.getTimeout());

	PartitionProducerStateChecker partitionStateChecker = new RpcPartitionStateChecker(jobMasterGateway);

	registerQueryableState(job.getJobId(), jobMasterGateway);

	return job.connect(
		resourceID,
		jobMasterGateway,
		taskManagerActions,
		checkpointResponder,
		aggregateManager,
		resultPartitionConsumableNotifier,
		partitionStateChecker);
}
 
Example 18
Source Project: flink   Source File: TaskStateManagerImpl.java    License: Apache License 2.0 5 votes vote down vote up
public TaskStateManagerImpl(
		@Nonnull JobID jobId,
		@Nonnull ExecutionAttemptID executionAttemptID,
		@Nonnull TaskLocalStateStore localStateStore,
		@Nullable JobManagerTaskRestore jobManagerTaskRestore,
		@Nonnull CheckpointResponder checkpointResponder) {
	this(
		jobId,
		executionAttemptID,
		localStateStore,
		jobManagerTaskRestore,
		checkpointResponder,
		new ChannelStateReaderImpl(jobManagerTaskRestore == null ? new TaskStateSnapshot() : jobManagerTaskRestore.getTaskStateSnapshot())
	);
}
 
Example 19
Source Project: flink   Source File: TaskStateManagerImpl.java    License: Apache License 2.0 5 votes vote down vote up
public TaskStateManagerImpl(
		@Nonnull JobID jobId,
		@Nonnull ExecutionAttemptID executionAttemptID,
		@Nonnull TaskLocalStateStore localStateStore,
		@Nullable JobManagerTaskRestore jobManagerTaskRestore,
		@Nonnull CheckpointResponder checkpointResponder,
		@Nonnull ChannelStateReader channelStateReader) {
	this.jobId = jobId;
	this.localStateStore = localStateStore;
	this.jobManagerTaskRestore = jobManagerTaskRestore;
	this.executionAttemptID = executionAttemptID;
	this.checkpointResponder = checkpointResponder;
	this.channelStateReader = channelStateReader;
}
 
Example 20
Source Project: flink   Source File: TestTaskStateManager.java    License: Apache License 2.0 5 votes vote down vote up
public TestTaskStateManager(
	JobID jobId,
	ExecutionAttemptID executionAttemptID,
	CheckpointResponder checkpointResponder,
	LocalRecoveryConfig localRecoveryConfig) {
	this.jobId = jobId;
	this.executionAttemptID = executionAttemptID;
	this.checkpointResponder = checkpointResponder;
	this.localRecoveryDirectoryProvider = localRecoveryConfig;
	this.jobManagerTaskStateSnapshotsByCheckpointId = new HashMap<>();
	this.taskManagerTaskStateSnapshotsByCheckpointId = new HashMap<>();
	this.reportedCheckpointId = -1L;
	this.notifiedCompletedCheckpointId = -1L;
	this.notifiedAbortedCheckpointId = -1L;
}
 
Example 21
Source Project: flink   Source File: TaskStateManagerImplTest.java    License: Apache License 2.0 5 votes vote down vote up
public static TaskStateManager taskStateManager(
	JobID jobID,
	ExecutionAttemptID executionAttemptID,
	CheckpointResponder checkpointResponderMock,
	JobManagerTaskRestore jobManagerTaskRestore,
	TaskLocalStateStore localStateStore) {

	return new TaskStateManagerImpl(
		jobID,
		executionAttemptID,
		localStateStore,
		jobManagerTaskRestore,
		checkpointResponderMock);
}
 
Example 22
Source Project: Flink-CEPplus   Source File: JobManagerConnection.java    License: Apache License 2.0 4 votes vote down vote up
public CheckpointResponder getCheckpointResponder() {
	return checkpointResponder;
}
 
Example 23
Source Project: Flink-CEPplus   Source File: TestTaskStateManager.java    License: Apache License 2.0 4 votes vote down vote up
public CheckpointResponder getCheckpointResponder() {
	return checkpointResponder;
}
 
Example 24
Source Project: Flink-CEPplus   Source File: TestTaskStateManager.java    License: Apache License 2.0 4 votes vote down vote up
public void setCheckpointResponder(CheckpointResponder checkpointResponder) {
	this.checkpointResponder = checkpointResponder;
}
 
Example 25
Source Project: Flink-CEPplus   Source File: JvmExitOnFatalErrorTest.java    License: Apache License 2.0 4 votes vote down vote up
public static void main(String[] args) throws Exception {

			System.err.println("creating task");

			// we suppress process exits via errors here to not
			// have a test that exits accidentally due to a programming error
			try {
				final Configuration taskManagerConfig = new Configuration();
				taskManagerConfig.setBoolean(TaskManagerOptions.KILL_ON_OUT_OF_MEMORY, true);

				final JobID jid = new JobID();
				final AllocationID allocationID = new AllocationID();
				final JobVertexID jobVertexId = new JobVertexID();
				final ExecutionAttemptID executionAttemptID = new ExecutionAttemptID();
				final AllocationID slotAllocationId = new AllocationID();

				final SerializedValue<ExecutionConfig> execConfig = new SerializedValue<>(new ExecutionConfig());

				final JobInformation jobInformation = new JobInformation(
						jid, "Test Job", execConfig, new Configuration(),
						Collections.emptyList(), Collections.emptyList());

				final TaskInformation taskInformation = new TaskInformation(
						jobVertexId, "Test Task", 1, 1, OomInvokable.class.getName(), new Configuration());

				final MemoryManager memoryManager = new MemoryManager(1024 * 1024, 1);
				final IOManager ioManager = new IOManagerAsync();

				final NetworkEnvironment networkEnvironment = mock(NetworkEnvironment.class);
				when(networkEnvironment.createKvStateTaskRegistry(jid, jobVertexId)).thenReturn(mock(TaskKvStateRegistry.class));
				TaskEventDispatcher taskEventDispatcher = mock(TaskEventDispatcher.class);
				when(networkEnvironment.getTaskEventDispatcher()).thenReturn(taskEventDispatcher);

				final TaskManagerRuntimeInfo tmInfo = TaskManagerConfiguration.fromConfiguration(taskManagerConfig);

				final Executor executor = Executors.newCachedThreadPool();

				BlobCacheService blobService =
					new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

				final TaskLocalStateStore localStateStore =
					new TaskLocalStateStoreImpl(
						jid,
						allocationID,
						jobVertexId,
						0,
						TestLocalRecoveryConfig.disabled(),
						executor);

				final TaskStateManager slotStateManager =
					new TaskStateManagerImpl(
						jid,
						executionAttemptID,
						localStateStore,
						null,
						mock(CheckpointResponder.class));

				Task task = new Task(
						jobInformation,
						taskInformation,
						executionAttemptID,
						slotAllocationId,
						0,       // subtaskIndex
						0,       // attemptNumber
						Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
						Collections.<InputGateDeploymentDescriptor>emptyList(),
						0,       // targetSlotNumber
						memoryManager,
						ioManager,
						networkEnvironment,
						new BroadcastVariableManager(),
						slotStateManager,
						new NoOpTaskManagerActions(),
						new NoOpInputSplitProvider(),
						new NoOpCheckpointResponder(),
						new TestGlobalAggregateManager(),
						blobService,
						new BlobLibraryCacheManager(
							blobService.getPermanentBlobService(),
							FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
							new String[0]),
						new FileCache(tmInfo.getTmpDirectories(), blobService.getPermanentBlobService()),
						tmInfo,
						UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
						new NoOpResultPartitionConsumableNotifier(),
						new NoOpPartitionProducerStateChecker(),
						executor);

				System.err.println("starting task thread");

				task.startTaskThread();
			}
			catch (Throwable t) {
				System.err.println("ERROR STARTING TASK");
				t.printStackTrace();
			}

			System.err.println("parking the main thread");
			CommonTestUtils.blockForeverNonInterruptibly();
		}
 
Example 26
Source Project: Flink-CEPplus   Source File: StreamTaskTerminationTest.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * FLINK-6833
 *
 * <p>Tests that a finished stream task cannot be failed by an asynchronous checkpointing operation after
 * the stream task has stopped running.
 */
@Test
public void testConcurrentAsyncCheckpointCannotFailFinishedStreamTask() throws Exception {
	final Configuration taskConfiguration = new Configuration();
	final StreamConfig streamConfig = new StreamConfig(taskConfiguration);
	final NoOpStreamOperator<Long> noOpStreamOperator = new NoOpStreamOperator<>();

	final StateBackend blockingStateBackend = new BlockingStateBackend();

	streamConfig.setStreamOperator(noOpStreamOperator);
	streamConfig.setOperatorID(new OperatorID());
	streamConfig.setStateBackend(blockingStateBackend);

	final long checkpointId = 0L;
	final long checkpointTimestamp = 0L;

	final JobInformation jobInformation = new JobInformation(
		new JobID(),
		"Test Job",
		new SerializedValue<>(new ExecutionConfig()),
		new Configuration(),
		Collections.emptyList(),
		Collections.emptyList());

	final TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"Test Task",
		1,
		1,
		BlockingStreamTask.class.getName(),
		taskConfiguration);

	final TaskManagerRuntimeInfo taskManagerRuntimeInfo = new TestingTaskManagerRuntimeInfo();

	TaskEventDispatcher taskEventDispatcher = new TaskEventDispatcher();
	final NetworkEnvironment networkEnv = mock(NetworkEnvironment.class);
	when(networkEnv.createKvStateTaskRegistry(any(JobID.class), any(JobVertexID.class))).thenReturn(mock(TaskKvStateRegistry.class));
	when(networkEnv.getTaskEventDispatcher()).thenReturn(taskEventDispatcher);

	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	final Task task = new Task(
		jobInformation,
		taskInformation,
		new ExecutionAttemptID(),
		new AllocationID(),
		0,
		0,
		Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
		Collections.<InputGateDeploymentDescriptor>emptyList(),
		0,
		new MemoryManager(32L * 1024L, 1),
		new IOManagerAsync(),
		networkEnv,
		mock(BroadcastVariableManager.class),
		new TestTaskStateManager(),
		mock(TaskManagerActions.class),
		mock(InputSplitProvider.class),
		mock(CheckpointResponder.class),
		new TestGlobalAggregateManager(),
		blobService,
		new BlobLibraryCacheManager(
			blobService.getPermanentBlobService(),
			FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
			new String[0]),
		mock(FileCache.class),
		taskManagerRuntimeInfo,
		UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
		new NoOpResultPartitionConsumableNotifier(),
		mock(PartitionProducerStateChecker.class),
		Executors.directExecutor());

	CompletableFuture<Void> taskRun = CompletableFuture.runAsync(
		() -> task.run(),
		TestingUtils.defaultExecutor());

	// wait until the stream task started running
	RUN_LATCH.await();

	// trigger a checkpoint
	task.triggerCheckpointBarrier(checkpointId, checkpointTimestamp, CheckpointOptions.forCheckpointWithDefaultLocation());

	// wait until the task has completed execution
	taskRun.get();

	// check that no failure occurred
	if (task.getFailureCause() != null) {
		throw new Exception("Task failed", task.getFailureCause());
	}

	// check that we have entered the finished state
	assertEquals(ExecutionState.FINISHED, task.getExecutionState());
}
 
Example 27
Source Project: Flink-CEPplus   Source File: StreamTaskTest.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * FLINK-5985
 *
 * <p>This test ensures that empty snapshots (no op/keyed stated whatsoever) will be reported as stateless tasks. This
 * happens by translating an empty {@link SubtaskState} into reporting 'null' to #acknowledgeCheckpoint.
 */
@Test
public void testEmptySubtaskStateLeadsToStatelessAcknowledgment() throws Exception {
	final long checkpointId = 42L;
	final long timestamp = 1L;

	Environment mockEnvironment = spy(new MockEnvironmentBuilder().build());

	// latch blocks until the async checkpoint thread acknowledges
	final OneShotLatch checkpointCompletedLatch = new OneShotLatch();
	final List<SubtaskState> checkpointResult = new ArrayList<>(1);

	CheckpointResponder checkpointResponder = mock(CheckpointResponder.class);
	doAnswer(new Answer() {
		@Override
		public Object answer(InvocationOnMock invocation) throws Throwable {
			SubtaskState subtaskState = invocation.getArgument(4);
			checkpointResult.add(subtaskState);
			checkpointCompletedLatch.trigger();
			return null;
		}
	}).when(checkpointResponder).acknowledgeCheckpoint(
		any(JobID.class),
		any(ExecutionAttemptID.class),
		anyLong(),
		any(CheckpointMetrics.class),
		nullable(TaskStateSnapshot.class));

	TaskStateManager taskStateManager = new TaskStateManagerImpl(
		new JobID(1L, 2L),
		new ExecutionAttemptID(1L, 2L),
		mock(TaskLocalStateStoreImpl.class),
		null,
		checkpointResponder);

	when(mockEnvironment.getTaskStateManager()).thenReturn(taskStateManager);

	StreamTask<?, ?> streamTask = new EmptyStreamTask(mockEnvironment);
	CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, timestamp);

	// mock the operators
	StreamOperator<?> statelessOperator =
			mock(StreamOperator.class);

	final OperatorID operatorID = new OperatorID();
	when(statelessOperator.getOperatorID()).thenReturn(operatorID);

	// mock the returned empty snapshot result (all state handles are null)
	OperatorSnapshotFutures statelessOperatorSnapshotResult = new OperatorSnapshotFutures();
	when(statelessOperator.snapshotState(anyLong(), anyLong(), any(CheckpointOptions.class), any(CheckpointStreamFactory.class)))
			.thenReturn(statelessOperatorSnapshotResult);

	// set up the task
	StreamOperator<?>[] streamOperators = {statelessOperator};
	OperatorChain<Void, AbstractStreamOperator<Void>> operatorChain = mock(OperatorChain.class);
	when(operatorChain.getAllOperators()).thenReturn(streamOperators);

	Whitebox.setInternalState(streamTask, "isRunning", true);
	Whitebox.setInternalState(streamTask, "lock", new Object());
	Whitebox.setInternalState(streamTask, "operatorChain", operatorChain);
	Whitebox.setInternalState(streamTask, "cancelables", new CloseableRegistry());
	Whitebox.setInternalState(streamTask, "configuration", new StreamConfig(new Configuration()));
	Whitebox.setInternalState(streamTask, "asyncOperationsThreadPool", Executors.newCachedThreadPool());
	Whitebox.setInternalState(streamTask, "checkpointStorage", new MemoryBackendCheckpointStorage(new JobID(), null, null, Integer.MAX_VALUE));

	streamTask.triggerCheckpoint(checkpointMetaData, CheckpointOptions.forCheckpointWithDefaultLocation());
	checkpointCompletedLatch.await(30, TimeUnit.SECONDS);
	streamTask.cancel();

	// ensure that 'null' was acknowledged as subtask state
	Assert.assertNull(checkpointResult.get(0));
}
 
Example 28
Source Project: Flink-CEPplus   Source File: StreamTaskTest.java    License: Apache License 2.0 4 votes vote down vote up
public static Task createTask(
		Class<? extends AbstractInvokable> invokable,
		StreamConfig taskConfig,
		Configuration taskManagerConfig,
		TestTaskStateManager taskStateManager,
		TaskManagerActions taskManagerActions) throws Exception {

	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	LibraryCacheManager libCache = mock(LibraryCacheManager.class);
	when(libCache.getClassLoader(any(JobID.class))).thenReturn(StreamTaskTest.class.getClassLoader());

	ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
	ResultPartitionConsumableNotifier consumableNotifier = new NoOpResultPartitionConsumableNotifier();
	PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
	Executor executor = mock(Executor.class);
	TaskEventDispatcher taskEventDispatcher = new TaskEventDispatcher();

	NetworkEnvironment network = mock(NetworkEnvironment.class);
	when(network.getResultPartitionManager()).thenReturn(partitionManager);
	when(network.getDefaultIOMode()).thenReturn(IOManager.IOMode.SYNC);
	when(network.createKvStateTaskRegistry(any(JobID.class), any(JobVertexID.class)))
			.thenReturn(mock(TaskKvStateRegistry.class));
	when(network.getTaskEventDispatcher()).thenReturn(taskEventDispatcher);

	JobInformation jobInformation = new JobInformation(
		new JobID(),
		"Job Name",
		new SerializedValue<>(new ExecutionConfig()),
		new Configuration(),
		Collections.emptyList(),
		Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"Test Task",
		1,
		1,
		invokable.getName(),
		taskConfig.getConfiguration());

	return new Task(
		jobInformation,
		taskInformation,
		new ExecutionAttemptID(),
		new AllocationID(),
		0,
		0,
		Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
		Collections.<InputGateDeploymentDescriptor>emptyList(),
		0,
		mock(MemoryManager.class),
		mock(IOManager.class),
		network,
		mock(BroadcastVariableManager.class),
		taskStateManager,
		taskManagerActions,
		mock(InputSplitProvider.class),
		mock(CheckpointResponder.class),
		new TestGlobalAggregateManager(),
		blobService,
		libCache,
		mock(FileCache.class),
		new TestingTaskManagerRuntimeInfo(taskManagerConfig, new String[] {System.getProperty("java.io.tmpdir")}),
		UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
		consumableNotifier,
		partitionProducerStateChecker,
		executor);
}
 
Example 29
private static Task createTask(
	StreamOperator<?> op,
	StateBackend backend,
	CheckpointResponder checkpointResponder,
	boolean failOnCheckpointErrors) throws IOException {

	Configuration taskConfig = new Configuration();
	StreamConfig cfg = new StreamConfig(taskConfig);
	cfg.setStreamOperator(op);
	cfg.setOperatorID(new OperatorID());
	cfg.setStateBackend(backend);

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setFailTaskOnCheckpointError(failOnCheckpointErrors);

	JobInformation jobInformation = new JobInformation(
			new JobID(),
			"test job name",
			new SerializedValue<>(executionConfig),
			new Configuration(),
			Collections.emptyList(),
			Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
			new JobVertexID(),
			"test task name",
			1,
			11,
			TestStreamTask.class.getName(),
			taskConfig);

	TaskKvStateRegistry mockKvRegistry = mock(TaskKvStateRegistry.class);
	TaskEventDispatcher taskEventDispatcher = new TaskEventDispatcher();
	NetworkEnvironment network = mock(NetworkEnvironment.class);
	when(network.createKvStateTaskRegistry(any(JobID.class), any(JobVertexID.class))).thenReturn(mockKvRegistry);
	when(network.getTaskEventDispatcher()).thenReturn(taskEventDispatcher);

	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	return new Task(
			jobInformation,
			taskInformation,
			new ExecutionAttemptID(),
			new AllocationID(),
			0,
			0,
			Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
			Collections.<InputGateDeploymentDescriptor>emptyList(),
			0,
			mock(MemoryManager.class),
			mock(IOManager.class),
			network,
			mock(BroadcastVariableManager.class),
			new TestTaskStateManager(),
			mock(TaskManagerActions.class),
			mock(InputSplitProvider.class),
			checkpointResponder,
			new TestGlobalAggregateManager(),
			blobService,
			new BlobLibraryCacheManager(
				blobService.getPermanentBlobService(),
				FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
				new String[0]),
			new FileCache(new String[] { EnvironmentInformation.getTemporaryFileDirectory() },
				blobService.getPermanentBlobService()),
			new TestingTaskManagerRuntimeInfo(),
			UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
			new NoOpResultPartitionConsumableNotifier(),
			mock(PartitionProducerStateChecker.class),
			Executors.directExecutor());
}
 
Example 30
Source Project: flink   Source File: JobManagerConnection.java    License: Apache License 2.0 4 votes vote down vote up
public CheckpointResponder getCheckpointResponder() {
	return checkpointResponder;
}