org.apache.flink.runtime.taskmanager.Task Java Examples

The following examples show how to use org.apache.flink.runtime.taskmanager.Task. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: flink   Author: apache   File: TaskExecutorTest.java    License: Apache License 2.0 6 votes vote down vote up
private TaskExecutorTestingContext createTaskExecutorTestingContext(final TaskSlotTable<Task> taskSlotTable) throws IOException {
	final OneShotLatch offerSlotsLatch = new OneShotLatch();
	final TestingJobMasterGateway jobMasterGateway = new TestingJobMasterGatewayBuilder()
		.setOfferSlotsFunction((resourceID, slotOffers) -> {
			offerSlotsLatch.trigger();
			return CompletableFuture.completedFuture(slotOffers);
		}).build();
	rpc.registerGateway(jobMasterGateway.getAddress(), jobMasterGateway);

	final JobLeaderService jobLeaderService = new DefaultJobLeaderService(
		unresolvedTaskManagerLocation,
		RetryingRegistrationConfiguration.defaultConfiguration());

	TaskExecutorLocalStateStoresManager stateStoresManager = createTaskExecutorLocalStateStoresManager();
	final TestingTaskExecutor taskExecutor = createTestingTaskExecutor(new TaskManagerServicesBuilder()
		.setTaskSlotTable(taskSlotTable)
		.setJobLeaderService(jobLeaderService)
		.setTaskStateManager(stateStoresManager)
		.build());

	jobManagerLeaderRetriever.notifyListener(jobMasterGateway.getAddress(), jobMasterGateway.getFencingToken().toUUID());
	return new TaskExecutorTestingContext(jobMasterGateway, taskSlotTable, taskExecutor);
}
 
Example #2
Source Project: flink   Author: apache   File: TaskExecutor.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<Acknowledge> cancelTask(ExecutionAttemptID executionAttemptID, Time timeout) {
	final Task task = taskSlotTable.getTask(executionAttemptID);

	if (task != null) {
		try {
			task.cancelExecution();
			return CompletableFuture.completedFuture(Acknowledge.get());
		} catch (Throwable t) {
			return FutureUtils.completedExceptionally(
				new TaskException("Cannot cancel task for execution " + executionAttemptID + '.', t));
		}
	} else {
		final String message = "Cannot find task to stop for execution " + executionAttemptID + '.';

		log.debug(message);
		return FutureUtils.completedExceptionally(new TaskException(message));
	}
}
 
Example #3
Source Project: Flink-CEPplus   Author: ljygz   File: TaskExecutor.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<Acknowledge> confirmCheckpoint(
		ExecutionAttemptID executionAttemptID,
		long checkpointId,
		long checkpointTimestamp) {
	log.debug("Confirm checkpoint {}@{} for {}.", checkpointId, checkpointTimestamp, executionAttemptID);

	final Task task = taskSlotTable.getTask(executionAttemptID);

	if (task != null) {
		task.notifyCheckpointComplete(checkpointId);

		return CompletableFuture.completedFuture(Acknowledge.get());
	} else {
		final String message = "TaskManager received a checkpoint confirmation for unknown task " + executionAttemptID + '.';

		log.debug(message);
		return FutureUtils.completedExceptionally(new CheckpointException(message));
	}
}
 
Example #4
Source Project: flink   Author: apache   File: TaskExecutorTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTaskSlotTableTerminationOnShutdown() throws Exception {
	CompletableFuture<Void> taskSlotTableClosingFuture = new CompletableFuture<>();
	TaskExecutorTestingContext submissionContext = createTaskExecutorTestingContext(
		TestingTaskSlotTable.<Task>newBuilder().closeAsyncReturns(taskSlotTableClosingFuture).build());
	final CompletableFuture<Void> taskExecutorTerminationFuture;
	try {
		submissionContext.start();
	} finally {
		taskExecutorTerminationFuture = submissionContext.taskExecutor.closeAsync();
	}

	// check task executor is waiting for the task completion and has not terminated yet
	assertThat(taskExecutorTerminationFuture.isDone(), is(false));

	// check task executor has exited after task slot table termination
	taskSlotTableClosingFuture.complete(null);
	taskExecutorTerminationFuture.get();
}
 
Example #5
Source Project: Flink-CEPplus   Author: ljygz   File: TaskSlot.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Add the given task to the task slot. This is only possible if there is not already another
 * task with the same execution attempt id added to the task slot. In this case, the method
 * returns true. Otherwise the task slot is left unchanged and false is returned.
 *
 * <p>In case that the task slot state is not active an {@link IllegalStateException} is thrown.
 * In case that the task's job id and allocation id don't match with the job id and allocation
 * id for which the task slot has been allocated, an {@link IllegalArgumentException} is thrown.
 *
 * @param task to be added to the task slot
 * @throws IllegalStateException if the task slot is not in state active
 * @return true if the task was added to the task slot; otherwise false
 */
public boolean add(Task task) {
	// Check that this slot has been assigned to the job sending this task
	Preconditions.checkArgument(task.getJobID().equals(jobId), "The task's job id does not match the " +
		"job id for which the slot has been allocated.");
	Preconditions.checkArgument(task.getAllocationId().equals(allocationId), "The task's allocation " +
		"id does not match the allocation id for which the slot has been allocated.");
	Preconditions.checkState(TaskSlotState.ACTIVE == state, "The task slot is not in state active.");

	Task oldTask = tasks.put(task.getExecutionId(), task);

	if (oldTask != null) {
		tasks.put(task.getExecutionId(), oldTask);
		return false;
	} else {
		return true;
	}
}
 
Example #6
Source Project: flink   Author: apache   File: TaskCheckpointingBehaviourTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testBlockingNonInterruptibleCheckpoint() throws Exception {

	StateBackend lockingStateBackend = new BackendForTestStream(LockingOutputStream::new);

	Task task =
		createTask(new TestOperator(), lockingStateBackend, mock(CheckpointResponder.class));

	// start the task and wait until it is in "restore"
	task.startTaskThread();
	IN_CHECKPOINT_LATCH.await();

	// cancel the task and wait. unless cancellation properly closes
	// the streams, this will never terminate
	task.cancelExecution();
	task.getExecutingThread().join();

	assertEquals(ExecutionState.CANCELED, task.getExecutionState());
	assertNull(task.getFailureCause());
}
 
Example #7
Source Project: flink   Author: apache   File: TaskExecutor.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<Acknowledge> abortCheckpoint(
		ExecutionAttemptID executionAttemptID,
		long checkpointId,
		long checkpointTimestamp) {
	log.debug("Abort checkpoint {}@{} for {}.", checkpointId, checkpointTimestamp, executionAttemptID);

	final Task task = taskSlotTable.getTask(executionAttemptID);

	if (task != null) {
		task.notifyCheckpointAborted(checkpointId);

		return CompletableFuture.completedFuture(Acknowledge.get());
	} else {
		final String message = "TaskManager received an aborted checkpoint for unknown task " + executionAttemptID + '.';

		log.debug(message);
		return FutureUtils.completedExceptionally(new CheckpointException(message, CheckpointFailureReason.UNKNOWN_TASK_CHECKPOINT_NOTIFICATION_FAILURE));
	}
}
 
Example #8
Source Project: Flink-CEPplus   Author: ljygz   File: NetworkEnvironment.java    License: Apache License 2.0 6 votes vote down vote up
public void registerTask(Task task) throws IOException {
	final ResultPartition[] producedPartitions = task.getProducedPartitions();

	synchronized (lock) {
		if (isShutdown) {
			throw new IllegalStateException("NetworkEnvironment is shut down");
		}

		for (final ResultPartition partition : producedPartitions) {
			setupPartition(partition);
		}

		// Setup the buffer pool for each buffer reader
		final SingleInputGate[] inputGates = task.getAllInputGates();
		for (SingleInputGate gate : inputGates) {
			setupInputGate(gate);
		}
	}
}
 
Example #9
Source Project: Flink-CEPplus   Author: ljygz   File: AbstractUdfStreamOperatorLifecycleTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testLifeCycleFull() throws Exception {
	ACTUAL_ORDER_TRACKING.clear();

	Configuration taskManagerConfig = new Configuration();
	StreamConfig cfg = new StreamConfig(new Configuration());
	MockSourceFunction srcFun = new MockSourceFunction();

	cfg.setStreamOperator(new LifecycleTrackingStreamSource<>(srcFun, true));
	cfg.setOperatorID(new OperatorID());
	cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	Task task = StreamTaskTest.createTask(SourceStreamTask.class, cfg, taskManagerConfig);

	task.startTaskThread();

	LifecycleTrackingStreamSource.runStarted.await();

	// wait for clean termination
	task.getExecutingThread().join();
	assertEquals(ExecutionState.FINISHED, task.getExecutionState());
	assertEquals(EXPECTED_CALL_ORDER_FULL, ACTUAL_ORDER_TRACKING);
}
 
Example #10
Source Project: flink   Author: apache   File: TaskExecutor.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public AccumulatorReport retrievePayload(ResourceID resourceID) {
	validateRunsInMainThread();
	return jobTable.getConnection(resourceID).map(
		jobManagerConnection -> {
			JobID jobId = jobManagerConnection.getJobId();

			List<AccumulatorSnapshot> accumulatorSnapshots = new ArrayList<>(16);
			Iterator<Task> allTasks = taskSlotTable.getTasks(jobId);

			while (allTasks.hasNext()) {
				Task task = allTasks.next();
				accumulatorSnapshots.add(task.getAccumulatorRegistry().getSnapshot());
			}
			return new AccumulatorReport(accumulatorSnapshots);
		}
	).orElseGet(() -> new AccumulatorReport(Collections.emptyList()));
}
 
Example #11
Source Project: flink   Author: apache   File: TaskExecutor.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<Acknowledge> sendOperatorEventToTask(
		ExecutionAttemptID executionAttemptID,
		OperatorID operatorId,
		SerializedValue<OperatorEvent> evt) {

	log.debug("Operator event for {} - {}", executionAttemptID, operatorId);

	final Task task = taskSlotTable.getTask(executionAttemptID);
	if (task == null) {
		return FutureUtils.completedExceptionally(new TaskNotRunningException(
			"Task " + executionAttemptID.toHexString() + " not running on TaskManager"));
	}

	try {
		task.deliverOperatorEvent(operatorId, evt);
		return CompletableFuture.completedFuture(Acknowledge.get());
	}
	catch (Throwable t) {
		ExceptionUtils.rethrowIfFatalError(t);
		return FutureUtils.completedExceptionally(t);
	}
}
 
Example #12
Source Project: flink   Author: flink-tpc-ds   File: StreamTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCancellationNotBlockedOnLock() throws Exception {
	syncLatch = new OneShotLatch();

	StreamConfig cfg = new StreamConfig(new Configuration());
	Task task = createTask(CancelLockingTask.class, cfg, new Configuration());

	// start the task and wait until it runs
	// execution state RUNNING is not enough, we need to wait until the stream task's run() method
	// is entered
	task.startTaskThread();
	syncLatch.await();

	// cancel the execution - this should lead to smooth shutdown
	task.cancelExecution();
	task.getExecutingThread().join();

	assertEquals(ExecutionState.CANCELED, task.getExecutionState());
}
 
Example #13
Source Project: Flink-CEPplus   Author: ljygz   File: TaskCheckpointingBehaviourTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testBlockingNonInterruptibleCheckpoint() throws Exception {

	StateBackend lockingStateBackend = new BackendForTestStream(LockingOutputStream::new);

	Task task =
		createTask(new TestOperator(), lockingStateBackend, mock(CheckpointResponder.class), true);

	// start the task and wait until it is in "restore"
	task.startTaskThread();
	IN_CHECKPOINT_LATCH.await();

	// cancel the task and wait. unless cancellation properly closes
	// the streams, this will never terminate
	task.cancelExecution();
	task.getExecutingThread().join();

	assertEquals(ExecutionState.CANCELED, task.getExecutionState());
	assertNull(task.getFailureCause());
}
 
Example #14
Source Project: flink   Author: flink-tpc-ds   File: TaskExecutor.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<StackTraceSampleResponse> requestStackTraceSample(
		final ExecutionAttemptID executionAttemptId,
		final int sampleId,
		final int numSamples,
		final Time delayBetweenSamples,
		final int maxStackTraceDepth,
		final Time timeout) {

	final Task task = taskSlotTable.getTask(executionAttemptId);
	if (task == null) {
		return FutureUtils.completedExceptionally(
			new IllegalStateException(String.format("Cannot sample task %s. " +
				"Task is not known to the task manager.", executionAttemptId)));
	}

	final CompletableFuture<List<StackTraceElement[]>> stackTracesFuture = stackTraceSampleService.requestStackTraceSample(
		TaskStackTraceSampleableTaskAdapter.fromTask(task),
		numSamples,
		delayBetweenSamples,
		maxStackTraceDepth);

	return stackTracesFuture.thenApply(stackTraces ->
		new StackTraceSampleResponse(sampleId, executionAttemptId, stackTraces));
}
 
Example #15
Source Project: flink   Author: flink-tpc-ds   File: TaskExecutor.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<Acknowledge> cancelTask(ExecutionAttemptID executionAttemptID, Time timeout) {
	final Task task = taskSlotTable.getTask(executionAttemptID);

	if (task != null) {
		try {
			task.cancelExecution();
			return CompletableFuture.completedFuture(Acknowledge.get());
		} catch (Throwable t) {
			return FutureUtils.completedExceptionally(
				new TaskException("Cannot cancel task for execution " + executionAttemptID + '.', t));
		}
	} else {
		final String message = "Cannot find task to stop for execution " + executionAttemptID + '.';

		log.debug(message);
		return FutureUtils.completedExceptionally(new TaskException(message));
	}
}
 
Example #16
Source Project: flink   Author: flink-tpc-ds   File: TaskExecutor.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public AccumulatorReport retrievePayload(ResourceID resourceID) {
	validateRunsInMainThread();
	JobManagerConnection jobManagerConnection = jobManagerConnections.get(resourceID);
	if (jobManagerConnection != null) {
		JobID jobId = jobManagerConnection.getJobID();

		List<AccumulatorSnapshot> accumulatorSnapshots = new ArrayList<>(16);
		Iterator<Task> allTasks = taskSlotTable.getTasks(jobId);

		while (allTasks.hasNext()) {
			Task task = allTasks.next();
			accumulatorSnapshots.add(task.getAccumulatorRegistry().getSnapshot());
		}
		return new AccumulatorReport(accumulatorSnapshots);
	} else {
		return new AccumulatorReport(Collections.emptyList());
	}
}
 
Example #17
Source Project: flink   Author: apache   File: TaskExecutor.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<Acknowledge> confirmCheckpoint(
		ExecutionAttemptID executionAttemptID,
		long checkpointId,
		long checkpointTimestamp) {
	log.debug("Confirm checkpoint {}@{} for {}.", checkpointId, checkpointTimestamp, executionAttemptID);

	final Task task = taskSlotTable.getTask(executionAttemptID);

	if (task != null) {
		task.notifyCheckpointComplete(checkpointId);

		return CompletableFuture.completedFuture(Acknowledge.get());
	} else {
		final String message = "TaskManager received a checkpoint confirmation for unknown task " + executionAttemptID + '.';

		log.debug(message);
		return FutureUtils.completedExceptionally(new CheckpointException(message, CheckpointFailureReason.UNKNOWN_TASK_CHECKPOINT_NOTIFICATION_FAILURE));
	}
}
 
Example #18
Source Project: flink   Author: flink-tpc-ds   File: TaskSlotTable.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Add the given task to the slot identified by the task's allocation id.
 *
 * @param task to add to the task slot with the respective allocation id
 * @throws SlotNotFoundException if there was no slot for the given allocation id
 * @throws SlotNotActiveException if there was no slot active for task's job and allocation id
 * @return True if the task could be added to the task slot; otherwise false
 */
public boolean addTask(Task task) throws SlotNotFoundException, SlotNotActiveException {
	Preconditions.checkNotNull(task);

	TaskSlot taskSlot = getTaskSlot(task.getAllocationId());

	if (taskSlot != null) {
		if (taskSlot.isActive(task.getJobID(), task.getAllocationId())) {
			if (taskSlot.add(task)) {
				taskSlotMappings.put(task.getExecutionId(), new TaskSlotMapping(task, taskSlot));

				return true;
			} else {
				return false;
			}
		} else {
			throw new SlotNotActiveException(task.getJobID(), task.getAllocationId());
		}
	} else {
		throw new SlotNotFoundException(task.getAllocationId());
	}
}
 
Example #19
Source Project: flink   Author: flink-tpc-ds   File: TaskCheckpointingBehaviourTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testBlockingNonInterruptibleCheckpoint() throws Exception {

	StateBackend lockingStateBackend = new BackendForTestStream(LockingOutputStream::new);

	Task task =
		createTask(new TestOperator(), lockingStateBackend, mock(CheckpointResponder.class));

	// start the task and wait until it is in "restore"
	task.startTaskThread();
	IN_CHECKPOINT_LATCH.await();

	// cancel the task and wait. unless cancellation properly closes
	// the streams, this will never terminate
	task.cancelExecution();
	task.getExecutingThread().join();

	assertEquals(ExecutionState.CANCELED, task.getExecutionState());
	assertNull(task.getFailureCause());
}
 
Example #20
Source Project: flink   Author: flink-tpc-ds   File: TaskSlotTable.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Task next() {
	while ((currentTasks == null || !currentTasks.hasNext())) {
		TaskSlot taskSlot;

		try {
			taskSlot = taskSlotIterator.next();
		} catch (NoSuchElementException e) {
			throw new NoSuchElementException("No more tasks.");
		}

		currentTasks = taskSlot.getTasks();
	}

	return currentTasks.next();
}
 
Example #21
Source Project: flink   Author: flink-tpc-ds   File: AbstractUdfStreamOperatorLifecycleTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testLifeCycleFull() throws Exception {
	ACTUAL_ORDER_TRACKING.clear();

	Configuration taskManagerConfig = new Configuration();
	StreamConfig cfg = new StreamConfig(new Configuration());
	MockSourceFunction srcFun = new MockSourceFunction();

	cfg.setStreamOperator(new LifecycleTrackingStreamSource<>(srcFun, true));
	cfg.setOperatorID(new OperatorID());
	cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	Task task = StreamTaskTest.createTask(SourceStreamTask.class, cfg, taskManagerConfig);

	task.startTaskThread();

	LifecycleTrackingStreamSource.runStarted.await();

	// wait for clean termination
	task.getExecutingThread().join();
	assertEquals(ExecutionState.FINISHED, task.getExecutionState());
	assertEquals(EXPECTED_CALL_ORDER_FULL, ACTUAL_ORDER_TRACKING);
}
 
Example #22
Source Project: flink   Author: flink-tpc-ds   File: AbstractUdfStreamOperatorLifecycleTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testLifeCycleCancel() throws Exception {
	ACTUAL_ORDER_TRACKING.clear();

	Configuration taskManagerConfig = new Configuration();
	StreamConfig cfg = new StreamConfig(new Configuration());
	MockSourceFunction srcFun = new MockSourceFunction();
	cfg.setStreamOperator(new LifecycleTrackingStreamSource<>(srcFun, false));
	cfg.setOperatorID(new OperatorID());
	cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	Task task = StreamTaskTest.createTask(SourceStreamTask.class, cfg, taskManagerConfig);

	task.startTaskThread();
	LifecycleTrackingStreamSource.runStarted.await();

	// this should cancel the task even though it is blocked on runFinished
	task.cancelExecution();

	// wait for clean termination
	task.getExecutingThread().join();
	assertEquals(ExecutionState.CANCELED, task.getExecutionState());
	assertEquals(EXPECTED_CALL_ORDER_CANCEL_RUNNING, ACTUAL_ORDER_TRACKING);
}
 
Example #23
Source Project: Flink-CEPplus   Author: ljygz   File: TaskIOMetricGroup.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Initialize Buffer Metrics for a task.
 */
public void initializeBufferMetrics(Task task) {
	final MetricGroup buffers = addGroup("buffers");
	buffers.gauge("inputQueueLength", new InputBuffersGauge(task));
	buffers.gauge("outputQueueLength", new OutputBuffersGauge(task));
	buffers.gauge("inPoolUsage", new InputBufferPoolUsageGauge(task));
	buffers.gauge("outPoolUsage", new OutputBufferPoolUsageGauge(task));
}
 
Example #24
Source Project: flink   Author: apache   File: StreamTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
public static Task createTask(
		Class<? extends AbstractInvokable> invokable,
		ShuffleEnvironment shuffleEnvironment,
		StreamConfig taskConfig,
		Configuration taskManagerConfig) throws Exception {

	return new TestTaskBuilder(shuffleEnvironment)
		.setTaskManagerConfig(taskManagerConfig)
		.setInvokable(invokable)
		.setTaskConfig(taskConfig.getConfiguration())
		.build();
}
 
Example #25
Source Project: flink   Author: apache   File: TaskExecutor.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public CompletableFuture<Acknowledge> updatePartitions(
		final ExecutionAttemptID executionAttemptID,
		Iterable<PartitionInfo> partitionInfos,
		Time timeout) {
	final Task task = taskSlotTable.getTask(executionAttemptID);

	if (task != null) {
		for (final PartitionInfo partitionInfo: partitionInfos) {
			// Run asynchronously because it might be blocking
			FutureUtils.assertNoException(
				CompletableFuture.runAsync(
					() -> {
						try {
							if (!shuffleEnvironment.updatePartitionInfo(executionAttemptID, partitionInfo)) {
								log.debug(
									"Discard update for input gate partition {} of result {} in task {}. " +
										"The partition is no longer available.",
									partitionInfo.getShuffleDescriptor().getResultPartitionID(),
									partitionInfo.getIntermediateDataSetID(),
									executionAttemptID);
							}
						} catch (IOException | InterruptedException e) {
							log.error(
								"Could not update input data location for task {}. Trying to fail task.",
								task.getTaskInfo().getTaskName(),
								e);
							task.failExternally(e);
						}
					},
					getRpcService().getExecutor()));
		}
		return CompletableFuture.completedFuture(Acknowledge.get());
	} else {
		log.debug("Discard update for input partitions of task {}. Task is no longer running.", executionAttemptID);
		return CompletableFuture.completedFuture(Acknowledge.get());
	}
}
 
Example #26
Source Project: flink   Author: flink-tpc-ds   File: StreamTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testStateBackendClosingOnFailure() throws Exception {
	Configuration taskManagerConfig = new Configuration();
	taskManagerConfig.setString(CheckpointingOptions.STATE_BACKEND, TestMemoryStateBackendFactory.class.getName());

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateKeySerializer(mock(TypeSerializer.class));
	cfg.setOperatorID(new OperatorID(4711L, 42L));
	TestStreamSource<Long, MockSourceFunction> streamSource = new TestStreamSource<>(new MockSourceFunction());
	cfg.setStreamOperator(streamSource);
	cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	Task task = createTask(StateBackendTestSource.class, cfg, taskManagerConfig);

	StateBackendTestSource.fail = true;
	task.startTaskThread();

	// wait for clean termination
	task.getExecutingThread().join();

	// ensure that the state backends and stream iterables are closed ...
	verify(TestStreamSource.operatorStateBackend).close();
	verify(TestStreamSource.keyedStateBackend).close();
	verify(TestStreamSource.rawOperatorStateInputs).close();
	verify(TestStreamSource.rawKeyedStateInputs).close();
	// ... and disposed
	verify(TestStreamSource.operatorStateBackend).dispose();
	verify(TestStreamSource.keyedStateBackend).dispose();

	assertEquals(ExecutionState.FAILED, task.getExecutionState());
}
 
Example #27
Source Project: flink   Author: apache   File: TaskExecutor.java    License: Apache License 2.0 5 votes vote down vote up
private void failTask(final ExecutionAttemptID executionAttemptID, final Throwable cause) {
	final Task task = taskSlotTable.getTask(executionAttemptID);

	if (task != null) {
		try {
			task.failExternally(cause);
		} catch (Throwable t) {
			log.error("Could not fail task {}.", executionAttemptID, t);
		}
	} else {
		log.debug("Cannot find task to fail for execution {}.", executionAttemptID);
	}
}
 
Example #28
Source Project: Flink-CEPplus   Author: ljygz   File: NetworkEnvironment.java    License: Apache License 2.0 5 votes vote down vote up
public void unregisterTask(Task task) {
	LOG.debug("Unregister task {} from network environment (state: {}).",
			task.getTaskInfo().getTaskNameWithSubtasks(), task.getExecutionState());

	final ExecutionAttemptID executionId = task.getExecutionId();

	synchronized (lock) {
		if (isShutdown) {
			// no need to do anything when we are not operational
			return;
		}

		if (task.isCanceledOrFailed()) {
			resultPartitionManager.releasePartitionsProducedBy(executionId, task.getFailureCause());
		}

		for (ResultPartition partition : task.getProducedPartitions()) {
			taskEventDispatcher.unregisterPartition(partition.getPartitionId());
			partition.destroyBufferPool();
		}

		final SingleInputGate[] inputGates = task.getAllInputGates();

		if (inputGates != null) {
			for (SingleInputGate gate : inputGates) {
				try {
					if (gate != null) {
						gate.releaseAllResources();
					}
				}
				catch (IOException e) {
					LOG.error("Error during release of reader resources: " + e.getMessage(), e);
				}
			}
		}
	}
}
 
Example #29
Source Project: flink   Author: apache   File: StreamTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * This test checks that cancel calls that are issued before the operator is
 * instantiated still lead to proper canceling.
 */
@Test
public void testEarlyCanceling() throws Exception {
	final StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setOperatorID(new OperatorID(4711L, 42L));
	cfg.setStreamOperator(new SlowlyDeserializingOperator());
	cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	final TaskManagerActions taskManagerActions = spy(new NoOpTaskManagerActions());
	try (NettyShuffleEnvironment shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build()) {
		final Task task =  new TestTaskBuilder(shuffleEnvironment)
			.setInvokable(SourceStreamTask.class)
			.setTaskConfig(cfg.getConfiguration())
			.setTaskManagerActions(taskManagerActions)
			.build();

		final TaskExecutionState state = new TaskExecutionState(
			task.getJobID(), task.getExecutionId(), ExecutionState.RUNNING);

		task.startTaskThread();

		verify(taskManagerActions, timeout(2000L)).updateTaskExecutionState(eq(state));

		// send a cancel. because the operator takes a long time to deserialize, this should
		// hit the task before the operator is deserialized
		task.cancelExecution();

		task.getExecutingThread().join();

		assertFalse("Task did not cancel", task.getExecutingThread().isAlive());
		assertEquals(ExecutionState.CANCELED, task.getExecutionState());
	}
}
 
Example #30
Source Project: Flink-CEPplus   Author: ljygz   File: StreamTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * This test checks that cancel calls that are issued before the operator is
 * instantiated still lead to proper canceling.
 */
@Test
public void testEarlyCanceling() throws Exception {
	final StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setOperatorID(new OperatorID(4711L, 42L));
	cfg.setStreamOperator(new SlowlyDeserializingOperator());
	cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	final TaskManagerActions taskManagerActions = spy(new NoOpTaskManagerActions());
	final Task task = createTask(SourceStreamTask.class, cfg, new Configuration(), taskManagerActions);

	final TaskExecutionState state = new TaskExecutionState(
		task.getJobID(), task.getExecutionId(), ExecutionState.RUNNING);

	task.startTaskThread();

	verify(taskManagerActions, timeout(2000L)).updateTaskExecutionState(eq(state));

	// send a cancel. because the operator takes a long time to deserialize, this should
	// hit the task before the operator is deserialized
	task.cancelExecution();

	task.getExecutingThread().join();

	assertFalse("Task did not cancel", task.getExecutingThread().isAlive());
	assertEquals(ExecutionState.CANCELED, task.getExecutionState());
}