org.apache.flink.runtime.checkpoint.CheckpointMetrics Java Examples

The following examples show how to use org.apache.flink.runtime.checkpoint.CheckpointMetrics. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FailoverRegionTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Let the checkpoint coordinator to receive all acknowledges from given executionVertexes so that to complete the expected checkpoint.
 */
private void acknowledgeAllCheckpoints(CheckpointCoordinator checkpointCoordinator, Iterator<ExecutionVertex> executionVertexes) throws IOException, CheckpointException {
	while (executionVertexes.hasNext()) {
		ExecutionVertex executionVertex = executionVertexes.next();
		for (int index = 0; index < executionVertex.getJobVertex().getParallelism(); index++) {
			JobVertexID jobVertexID = executionVertex.getJobvertexId();
			OperatorStateHandle opStateBackend = CheckpointCoordinatorTest.generatePartitionableStateHandle(jobVertexID, index, 2, 8, false);
			OperatorSubtaskState operatorSubtaskState = new OperatorSubtaskState(opStateBackend, null, null, null);
			TaskStateSnapshot taskOperatorSubtaskStates = new TaskStateSnapshot();
			taskOperatorSubtaskStates.putSubtaskStateByOperatorID(OperatorID.fromJobVertexID(jobVertexID), operatorSubtaskState);

			AcknowledgeCheckpoint acknowledgeCheckpoint = new AcknowledgeCheckpoint(
				executionVertex.getJobId(),
				executionVertex.getJobVertex().getTaskVertices()[index].getCurrentExecutionAttempt().getAttemptId(),
				checkpointId,
				new CheckpointMetrics(),
				taskOperatorSubtaskStates);

			checkpointCoordinator.receiveAcknowledgeMessage(acknowledgeCheckpoint, "Unknown location");
		}
	}
}
 
Example #2
Source File: StreamTask.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public boolean triggerCheckpoint(CheckpointMetaData checkpointMetaData, CheckpointOptions checkpointOptions) throws Exception {
	try {
		// No alignment if we inject a checkpoint
		CheckpointMetrics checkpointMetrics = new CheckpointMetrics()
				.setBytesBufferedInAlignment(0L)
				.setAlignmentDurationNanos(0L);

		return performCheckpoint(checkpointMetaData, checkpointOptions, checkpointMetrics);
	}
	catch (Exception e) {
		// propagate exceptions only if the task is still in "running" state
		if (isRunning) {
			throw new Exception("Could not perform checkpoint " + checkpointMetaData.getCheckpointId() +
				" for operator " + getName() + '.', e);
		} else {
			LOG.debug("Could not perform checkpoint {} for operator {} while the " +
				"invokable was not in state running.", checkpointMetaData.getCheckpointId(), getName(), e);
			return false;
		}
	}
}
 
Example #3
Source File: TaskStateManagerImpl.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void reportTaskStateSnapshots(
	@Nonnull CheckpointMetaData checkpointMetaData,
	@Nonnull CheckpointMetrics checkpointMetrics,
	@Nullable TaskStateSnapshot acknowledgedState,
	@Nullable TaskStateSnapshot localState) {

	long checkpointId = checkpointMetaData.getCheckpointId();

	localStateStore.storeLocalState(checkpointId, localState);

	checkpointResponder.acknowledgeCheckpoint(
		jobId,
		executionAttemptID,
		checkpointId,
		checkpointMetrics,
		acknowledgedState);
}
 
Example #4
Source File: TestCheckpointResponder.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public void acknowledgeCheckpoint(
	JobID jobID,
	ExecutionAttemptID executionAttemptID,
	long checkpointId,
	CheckpointMetrics checkpointMetrics,
	TaskStateSnapshot subtaskState) {

	AcknowledgeReport acknowledgeReport = new AcknowledgeReport(
		jobID,
		executionAttemptID,
		checkpointId,
		checkpointMetrics,
		subtaskState);

	acknowledgeReports.add(acknowledgeReport);

	if (acknowledgeLatch != null) {
		acknowledgeLatch.trigger();
	}
}
 
Example #5
Source File: StreamTask.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public boolean triggerCheckpoint(
		CheckpointMetaData checkpointMetaData,
		CheckpointOptions checkpointOptions,
		boolean advanceToEndOfEventTime) throws Exception {

	try {
		// No alignment if we inject a checkpoint
		CheckpointMetrics checkpointMetrics = new CheckpointMetrics()
				.setBytesBufferedInAlignment(0L)
				.setAlignmentDurationNanos(0L);

		return performCheckpoint(checkpointMetaData, checkpointOptions, checkpointMetrics, advanceToEndOfEventTime);
	}
	catch (Exception e) {
		// propagate exceptions only if the task is still in "running" state
		if (isRunning) {
			throw new Exception("Could not perform checkpoint " + checkpointMetaData.getCheckpointId() +
				" for operator " + getName() + '.', e);
		} else {
			LOG.debug("Could not perform checkpoint {} for operator {} while the " +
				"invokable was not in state running.", checkpointMetaData.getCheckpointId(), getName(), e);
			return false;
		}
	}
}
 
Example #6
Source File: TaskStateManagerImpl.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void reportTaskStateSnapshots(
	@Nonnull CheckpointMetaData checkpointMetaData,
	@Nonnull CheckpointMetrics checkpointMetrics,
	@Nullable TaskStateSnapshot acknowledgedState,
	@Nullable TaskStateSnapshot localState) {

	long checkpointId = checkpointMetaData.getCheckpointId();

	localStateStore.storeLocalState(checkpointId, localState);

	checkpointResponder.acknowledgeCheckpoint(
		jobId,
		executionAttemptID,
		checkpointId,
		checkpointMetrics,
		acknowledgedState);
}
 
Example #7
Source File: StreamTaskTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that checkpoints are declined if operators are (partially) closed.
 *
 * <p>See FLINK-16383.
 */
@Test
public void testCheckpointDeclinedOnClosedOperator() throws Throwable {
	ClosingOperator operator = new ClosingOperator();
	MultipleInputStreamTaskTestHarnessBuilder<Integer> builder =
		new MultipleInputStreamTaskTestHarnessBuilder<>(OneInputStreamTask::new, BasicTypeInfo.INT_TYPE_INFO)
				.addInput(BasicTypeInfo.INT_TYPE_INFO);
	StreamTaskMailboxTestHarness<Integer> harness = builder
		.setupOutputForSingletonOperatorChain(operator)
		.build();
	// keeps the mailbox from suspending
	harness.setAutoProcess(false);
	harness.processElement(new StreamRecord<>(1));

	harness.streamTask.operatorChain.closeOperators(harness.streamTask.getActionExecutor());
	assertEquals(true, operator.closed.get());

	harness.streamTask.triggerCheckpointOnBarrier(new CheckpointMetaData(1, 0), CheckpointOptions.forCheckpointWithDefaultLocation(), new CheckpointMetrics());
	assertEquals(1, harness.getCheckpointResponder().getDeclineReports().size());
}
 
Example #8
Source File: TaskStateManagerImpl.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public void reportTaskStateSnapshots(
	@Nonnull CheckpointMetaData checkpointMetaData,
	@Nonnull CheckpointMetrics checkpointMetrics,
	@Nullable TaskStateSnapshot acknowledgedState,
	@Nullable TaskStateSnapshot localState) {

	long checkpointId = checkpointMetaData.getCheckpointId();

	localStateStore.storeLocalState(checkpointId, localState);

	checkpointResponder.acknowledgeCheckpoint(
		jobId,
		executionAttemptID,
		checkpointId,
		checkpointMetrics,
		acknowledgedState);
}
 
Example #9
Source File: TestCheckpointResponder.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void acknowledgeCheckpoint(
	JobID jobID,
	ExecutionAttemptID executionAttemptID,
	long checkpointId,
	CheckpointMetrics checkpointMetrics,
	TaskStateSnapshot subtaskState) {

	AcknowledgeReport acknowledgeReport = new AcknowledgeReport(
		jobID,
		executionAttemptID,
		checkpointId,
		checkpointMetrics,
		subtaskState);

	acknowledgeReports.add(acknowledgeReport);

	if (acknowledgeLatch != null) {
		acknowledgeLatch.trigger();
	}
}
 
Example #10
Source File: JobMasterTriggerSavepointITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public Future<Boolean> triggerCheckpointAsync(final CheckpointMetaData checkpointMetaData, final CheckpointOptions checkpointOptions, final boolean advanceToEndOfEventTime) {
	final TaskStateSnapshot checkpointStateHandles = new TaskStateSnapshot();
	checkpointStateHandles.putSubtaskStateByOperatorID(
		OperatorID.fromJobVertexID(getEnvironment().getJobVertexId()),
		new OperatorSubtaskState());

	getEnvironment().acknowledgeCheckpoint(
		checkpointMetaData.getCheckpointId(),
		new CheckpointMetrics(),
		checkpointStateHandles);

	triggerCheckpointLatch.countDown();

	return CompletableFuture.completedFuture(true);
}
 
Example #11
Source File: SchedulerTestingUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
public static void acknowledgeCurrentCheckpoint(DefaultScheduler scheduler) {
	final CheckpointCoordinator checkpointCoordinator = getCheckpointCoordinator(scheduler);
	assertEquals("Coordinator has not ", 1, checkpointCoordinator.getNumberOfPendingCheckpoints());

	final PendingCheckpoint pc = checkpointCoordinator.getPendingCheckpoints().values().iterator().next();

	// because of races against the async thread in the coordinator, we need to wait here until the
	// coordinator state is acknowledged. This can be removed once the CheckpointCoordinator is
	// executes all actions in the Scheduler's main thread executor.
	while (pc.getNumberOfNonAcknowledgedOperatorCoordinators() > 0) {
		try {
			Thread.sleep(1);
		} catch (InterruptedException e) {
			Thread.currentThread().interrupt();
			fail("interrupted");
		}
	}

	getAllCurrentExecutionAttempts(scheduler).forEach(
		(attemptId) -> scheduler.acknowledgeCheckpoint(pc.getJobId(), attemptId, pc.getCheckpointId(), new CheckpointMetrics(), null));
}
 
Example #12
Source File: SubtaskCheckpointCoordinatorTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testNotifyCheckpointAbortedAfterAsyncPhase() throws Exception {
	TestTaskStateManager stateManager = new TestTaskStateManager();
	MockEnvironment mockEnvironment = MockEnvironment.builder().setTaskStateManager(stateManager).build();
	SubtaskCheckpointCoordinatorImpl subtaskCheckpointCoordinator = (SubtaskCheckpointCoordinatorImpl) new MockSubtaskCheckpointCoordinatorBuilder()
		.setEnvironment(mockEnvironment)
		.build();

	final OperatorChain<?, ?> operatorChain = getOperatorChain(mockEnvironment);

	long checkpointId = 42L;
	subtaskCheckpointCoordinator.checkpointState(
		new CheckpointMetaData(checkpointId, System.currentTimeMillis()),
		CheckpointOptions.forCheckpointWithDefaultLocation(),
		new CheckpointMetrics(),
		operatorChain,
		() -> true);
	subtaskCheckpointCoordinator.notifyCheckpointAborted(checkpointId, operatorChain, () -> true);
	assertEquals(0, subtaskCheckpointCoordinator.getAbortedCheckpointSize());
	assertEquals(checkpointId, stateManager.getNotifiedAbortedCheckpointId());
}
 
Example #13
Source File: AcknowledgeCheckpoint.java    From flink with Apache License 2.0 5 votes vote down vote up
public AcknowledgeCheckpoint(
		JobID job,
		ExecutionAttemptID taskExecutionId,
		long checkpointId,
		CheckpointMetrics checkpointMetrics,
		TaskStateSnapshot subtaskState) {

	super(job, taskExecutionId, checkpointId);

	this.subtaskState = subtaskState;
	this.checkpointMetrics = checkpointMetrics;
}
 
Example #14
Source File: LegacyScheduler.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void acknowledgeCheckpoint(final JobID jobID, final ExecutionAttemptID executionAttemptID, final long checkpointId, final CheckpointMetrics checkpointMetrics, final TaskStateSnapshot checkpointState) {
	mainThreadExecutor.assertRunningInMainThread();

	final CheckpointCoordinator checkpointCoordinator = executionGraph.getCheckpointCoordinator();
	final AcknowledgeCheckpoint ackMessage = new AcknowledgeCheckpoint(
		jobID,
		executionAttemptID,
		checkpointId,
		checkpointMetrics,
		checkpointState);

	final String taskManagerLocationInfo = retrieveTaskManagerLocation(executionAttemptID);

	if (checkpointCoordinator != null) {
		ioExecutor.execute(() -> {
			try {
				checkpointCoordinator.receiveAcknowledgeMessage(ackMessage, taskManagerLocationInfo);
			} catch (Throwable t) {
				log.warn("Error while processing checkpoint acknowledgement message", t);
			}
		});
	} else {
		String errorMessage = "Received AcknowledgeCheckpoint message for job {} with no CheckpointCoordinator";
		if (executionGraph.getState() == JobStatus.RUNNING) {
			log.error(errorMessage, jobGraph.getJobID());
		} else {
			log.debug(errorMessage, jobGraph.getJobID());
		}
	}
}
 
Example #15
Source File: StreamMockEnvironment.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void acknowledgeCheckpoint(long checkpointId, CheckpointMetrics checkpointMetrics, TaskStateSnapshot subtaskState) {
	taskStateManager.reportTaskStateSnapshots(
		new CheckpointMetaData(checkpointId, 0L),
		checkpointMetrics,
		subtaskState,
		null);
}
 
Example #16
Source File: RpcCheckpointResponder.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void acknowledgeCheckpoint(
		JobID jobID,
		ExecutionAttemptID executionAttemptID,
		long checkpointId,
		CheckpointMetrics checkpointMetrics,
		TaskStateSnapshot subtaskState) {

	checkpointCoordinatorGateway.acknowledgeCheckpoint(
		jobID,
		executionAttemptID,
		checkpointId,
		checkpointMetrics,
		subtaskState);
}
 
Example #17
Source File: JobMaster.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void acknowledgeCheckpoint(
		final JobID jobID,
		final ExecutionAttemptID executionAttemptID,
		final long checkpointId,
		final CheckpointMetrics checkpointMetrics,
		final TaskStateSnapshot checkpointState) {

	final CheckpointCoordinator checkpointCoordinator = executionGraph.getCheckpointCoordinator();
	final AcknowledgeCheckpoint ackMessage = new AcknowledgeCheckpoint(
		jobID,
		executionAttemptID,
		checkpointId,
		checkpointMetrics,
		checkpointState);

	if (checkpointCoordinator != null) {
		getRpcService().execute(() -> {
			try {
				checkpointCoordinator.receiveAcknowledgeMessage(ackMessage);
			} catch (Throwable t) {
				log.warn("Error while processing checkpoint acknowledgement message", t);
			}
		});
	} else {
		String errorMessage = "Received AcknowledgeCheckpoint message for job {} with no CheckpointCoordinator";
		if (executionGraph.getState() == JobStatus.RUNNING) {
			log.error(errorMessage, jobGraph.getJobID());
		} else {
			log.debug(errorMessage, jobGraph.getJobID());
		}
	}
}
 
Example #18
Source File: BarrierBufferTestBase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testSingleChannelAbortCheckpoint() throws Exception {
	BufferOrEvent[] sequence = {
		createBuffer(0, PAGE_SIZE),
		createBarrier(1, 0),
		createBuffer(0, PAGE_SIZE),
		createBarrier(2, 0),
		createCancellationBarrier(4, 0),
		createBarrier(5, 0),
		createBuffer(0, PAGE_SIZE),
		createCancellationBarrier(6, 0),
		createBuffer(0, PAGE_SIZE)
	};

	MockInputGate gate = new MockInputGate(PAGE_SIZE, 1, Arrays.asList(sequence));
	BarrierBuffer buffer = createBarrierHandler(gate);

	AbstractInvokable toNotify = mock(AbstractInvokable.class);
	buffer.registerCheckpointEventHandler(toNotify);

	check(sequence[0], buffer.getNextNonBlocked(), PAGE_SIZE);
	check(sequence[2], buffer.getNextNonBlocked(), PAGE_SIZE);
	verify(toNotify, times(1)).triggerCheckpointOnBarrier(argThat(new CheckpointMatcher(1L)), any(CheckpointOptions.class), any(CheckpointMetrics.class));
	assertEquals(0L, buffer.getAlignmentDurationNanos());

	check(sequence[6], buffer.getNextNonBlocked(), PAGE_SIZE);
	assertEquals(5L, buffer.getCurrentCheckpointId());
	verify(toNotify, times(1)).triggerCheckpointOnBarrier(argThat(new CheckpointMatcher(2L)), any(CheckpointOptions.class), any(CheckpointMetrics.class));
	verify(toNotify, times(1)).abortCheckpointOnBarrier(eq(4L), any(CheckpointDeclineOnCancellationBarrierException.class));
	verify(toNotify, times(1)).triggerCheckpointOnBarrier(argThat(new CheckpointMatcher(5L)), any(CheckpointOptions.class), any(CheckpointMetrics.class));
	assertEquals(0L, buffer.getAlignmentDurationNanos());

	check(sequence[8], buffer.getNextNonBlocked(), PAGE_SIZE);
	assertEquals(6L, buffer.getCurrentCheckpointId());
	verify(toNotify, times(1)).abortCheckpointOnBarrier(eq(6L), any(CheckpointDeclineOnCancellationBarrierException.class));
	assertEquals(0L, buffer.getAlignmentDurationNanos());

	buffer.cleanup();
}
 
Example #19
Source File: CheckpointBarrierHandler.java    From flink with Apache License 2.0 5 votes vote down vote up
protected void notifyCheckpoint(CheckpointBarrier checkpointBarrier, long bufferedBytes, long alignmentDurationNanos) throws Exception {
	if (toNotifyOnCheckpoint != null) {
		CheckpointMetaData checkpointMetaData =
			new CheckpointMetaData(checkpointBarrier.getId(), checkpointBarrier.getTimestamp());

		CheckpointMetrics checkpointMetrics = new CheckpointMetrics()
			.setBytesBufferedInAlignment(bufferedBytes)
			.setAlignmentDurationNanos(alignmentDurationNanos);

		toNotifyOnCheckpoint.triggerCheckpointOnBarrier(
			checkpointMetaData,
			checkpointBarrier.getCheckpointOptions(),
			checkpointMetrics);
	}
}
 
Example #20
Source File: SubtaskCheckpointCoordinatorTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testNotifyCheckpointAbortedBeforeAsyncPhase() throws Exception {
	TestTaskStateManager stateManager = new TestTaskStateManager();
	MockEnvironment mockEnvironment = MockEnvironment.builder().setTaskStateManager(stateManager).build();
	SubtaskCheckpointCoordinatorImpl subtaskCheckpointCoordinator = (SubtaskCheckpointCoordinatorImpl) new MockSubtaskCheckpointCoordinatorBuilder()
		.setEnvironment(mockEnvironment)
		.setUnalignedCheckpointEnabled(true)
		.build();

	CheckpointOperator checkpointOperator = new CheckpointOperator(new OperatorSnapshotFutures());

	final OperatorChain<String, AbstractStreamOperator<String>> operatorChain = operatorChain(checkpointOperator);

	long checkpointId = 42L;
	// notify checkpoint aborted before execution.
	subtaskCheckpointCoordinator.notifyCheckpointAborted(checkpointId, operatorChain, () -> true);
	assertEquals(1, subtaskCheckpointCoordinator.getAbortedCheckpointSize());

	subtaskCheckpointCoordinator.getChannelStateWriter().start(checkpointId, CheckpointOptions.forCheckpointWithDefaultLocation());
	subtaskCheckpointCoordinator.checkpointState(
		new CheckpointMetaData(checkpointId, System.currentTimeMillis()),
		CheckpointOptions.forCheckpointWithDefaultLocation(),
		new CheckpointMetrics(),
		operatorChain,
		() -> true);
	assertFalse(checkpointOperator.isCheckpointed());
	assertEquals(-1, stateManager.getReportedCheckpointId());
	assertEquals(0, subtaskCheckpointCoordinator.getAbortedCheckpointSize());
	assertEquals(0, subtaskCheckpointCoordinator.getAsyncCheckpointRunnableSize());
}
 
Example #21
Source File: TestCheckpointResponder.java    From flink with Apache License 2.0 5 votes vote down vote up
public AcknowledgeReport(
	JobID jobID,
	ExecutionAttemptID executionAttemptID,
	long checkpointId,
	CheckpointMetrics checkpointMetrics,
	TaskStateSnapshot subtaskState) {

	super(jobID, executionAttemptID, checkpointId);
	this.checkpointMetrics = checkpointMetrics;
	this.subtaskState = subtaskState;
}
 
Example #22
Source File: BarrierTrackerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void triggerCheckpointOnBarrier(CheckpointMetaData checkpointMetaData, CheckpointOptions checkpointOptions, CheckpointMetrics checkpointMetrics) throws Exception {
	assertTrue("More checkpoints than expected", i < checkpointIDs.length);

	final long expectedId = checkpointIDs[i++];
	if (expectedId >= 0) {
		assertEquals("wrong checkpoint id", expectedId, checkpointMetaData.getCheckpointId());
		assertTrue(checkpointMetaData.getTimestamp() > 0);
	} else {
		fail("got 'triggerCheckpointOnBarrier()' when expecting an 'abortCheckpointOnBarrier()'");
	}
}
 
Example #23
Source File: CheckpointBarrierUnalignerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
public void triggerCheckpointOnBarrier(
		CheckpointMetaData checkpointMetaData,
		CheckpointOptions checkpointOptions,
		CheckpointMetrics checkpointMetrics) {
	expectedCheckpointId = checkpointMetaData.getCheckpointId();
	totalNumCheckpoints++;
}
 
Example #24
Source File: CheckpointBarrierAlignerTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void triggerCheckpointOnBarrier(
		CheckpointMetaData checkpointMetaData,
		CheckpointOptions checkpointOptions,
		CheckpointMetrics checkpointMetrics) throws Exception {
	assertTrue("wrong checkpoint id", nextExpectedCheckpointId == -1L ||
		nextExpectedCheckpointId == checkpointMetaData.getCheckpointId());

	assertTrue(checkpointMetaData.getTimestamp() > 0);
	assertTrue(checkpointMetrics.getBytesBufferedInAlignment() >= 0);
	assertTrue(checkpointMetrics.getAlignmentDurationNanos() >= 0);

	nextExpectedCheckpointId++;
	lastReportedBytesBufferedInAlignment = checkpointMetrics.getBytesBufferedInAlignment();
}
 
Example #25
Source File: StreamMockEnvironment.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void acknowledgeCheckpoint(long checkpointId, CheckpointMetrics checkpointMetrics, TaskStateSnapshot subtaskState) {
	taskStateManager.reportTaskStateSnapshots(
		new CheckpointMetaData(checkpointId, 0L),
		checkpointMetrics,
		subtaskState,
		null);
}
 
Example #26
Source File: StreamTask.java    From flink with Apache License 2.0 5 votes vote down vote up
private boolean triggerCheckpoint(
		CheckpointMetaData checkpointMetaData,
		CheckpointOptions checkpointOptions,
		boolean advanceToEndOfEventTime) throws Exception {
	try {
		// No alignment if we inject a checkpoint
		CheckpointMetrics checkpointMetrics = new CheckpointMetrics().setAlignmentDurationNanos(0L);

		subtaskCheckpointCoordinator.initCheckpoint(checkpointMetaData.getCheckpointId(), checkpointOptions);

		boolean success = performCheckpoint(checkpointMetaData, checkpointOptions, checkpointMetrics, advanceToEndOfEventTime);
		if (!success) {
			declineCheckpoint(checkpointMetaData.getCheckpointId());
		}
		return success;
	} catch (Exception e) {
		// propagate exceptions only if the task is still in "running" state
		if (isRunning) {
			throw new Exception("Could not perform checkpoint " + checkpointMetaData.getCheckpointId() +
				" for operator " + getName() + '.', e);
		} else {
			LOG.debug("Could not perform checkpoint {} for operator {} while the " +
				"invokable was not in state running.", checkpointMetaData.getCheckpointId(), getName(), e);
			return false;
		}
	}
}
 
Example #27
Source File: AcknowledgeCheckpoint.java    From flink with Apache License 2.0 5 votes vote down vote up
public AcknowledgeCheckpoint(
		JobID job,
		ExecutionAttemptID taskExecutionId,
		long checkpointId,
		CheckpointMetrics checkpointMetrics,
		TaskStateSnapshot subtaskState) {

	super(job, taskExecutionId, checkpointId);

	this.subtaskState = subtaskState;
	this.checkpointMetrics = checkpointMetrics;
}
 
Example #28
Source File: BarrierBufferTestBase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void triggerCheckpointOnBarrier(
		CheckpointMetaData checkpointMetaData,
		CheckpointOptions checkpointOptions,
		CheckpointMetrics checkpointMetrics) throws Exception {
	assertTrue("wrong checkpoint id", nextExpectedCheckpointId == -1L ||
		nextExpectedCheckpointId == checkpointMetaData.getCheckpointId());

	assertTrue(checkpointMetaData.getTimestamp() > 0);
	assertTrue(checkpointMetrics.getBytesBufferedInAlignment() >= 0);
	assertTrue(checkpointMetrics.getAlignmentDurationNanos() >= 0);

	nextExpectedCheckpointId++;
	lastReportedBytesBufferedInAlignment = checkpointMetrics.getBytesBufferedInAlignment();
}
 
Example #29
Source File: StreamTask.java    From flink with Apache License 2.0 5 votes vote down vote up
AsyncCheckpointRunnable(
	StreamTask<?, ?> owner,
	Map<OperatorID, OperatorSnapshotFutures> operatorSnapshotsInProgress,
	CheckpointMetaData checkpointMetaData,
	CheckpointMetrics checkpointMetrics,
	long asyncStartNanos) {

	this.owner = Preconditions.checkNotNull(owner);
	this.operatorSnapshotsInProgress = Preconditions.checkNotNull(operatorSnapshotsInProgress);
	this.checkpointMetaData = Preconditions.checkNotNull(checkpointMetaData);
	this.checkpointMetrics = Preconditions.checkNotNull(checkpointMetrics);
	this.asyncStartNanos = asyncStartNanos;
}
 
Example #30
Source File: CheckpointMessagesTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testConfirmTaskCheckpointed() {
	try {
		AcknowledgeCheckpoint noState = new AcknowledgeCheckpoint(
				new JobID(), new ExecutionAttemptID(), 569345L);

		KeyGroupRange keyGroupRange = KeyGroupRange.of(42,42);

		TaskStateSnapshot checkpointStateHandles = new TaskStateSnapshot();
		checkpointStateHandles.putSubtaskStateByOperatorID(
			new OperatorID(),
			new OperatorSubtaskState(
				CheckpointCoordinatorTest.generatePartitionableStateHandle(new JobVertexID(), 0, 2, 8, false),
				null,
				CheckpointCoordinatorTest.generateKeyGroupState(keyGroupRange, Collections.singletonList(new MyHandle())),
				null
			)
		);

		AcknowledgeCheckpoint withState = new AcknowledgeCheckpoint(
				new JobID(),
				new ExecutionAttemptID(),
				87658976143L,
				new CheckpointMetrics(),
				checkpointStateHandles);

		testSerializabilityEqualsHashCode(noState);
		testSerializabilityEqualsHashCode(withState);
	} catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}