Java Code Examples for org.apache.flink.runtime.io.network.api.CheckpointBarrier#getId()

The following examples show how to use org.apache.flink.runtime.io.network.api.CheckpointBarrier#getId() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: BarrierBuffer.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private void notifyCheckpoint(CheckpointBarrier checkpointBarrier) throws Exception {
	if (toNotifyOnCheckpoint != null) {
		CheckpointMetaData checkpointMetaData =
				new CheckpointMetaData(checkpointBarrier.getId(), checkpointBarrier.getTimestamp());

		long bytesBuffered = currentBuffered != null ? currentBuffered.size() : 0L;

		CheckpointMetrics checkpointMetrics = new CheckpointMetrics()
				.setBytesBufferedInAlignment(bytesBuffered)
				.setAlignmentDurationNanos(latestAlignmentDurationNanos);

		toNotifyOnCheckpoint.triggerCheckpointOnBarrier(
			checkpointMetaData,
			checkpointBarrier.getCheckpointOptions(),
			checkpointMetrics);
	}
}
 
Example 2
Source File: CheckpointBarrierUnaligner.java    From flink with Apache License 2.0 6 votes vote down vote up
private synchronized void handleNewCheckpoint(CheckpointBarrier barrier) throws IOException {
	long barrierId = barrier.getId();
	if (!allBarriersReceivedFuture.isDone()) {
		CheckpointException exception = new CheckpointException("Barrier id: " + barrierId, CHECKPOINT_DECLINED_SUBSUMED);
		if (isCheckpointPending()) {
			// we did not complete the current checkpoint, another started before
			LOG.warn("{}: Received checkpoint barrier for checkpoint {} before completing current checkpoint {}. " +
					"Skipping current checkpoint.",
				handler.taskName,
				barrierId,
				currentReceivedCheckpointId);

			// let the task know we are not completing this
			final long currentCheckpointId = currentReceivedCheckpointId;
			handler.executeInTaskThread(() -> handler.notifyAbort(currentCheckpointId, exception), "notifyAbort");
		}
		allBarriersReceivedFuture.completeExceptionally(exception);
	}

	currentReceivedCheckpointId = barrierId;
	storeNewBuffers.entrySet().forEach(storeNewBuffer -> storeNewBuffer.setValue(true));
	numBarriersReceived = 0;
	allBarriersReceivedFuture = new CompletableFuture<>();
	checkpointCoordinator.initCheckpoint(barrierId, barrier.getCheckpointOptions());
}
 
Example 3
Source File: CheckpointBarrierUnaligner.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public synchronized void notifyBarrierReceived(CheckpointBarrier barrier, InputChannelInfo channelInfo) throws IOException {
	long barrierId = barrier.getId();

	if (currentReceivedCheckpointId < barrierId) {
		handleNewCheckpoint(barrier);
		handler.executeInTaskThread(() -> handler.notifyCheckpoint(barrier), "notifyCheckpoint");
	}

	if (barrierId == currentReceivedCheckpointId && storeNewBuffers.get(channelInfo)) {
		if (LOG.isDebugEnabled()) {
			LOG.debug("{}: Received barrier from channel {} @ {}.", handler.taskName, channelInfo, barrierId);
		}

		storeNewBuffers.put(channelInfo, false);

		if (++numBarriersReceived == numOpenChannels) {
			allBarriersReceivedFuture.complete(null);
		}
	}
}
 
Example 4
Source File: RemoteInputChannel.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void spillInflightBuffers(long checkpointId, ChannelStateWriter channelStateWriter) throws IOException {
	synchronized (receivedBuffers) {
		checkState(checkpointId > lastRequestedCheckpointId, "Need to request the next checkpointId");

		final List<Buffer> inflightBuffers = new ArrayList<>(receivedBuffers.size());
		for (Buffer buffer : receivedBuffers) {
			CheckpointBarrier checkpointBarrier = parseCheckpointBarrierOrNull(buffer);
			if (checkpointBarrier != null && checkpointBarrier.getId() >= checkpointId) {
				break;
			}
			if (buffer.isBuffer()) {
				inflightBuffers.add(buffer.retainBuffer());
			}
		}

		lastRequestedCheckpointId = checkpointId;

		channelStateWriter.addInputData(
			checkpointId,
			channelInfo,
			ChannelStateWriter.SEQUENCE_NUMBER_UNKNOWN,
			CloseableIterator.fromList(inflightBuffers, Buffer::recycleBuffer));
	}
}
 
Example 5
Source File: AlternatingCheckpointBarrierHandler.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void processBarrier(CheckpointBarrier receivedBarrier, InputChannelInfo channelInfo) throws Exception {
	if (receivedBarrier.getId() < lastSeenBarrierId) {
		return;
	}

	lastSeenBarrierId = receivedBarrier.getId();
	CheckpointBarrierHandler previousHandler = activeHandler;
	activeHandler = receivedBarrier.isCheckpoint() ? unalignedHandler : alignedHandler;
	if (previousHandler != activeHandler) {
		previousHandler.abortPendingCheckpoint(
			lastSeenBarrierId,
			new CheckpointException(format("checkpoint subsumed by %d", lastSeenBarrierId), CHECKPOINT_DECLINED_SUBSUMED));
	}

	activeHandler.processBarrier(receivedBarrier, channelInfo);
}
 
Example 6
Source File: CheckpointBarrierUnaligner.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * We still need to trigger checkpoint via {@link ThreadSafeUnaligner#notifyBarrierReceived(CheckpointBarrier, InputChannelInfo)}
 * while reading the first barrier from one channel, because this might happen
 * earlier than the previous async trigger via mailbox by netty thread.
 *
 * <p>Note this is also suitable for the trigger case of local input channel.
 */
@Override
public void processBarrier(CheckpointBarrier receivedBarrier, InputChannelInfo channelInfo) throws Exception {
	long barrierId = receivedBarrier.getId();
	if (currentConsumedCheckpointId > barrierId || (currentConsumedCheckpointId == barrierId && !isCheckpointPending())) {
		// ignore old and cancelled barriers
		return;
	}
	if (currentConsumedCheckpointId < barrierId) {
		currentConsumedCheckpointId = barrierId;
		numBarrierConsumed = 0;
		hasInflightBuffers.entrySet().forEach(hasInflightBuffer -> hasInflightBuffer.setValue(true));
	}
	if (currentConsumedCheckpointId == barrierId) {
		hasInflightBuffers.put(channelInfo, false);
		numBarrierConsumed++;
	}
	threadSafeUnaligner.notifyBarrierReceived(receivedBarrier, channelInfo);
}
 
Example 7
Source File: CheckpointBarrierHandler.java    From flink with Apache License 2.0 5 votes vote down vote up
protected void notifyCheckpoint(CheckpointBarrier checkpointBarrier, long bufferedBytes, long alignmentDurationNanos) throws Exception {
	if (toNotifyOnCheckpoint != null) {
		CheckpointMetaData checkpointMetaData =
			new CheckpointMetaData(checkpointBarrier.getId(), checkpointBarrier.getTimestamp());

		CheckpointMetrics checkpointMetrics = new CheckpointMetrics()
			.setBytesBufferedInAlignment(bufferedBytes)
			.setAlignmentDurationNanos(alignmentDurationNanos);

		toNotifyOnCheckpoint.triggerCheckpointOnBarrier(
			checkpointMetaData,
			checkpointBarrier.getCheckpointOptions(),
			checkpointMetrics);
	}
}
 
Example 8
Source File: CheckpointBarrierHandler.java    From flink with Apache License 2.0 5 votes vote down vote up
protected void notifyCheckpoint(CheckpointBarrier checkpointBarrier, long alignmentDurationNanos) throws IOException {
	CheckpointMetaData checkpointMetaData =
		new CheckpointMetaData(checkpointBarrier.getId(), checkpointBarrier.getTimestamp());

	CheckpointMetrics checkpointMetrics = new CheckpointMetrics()
		.setAlignmentDurationNanos(alignmentDurationNanos)
		.setCheckpointStartDelayNanos(latestCheckpointStartDelayNanos);

	toNotifyOnCheckpoint.triggerCheckpointOnBarrier(
		checkpointMetaData,
		checkpointBarrier.getCheckpointOptions(),
		checkpointMetrics);
}
 
Example 9
Source File: CheckpointBarrierTracker.java    From flink with Apache License 2.0 4 votes vote down vote up
public void processBarrier(CheckpointBarrier receivedBarrier, InputChannelInfo channelInfo) throws Exception {
	final long barrierId = receivedBarrier.getId();

	// fast path for single channel trackers
	if (totalNumberOfInputChannels == 1) {
		notifyCheckpoint(receivedBarrier, 0);
		return;
	}

	// general path for multiple input channels
	if (LOG.isDebugEnabled()) {
		LOG.debug("Received barrier for checkpoint {} from channel {}", barrierId, channelInfo);
	}

	// find the checkpoint barrier in the queue of pending barriers
	CheckpointBarrierCount barrierCount = null;
	int pos = 0;

	for (CheckpointBarrierCount next : pendingCheckpoints) {
		if (next.checkpointId == barrierId) {
			barrierCount = next;
			break;
		}
		pos++;
	}

	if (barrierCount != null) {
		// add one to the count to that barrier and check for completion
		int numBarriersNew = barrierCount.incrementBarrierCount();
		if (numBarriersNew == totalNumberOfInputChannels) {
			// checkpoint can be triggered (or is aborted and all barriers have been seen)
			// first, remove this checkpoint and all all prior pending
			// checkpoints (which are now subsumed)
			for (int i = 0; i <= pos; i++) {
				pendingCheckpoints.pollFirst();
			}

			// notify the listener
			if (!barrierCount.isAborted()) {
				if (LOG.isDebugEnabled()) {
					LOG.debug("Received all barriers for checkpoint {}", barrierId);
				}

				notifyCheckpoint(receivedBarrier, 0);
			}
		}
	}
	else {
		// first barrier for that checkpoint ID
		// add it only if it is newer than the latest checkpoint.
		// if it is not newer than the latest checkpoint ID, then there cannot be a
		// successful checkpoint for that ID anyways
		if (barrierId > latestPendingCheckpointID) {
			markCheckpointStart(receivedBarrier.getTimestamp());
			latestPendingCheckpointID = barrierId;
			pendingCheckpoints.addLast(new CheckpointBarrierCount(barrierId));

			// make sure we do not track too many checkpoints
			if (pendingCheckpoints.size() > MAX_CHECKPOINTS_TO_TRACK) {
				pendingCheckpoints.pollFirst();
			}
		}
	}
}
 
Example 10
Source File: BarrierBuffer.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
private void processBarrier(CheckpointBarrier receivedBarrier, int channelIndex) throws Exception {
	final long barrierId = receivedBarrier.getId();

	// fast path for single channel cases
	if (totalNumberOfInputChannels == 1) {
		if (barrierId > currentCheckpointId) {
			// new checkpoint
			currentCheckpointId = barrierId;
			notifyCheckpoint(receivedBarrier);
		}
		return;
	}

	// -- general code path for multiple input channels --

	if (numBarriersReceived > 0) {
		// this is only true if some alignment is already progress and was not canceled

		if (barrierId == currentCheckpointId) {
			// regular case
			onBarrier(channelIndex);
		}
		else if (barrierId > currentCheckpointId) {
			// we did not complete the current checkpoint, another started before
			LOG.warn("{}: Received checkpoint barrier for checkpoint {} before completing current checkpoint {}. " +
					"Skipping current checkpoint.",
				inputGate.getOwningTaskName(),
				barrierId,
				currentCheckpointId);

			// let the task know we are not completing this
			notifyAbort(currentCheckpointId, new CheckpointDeclineSubsumedException(barrierId));

			// abort the current checkpoint
			releaseBlocksAndResetBarriers();

			// begin a the new checkpoint
			beginNewAlignment(barrierId, channelIndex);
		}
		else {
			// ignore trailing barrier from an earlier checkpoint (obsolete now)
			return;
		}
	}
	else if (barrierId > currentCheckpointId) {
		// first barrier of a new checkpoint
		beginNewAlignment(barrierId, channelIndex);
	}
	else {
		// either the current checkpoint was canceled (numBarriers == 0) or
		// this barrier is from an old subsumed checkpoint
		return;
	}

	// check if we have all barriers - since canceled checkpoints always have zero barriers
	// this can only happen on a non canceled checkpoint
	if (numBarriersReceived + numClosedChannels == totalNumberOfInputChannels) {
		// actually trigger checkpoint
		if (LOG.isDebugEnabled()) {
			LOG.debug("{}: Received all barriers, triggering checkpoint {} at {}.",
				inputGate.getOwningTaskName(),
				receivedBarrier.getId(),
				receivedBarrier.getTimestamp());
		}

		releaseBlocksAndResetBarriers();
		notifyCheckpoint(receivedBarrier);
	}
}
 
Example 11
Source File: CheckpointBarrierAligner.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public void processBarrier(CheckpointBarrier receivedBarrier, InputChannelInfo channelInfo) throws Exception {
	final long barrierId = receivedBarrier.getId();

	// fast path for single channel cases
	if (totalNumberOfInputChannels == 1) {
		resumeConsumption(channelInfo);
		if (barrierId > currentCheckpointId) {
			// new checkpoint
			currentCheckpointId = barrierId;
			notifyCheckpoint(receivedBarrier, latestAlignmentDurationNanos);
		}
		return;
	}

	// -- general code path for multiple input channels --

	if (isCheckpointPending()) {
		// this is only true if some alignment is already progress and was not canceled

		if (barrierId == currentCheckpointId) {
			// regular case
			onBarrier(channelInfo);
		}
		else if (barrierId > currentCheckpointId) {
			// we did not complete the current checkpoint, another started before
			LOG.warn("{}: Received checkpoint barrier for checkpoint {} before completing current checkpoint {}. " +
					"Skipping current checkpoint.",
				taskName,
				barrierId,
				currentCheckpointId);

			// let the task know we are not completing this
			notifyAbort(currentCheckpointId,
				new CheckpointException(
					"Barrier id: " + barrierId,
					CheckpointFailureReason.CHECKPOINT_DECLINED_SUBSUMED));

			// abort the current checkpoint
			releaseBlocksAndResetBarriers();

			// begin a new checkpoint
			beginNewAlignment(barrierId, channelInfo, receivedBarrier.getTimestamp());
		}
		else {
			// ignore trailing barrier from an earlier checkpoint (obsolete now)
			resumeConsumption(channelInfo);
		}
	}
	else if (barrierId > currentCheckpointId) {
		// first barrier of a new checkpoint
		beginNewAlignment(barrierId, channelInfo, receivedBarrier.getTimestamp());
	}
	else {
		// either the current checkpoint was canceled (numBarriers == 0) or
		// this barrier is from an old subsumed checkpoint
		resumeConsumption(channelInfo);
	}

	// check if we have all barriers - since canceled checkpoints always have zero barriers
	// this can only happen on a non canceled checkpoint
	if (numBarriersReceived + numClosedChannels == totalNumberOfInputChannels) {
		// actually trigger checkpoint
		if (LOG.isDebugEnabled()) {
			LOG.debug("{}: Received all barriers, triggering checkpoint {} at {}.",
				taskName,
				receivedBarrier.getId(),
				receivedBarrier.getTimestamp());
		}

		releaseBlocksAndResetBarriers();
		notifyCheckpoint(receivedBarrier, latestAlignmentDurationNanos);
	}
}
 
Example 12
Source File: CheckpointBarrierUnaligner.java    From flink with Apache License 2.0 4 votes vote down vote up
private void notifyCheckpoint(CheckpointBarrier barrier) throws IOException {
	// ignore the previous triggered checkpoint by netty thread if it was already canceled or aborted before.
	if (barrier.getId() >= threadSafeUnaligner.getCurrentCheckpointId()) {
		super.notifyCheckpoint(barrier, 0);
	}
}
 
Example 13
Source File: RemoteInputChannel.java    From flink with Apache License 2.0 4 votes vote down vote up
public void onBuffer(Buffer buffer, int sequenceNumber, int backlog) throws IOException {
	boolean recycleBuffer = true;

	try {
		if (expectedSequenceNumber != sequenceNumber) {
			onError(new BufferReorderingException(expectedSequenceNumber, sequenceNumber));
			return;
		}

		final boolean wasEmpty;
		final CheckpointBarrier notifyReceivedBarrier;
		final Buffer notifyReceivedBuffer;
		final BufferReceivedListener listener = inputGate.getBufferReceivedListener();
		synchronized (receivedBuffers) {
			// Similar to notifyBufferAvailable(), make sure that we never add a buffer
			// after releaseAllResources() released all buffers from receivedBuffers
			// (see above for details).
			if (isReleased.get()) {
				return;
			}

			wasEmpty = receivedBuffers.isEmpty();
			receivedBuffers.add(buffer);

			if (listener != null && buffer.isBuffer() && receivedCheckpointId < lastRequestedCheckpointId) {
				notifyReceivedBuffer = buffer.retainBuffer();
			} else {
				notifyReceivedBuffer = null;
			}
			notifyReceivedBarrier = listener != null ? parseCheckpointBarrierOrNull(buffer) : null;
		}
		recycleBuffer = false;

		++expectedSequenceNumber;

		if (wasEmpty) {
			notifyChannelNonEmpty();
		}

		if (backlog >= 0) {
			onSenderBacklog(backlog);
		}

		if (notifyReceivedBarrier != null) {
			receivedCheckpointId = notifyReceivedBarrier.getId();
			if (notifyReceivedBarrier.isCheckpoint()) {
				listener.notifyBarrierReceived(notifyReceivedBarrier, channelInfo);
			}
		} else if (notifyReceivedBuffer != null) {
			listener.notifyBufferReceived(notifyReceivedBuffer, channelInfo);
		}
	} finally {
		if (recycleBuffer) {
			buffer.recycleBuffer();
		}
	}
}
 
Example 14
Source File: LocalInputChannel.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
Optional<BufferAndAvailability> getNextBuffer() throws IOException, InterruptedException {
	checkError();

	ResultSubpartitionView subpartitionView = this.subpartitionView;
	if (subpartitionView == null) {
		// There is a possible race condition between writing a EndOfPartitionEvent (1) and flushing (3) the Local
		// channel on the sender side, and reading EndOfPartitionEvent (2) and processing flush notification (4). When
		// they happen in that order (1 - 2 - 3 - 4), flush notification can re-enqueue LocalInputChannel after (or
		// during) it was released during reading the EndOfPartitionEvent (2).
		if (isReleased) {
			return Optional.empty();
		}

		// this can happen if the request for the partition was triggered asynchronously
		// by the time trigger
		// would be good to avoid that, by guaranteeing that the requestPartition() and
		// getNextBuffer() always come from the same thread
		// we could do that by letting the timer insert a special "requesting channel" into the input gate's queue
		subpartitionView = checkAndWaitForSubpartitionView();
	}

	BufferAndBacklog next = subpartitionView.getNextBuffer();

	if (next == null) {
		if (subpartitionView.isReleased()) {
			throw new CancelTaskException("Consumed partition " + subpartitionView + " has been released.");
		} else {
			return Optional.empty();
		}
	}

	Buffer buffer = next.buffer();
	CheckpointBarrier notifyReceivedBarrier = parseCheckpointBarrierOrNull(buffer);
	if (notifyReceivedBarrier != null) {
		receivedCheckpointId = notifyReceivedBarrier.getId();
	} else if (receivedCheckpointId < lastRequestedCheckpointId && buffer.isBuffer()) {
		inputGate.getBufferReceivedListener().notifyBufferReceived(buffer.retainBuffer(), channelInfo);
	}

	numBytesIn.inc(buffer.getSize());
	numBuffersIn.inc();
	return Optional.of(new BufferAndAvailability(buffer, next.isDataAvailable(), next.buffersInBacklog()));
}
 
Example 15
Source File: CheckpointBarrierTracker.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public boolean processBarrier(CheckpointBarrier receivedBarrier, int channelIndex, long bufferedBytes) throws Exception {
	final long barrierId = receivedBarrier.getId();

	// fast path for single channel trackers
	if (totalNumberOfInputChannels == 1) {
		notifyCheckpoint(receivedBarrier, 0, 0);
		return false;
	}

	// general path for multiple input channels
	if (LOG.isDebugEnabled()) {
		LOG.debug("Received barrier for checkpoint {} from channel {}", barrierId, channelIndex);
	}

	// find the checkpoint barrier in the queue of pending barriers
	CheckpointBarrierCount barrierCount = null;
	int pos = 0;

	for (CheckpointBarrierCount next : pendingCheckpoints) {
		if (next.checkpointId == barrierId) {
			barrierCount = next;
			break;
		}
		pos++;
	}

	if (barrierCount != null) {
		// add one to the count to that barrier and check for completion
		int numBarriersNew = barrierCount.incrementBarrierCount();
		if (numBarriersNew == totalNumberOfInputChannels) {
			// checkpoint can be triggered (or is aborted and all barriers have been seen)
			// first, remove this checkpoint and all all prior pending
			// checkpoints (which are now subsumed)
			for (int i = 0; i <= pos; i++) {
				pendingCheckpoints.pollFirst();
			}

			// notify the listener
			if (!barrierCount.isAborted()) {
				if (LOG.isDebugEnabled()) {
					LOG.debug("Received all barriers for checkpoint {}", barrierId);
				}

				notifyCheckpoint(receivedBarrier, 0, 0);
			}
		}
	}
	else {
		// first barrier for that checkpoint ID
		// add it only if it is newer than the latest checkpoint.
		// if it is not newer than the latest checkpoint ID, then there cannot be a
		// successful checkpoint for that ID anyways
		if (barrierId > latestPendingCheckpointID) {
			latestPendingCheckpointID = barrierId;
			pendingCheckpoints.addLast(new CheckpointBarrierCount(barrierId));

			// make sure we do not track too many checkpoints
			if (pendingCheckpoints.size() > MAX_CHECKPOINTS_TO_TRACK) {
				pendingCheckpoints.pollFirst();
			}
		}
	}
	return false;
}
 
Example 16
Source File: CheckpointBarrierAligner.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public boolean processBarrier(CheckpointBarrier receivedBarrier, int channelIndex, long bufferedBytes) throws Exception {
	final long barrierId = receivedBarrier.getId();

	// fast path for single channel cases
	if (totalNumberOfInputChannels == 1) {
		if (barrierId > currentCheckpointId) {
			// new checkpoint
			currentCheckpointId = barrierId;
			notifyCheckpoint(receivedBarrier, bufferedBytes, latestAlignmentDurationNanos);
		}
		return false;
	}

	boolean checkpointAborted = false;

	// -- general code path for multiple input channels --

	if (numBarriersReceived > 0) {
		// this is only true if some alignment is already progress and was not canceled

		if (barrierId == currentCheckpointId) {
			// regular case
			onBarrier(channelIndex);
		}
		else if (barrierId > currentCheckpointId) {
			// we did not complete the current checkpoint, another started before
			LOG.warn("{}: Received checkpoint barrier for checkpoint {} before completing current checkpoint {}. " +
					"Skipping current checkpoint.",
				taskName,
				barrierId,
				currentCheckpointId);

			// let the task know we are not completing this
			notifyAbort(currentCheckpointId,
				new CheckpointException(
					"Barrier id: " + barrierId,
					CheckpointFailureReason.CHECKPOINT_DECLINED_SUBSUMED));

			// abort the current checkpoint
			releaseBlocksAndResetBarriers();
			checkpointAborted = true;

			// begin a the new checkpoint
			beginNewAlignment(barrierId, channelIndex);
		}
		else {
			// ignore trailing barrier from an earlier checkpoint (obsolete now)
			return false;
		}
	}
	else if (barrierId > currentCheckpointId) {
		// first barrier of a new checkpoint
		beginNewAlignment(barrierId, channelIndex);
	}
	else {
		// either the current checkpoint was canceled (numBarriers == 0) or
		// this barrier is from an old subsumed checkpoint
		return false;
	}

	// check if we have all barriers - since canceled checkpoints always have zero barriers
	// this can only happen on a non canceled checkpoint
	if (numBarriersReceived + numClosedChannels == totalNumberOfInputChannels) {
		// actually trigger checkpoint
		if (LOG.isDebugEnabled()) {
			LOG.debug("{}: Received all barriers, triggering checkpoint {} at {}.",
				taskName,
				receivedBarrier.getId(),
				receivedBarrier.getTimestamp());
		}

		releaseBlocksAndResetBarriers();
		notifyCheckpoint(receivedBarrier, bufferedBytes, latestAlignmentDurationNanos);
		return true;
	}
	return checkpointAborted;
}
 
Example 17
Source File: BarrierTracker.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
private void processBarrier(CheckpointBarrier receivedBarrier, int channelIndex) throws Exception {
	final long barrierId = receivedBarrier.getId();

	// fast path for single channel trackers
	if (totalNumberOfInputChannels == 1) {
		notifyCheckpoint(barrierId, receivedBarrier.getTimestamp(), receivedBarrier.getCheckpointOptions());
		return;
	}

	// general path for multiple input channels
	if (LOG.isDebugEnabled()) {
		LOG.debug("Received barrier for checkpoint {} from channel {}", barrierId, channelIndex);
	}

	// find the checkpoint barrier in the queue of pending barriers
	CheckpointBarrierCount cbc = null;
	int pos = 0;

	for (CheckpointBarrierCount next : pendingCheckpoints) {
		if (next.checkpointId == barrierId) {
			cbc = next;
			break;
		}
		pos++;
	}

	if (cbc != null) {
		// add one to the count to that barrier and check for completion
		int numBarriersNew = cbc.incrementBarrierCount();
		if (numBarriersNew == totalNumberOfInputChannels) {
			// checkpoint can be triggered (or is aborted and all barriers have been seen)
			// first, remove this checkpoint and all all prior pending
			// checkpoints (which are now subsumed)
			for (int i = 0; i <= pos; i++) {
				pendingCheckpoints.pollFirst();
			}

			// notify the listener
			if (!cbc.isAborted()) {
				if (LOG.isDebugEnabled()) {
					LOG.debug("Received all barriers for checkpoint {}", barrierId);
				}

				notifyCheckpoint(receivedBarrier.getId(), receivedBarrier.getTimestamp(), receivedBarrier.getCheckpointOptions());
			}
		}
	}
	else {
		// first barrier for that checkpoint ID
		// add it only if it is newer than the latest checkpoint.
		// if it is not newer than the latest checkpoint ID, then there cannot be a
		// successful checkpoint for that ID anyways
		if (barrierId > latestPendingCheckpointID) {
			latestPendingCheckpointID = barrierId;
			pendingCheckpoints.addLast(new CheckpointBarrierCount(barrierId));

			// make sure we do not track too many checkpoints
			if (pendingCheckpoints.size() > MAX_CHECKPOINTS_TO_TRACK) {
				pendingCheckpoints.pollFirst();
			}
		}
	}
}