Java Code Examples for org.apache.flink.streaming.connectors.kafka.config.OffsetCommitMode#ON_CHECKPOINTS

The following examples show how to use org.apache.flink.streaming.connectors.kafka.config.OffsetCommitMode#ON_CHECKPOINTS . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: FlinkKafkaConsumerBase.java From Flink-CEPplus with Apache License 2.0

4 votes

@Override
public final void snapshotState(FunctionSnapshotContext context) throws Exception {
	if (!running) {
		LOG.debug("snapshotState() called on closed source");
	} else {
		unionOffsetStates.clear();

		final AbstractFetcher<?, ?> fetcher = this.kafkaFetcher;
		if (fetcher == null) {
			// the fetcher has not yet been initialized, which means we need to return the
			// originally restored offsets or the assigned partitions
			for (Map.Entry<KafkaTopicPartition, Long> subscribedPartition : subscribedPartitionsToStartOffsets.entrySet()) {
				unionOffsetStates.add(Tuple2.of(subscribedPartition.getKey(), subscribedPartition.getValue()));
			}

			if (offsetCommitMode == OffsetCommitMode.ON_CHECKPOINTS) {
				// the map cannot be asynchronously updated, because only one checkpoint call can happen
				// on this function at a time: either snapshotState() or notifyCheckpointComplete()
				pendingOffsetsToCommit.put(context.getCheckpointId(), restoredState);
			}
		} else {
			HashMap<KafkaTopicPartition, Long> currentOffsets = fetcher.snapshotCurrentState();

			if (offsetCommitMode == OffsetCommitMode.ON_CHECKPOINTS) {
				// the map cannot be asynchronously updated, because only one checkpoint call can happen
				// on this function at a time: either snapshotState() or notifyCheckpointComplete()
				pendingOffsetsToCommit.put(context.getCheckpointId(), currentOffsets);
			}

			for (Map.Entry<KafkaTopicPartition, Long> kafkaTopicPartitionLongEntry : currentOffsets.entrySet()) {
				unionOffsetStates.add(
						Tuple2.of(kafkaTopicPartitionLongEntry.getKey(), kafkaTopicPartitionLongEntry.getValue()));
			}
		}

		if (offsetCommitMode == OffsetCommitMode.ON_CHECKPOINTS) {
			// truncate the map of pending offsets to commit, to prevent infinite growth
			while (pendingOffsetsToCommit.size() > MAX_NUM_PENDING_CHECKPOINTS) {
				pendingOffsetsToCommit.remove(0);
			}
		}
	}
}

Example 2

Source File: FlinkKafkaConsumerBase.java From Flink-CEPplus with Apache License 2.0

4 votes

@Override
public final void notifyCheckpointComplete(long checkpointId) throws Exception {
	if (!running) {
		LOG.debug("notifyCheckpointComplete() called on closed source");
		return;
	}

	final AbstractFetcher<?, ?> fetcher = this.kafkaFetcher;
	if (fetcher == null) {
		LOG.debug("notifyCheckpointComplete() called on uninitialized source");
		return;
	}

	if (offsetCommitMode == OffsetCommitMode.ON_CHECKPOINTS) {
		// only one commit operation must be in progress
		if (LOG.isDebugEnabled()) {
			LOG.debug("Committing offsets to Kafka/ZooKeeper for checkpoint " + checkpointId);
		}

		try {
			final int posInMap = pendingOffsetsToCommit.indexOf(checkpointId);
			if (posInMap == -1) {
				LOG.warn("Received confirmation for unknown checkpoint id {}", checkpointId);
				return;
			}

			@SuppressWarnings("unchecked")
			Map<KafkaTopicPartition, Long> offsets =
				(Map<KafkaTopicPartition, Long>) pendingOffsetsToCommit.remove(posInMap);

			// remove older checkpoints in map
			for (int i = 0; i < posInMap; i++) {
				pendingOffsetsToCommit.remove(0);
			}

			if (offsets == null || offsets.size() == 0) {
				LOG.debug("Checkpoint state was empty.");
				return;
			}

			fetcher.commitInternalOffsetsToKafka(offsets, offsetCommitCallback);
		} catch (Exception e) {
			if (running) {
				throw e;
			}
			// else ignore exception if we are no longer running
		}
	}
}

Example 3

Source File: FlinkKafkaConsumerBase.java From flink with Apache License 2.0

4 votes

@Override
public final void snapshotState(FunctionSnapshotContext context) throws Exception {
	if (!running) {
		LOG.debug("snapshotState() called on closed source");
	} else {
		unionOffsetStates.clear();

		final AbstractFetcher<?, ?> fetcher = this.kafkaFetcher;
		if (fetcher == null) {
			// the fetcher has not yet been initialized, which means we need to return the
			// originally restored offsets or the assigned partitions
			for (Map.Entry<KafkaTopicPartition, Long> subscribedPartition : subscribedPartitionsToStartOffsets.entrySet()) {
				unionOffsetStates.add(Tuple2.of(subscribedPartition.getKey(), subscribedPartition.getValue()));
			}

			if (offsetCommitMode == OffsetCommitMode.ON_CHECKPOINTS) {
				// the map cannot be asynchronously updated, because only one checkpoint call can happen
				// on this function at a time: either snapshotState() or notifyCheckpointComplete()
				pendingOffsetsToCommit.put(context.getCheckpointId(), restoredState);
			}
		} else {
			HashMap<KafkaTopicPartition, Long> currentOffsets = fetcher.snapshotCurrentState();

			if (offsetCommitMode == OffsetCommitMode.ON_CHECKPOINTS) {
				// the map cannot be asynchronously updated, because only one checkpoint call can happen
				// on this function at a time: either snapshotState() or notifyCheckpointComplete()
				pendingOffsetsToCommit.put(context.getCheckpointId(), currentOffsets);
			}

			for (Map.Entry<KafkaTopicPartition, Long> kafkaTopicPartitionLongEntry : currentOffsets.entrySet()) {
				unionOffsetStates.add(
						Tuple2.of(kafkaTopicPartitionLongEntry.getKey(), kafkaTopicPartitionLongEntry.getValue()));
			}
		}

		if (offsetCommitMode == OffsetCommitMode.ON_CHECKPOINTS) {
			// truncate the map of pending offsets to commit, to prevent infinite growth
			while (pendingOffsetsToCommit.size() > MAX_NUM_PENDING_CHECKPOINTS) {
				pendingOffsetsToCommit.remove(0);
			}
		}
	}
}

Example 4

Source File: FlinkKafkaConsumerBase.java From flink with Apache License 2.0

4 votes

@Override
public final void notifyCheckpointComplete(long checkpointId) throws Exception {
	if (!running) {
		LOG.debug("notifyCheckpointComplete() called on closed source");
		return;
	}

	final AbstractFetcher<?, ?> fetcher = this.kafkaFetcher;
	if (fetcher == null) {
		LOG.debug("notifyCheckpointComplete() called on uninitialized source");
		return;
	}

	if (offsetCommitMode == OffsetCommitMode.ON_CHECKPOINTS) {
		// only one commit operation must be in progress
		if (LOG.isDebugEnabled()) {
			LOG.debug("Consumer subtask {} committing offsets to Kafka/ZooKeeper for checkpoint {}.",
				getRuntimeContext().getIndexOfThisSubtask(), checkpointId);
		}

		try {
			final int posInMap = pendingOffsetsToCommit.indexOf(checkpointId);
			if (posInMap == -1) {
				LOG.warn("Consumer subtask {} received confirmation for unknown checkpoint id {}",
					getRuntimeContext().getIndexOfThisSubtask(), checkpointId);
				return;
			}

			@SuppressWarnings("unchecked")
			Map<KafkaTopicPartition, Long> offsets =
				(Map<KafkaTopicPartition, Long>) pendingOffsetsToCommit.remove(posInMap);

			// remove older checkpoints in map
			for (int i = 0; i < posInMap; i++) {
				pendingOffsetsToCommit.remove(0);
			}

			if (offsets == null || offsets.size() == 0) {
				LOG.debug("Consumer subtask {} has empty checkpoint state.", getRuntimeContext().getIndexOfThisSubtask());
				return;
			}

			fetcher.commitInternalOffsetsToKafka(offsets, offsetCommitCallback);
		} catch (Exception e) {
			if (running) {
				throw e;
			}
			// else ignore exception if we are no longer running
		}
	}
}

Example 5

Source File: FlinkKafkaConsumerBase.java From flink with Apache License 2.0

4 votes

@Override
public final void snapshotState(FunctionSnapshotContext context) throws Exception {
	if (!running) {
		LOG.debug("snapshotState() called on closed source");
	} else {
		unionOffsetStates.clear();

		final AbstractFetcher<?, ?> fetcher = this.kafkaFetcher;
		if (fetcher == null) {
			// the fetcher has not yet been initialized, which means we need to return the
			// originally restored offsets or the assigned partitions
			for (Map.Entry<KafkaTopicPartition, Long> subscribedPartition : subscribedPartitionsToStartOffsets.entrySet()) {
				unionOffsetStates.add(Tuple2.of(subscribedPartition.getKey(), subscribedPartition.getValue()));
			}

			if (offsetCommitMode == OffsetCommitMode.ON_CHECKPOINTS) {
				// the map cannot be asynchronously updated, because only one checkpoint call can happen
				// on this function at a time: either snapshotState() or notifyCheckpointComplete()
				pendingOffsetsToCommit.put(context.getCheckpointId(), restoredState);
			}
		} else {
			HashMap<KafkaTopicPartition, Long> currentOffsets = fetcher.snapshotCurrentState();

			if (offsetCommitMode == OffsetCommitMode.ON_CHECKPOINTS) {
				// the map cannot be asynchronously updated, because only one checkpoint call can happen
				// on this function at a time: either snapshotState() or notifyCheckpointComplete()
				pendingOffsetsToCommit.put(context.getCheckpointId(), currentOffsets);
			}

			for (Map.Entry<KafkaTopicPartition, Long> kafkaTopicPartitionLongEntry : currentOffsets.entrySet()) {
				unionOffsetStates.add(
						Tuple2.of(kafkaTopicPartitionLongEntry.getKey(), kafkaTopicPartitionLongEntry.getValue()));
			}
		}

		if (offsetCommitMode == OffsetCommitMode.ON_CHECKPOINTS) {
			// truncate the map of pending offsets to commit, to prevent infinite growth
			while (pendingOffsetsToCommit.size() > MAX_NUM_PENDING_CHECKPOINTS) {
				pendingOffsetsToCommit.remove(0);
			}
		}
	}
}

Example 6

Source File: FlinkKafkaConsumerBase.java From flink with Apache License 2.0

4 votes

@Override
public final void notifyCheckpointComplete(long checkpointId) throws Exception {
	if (!running) {
		LOG.debug("notifyCheckpointComplete() called on closed source");
		return;
	}

	final AbstractFetcher<?, ?> fetcher = this.kafkaFetcher;
	if (fetcher == null) {
		LOG.debug("notifyCheckpointComplete() called on uninitialized source");
		return;
	}

	if (offsetCommitMode == OffsetCommitMode.ON_CHECKPOINTS) {
		// only one commit operation must be in progress
		if (LOG.isDebugEnabled()) {
			LOG.debug("Consumer subtask {} committing offsets to Kafka/ZooKeeper for checkpoint {}.",
				getRuntimeContext().getIndexOfThisSubtask(), checkpointId);
		}

		try {
			final int posInMap = pendingOffsetsToCommit.indexOf(checkpointId);
			if (posInMap == -1) {
				LOG.warn("Consumer subtask {} received confirmation for unknown checkpoint id {}",
					getRuntimeContext().getIndexOfThisSubtask(), checkpointId);
				return;
			}

			@SuppressWarnings("unchecked")
			Map<KafkaTopicPartition, Long> offsets =
				(Map<KafkaTopicPartition, Long>) pendingOffsetsToCommit.remove(posInMap);

			// remove older checkpoints in map
			for (int i = 0; i < posInMap; i++) {
				pendingOffsetsToCommit.remove(0);
			}

			if (offsets == null || offsets.size() == 0) {
				LOG.debug("Consumer subtask {} has empty checkpoint state.", getRuntimeContext().getIndexOfThisSubtask());
				return;
			}

			fetcher.commitInternalOffsetsToKafka(offsets, offsetCommitCallback);
		} catch (Exception e) {
			if (running) {
				throw e;
			}
			// else ignore exception if we are no longer running
		}
	}
}

Example 7

Source File: FlinkKafkaConsumerBase.java From Flink-CEPplus with Apache License 2.0

2 votes

/**
 * Make sure that auto commit is disabled when our offset commit mode is ON_CHECKPOINTS.
 * This overwrites whatever setting the user configured in the properties.
 * @param properties - Kafka configuration properties to be adjusted
 * @param offsetCommitMode offset commit mode
 */
static void adjustAutoCommitConfig(Properties properties, OffsetCommitMode offsetCommitMode) {
	if (offsetCommitMode == OffsetCommitMode.ON_CHECKPOINTS || offsetCommitMode == OffsetCommitMode.DISABLED) {
		properties.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
	}
}

Example 8

Source File: FlinkKafkaConsumerBase.java From flink with Apache License 2.0

2 votes

/**
 * Make sure that auto commit is disabled when our offset commit mode is ON_CHECKPOINTS.
 * This overwrites whatever setting the user configured in the properties.
 * @param properties - Kafka configuration properties to be adjusted
 * @param offsetCommitMode offset commit mode
 */
static void adjustAutoCommitConfig(Properties properties, OffsetCommitMode offsetCommitMode) {
	if (offsetCommitMode == OffsetCommitMode.ON_CHECKPOINTS || offsetCommitMode == OffsetCommitMode.DISABLED) {
		properties.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
	}
}

Example 9

Source File: FlinkKafkaConsumerBase.java From flink with Apache License 2.0

2 votes

/**
 * Make sure that auto commit is disabled when our offset commit mode is ON_CHECKPOINTS.
 * This overwrites whatever setting the user configured in the properties.
 * @param properties - Kafka configuration properties to be adjusted
 * @param offsetCommitMode offset commit mode
 */
protected static void adjustAutoCommitConfig(Properties properties, OffsetCommitMode offsetCommitMode) {
	if (offsetCommitMode == OffsetCommitMode.ON_CHECKPOINTS || offsetCommitMode == OffsetCommitMode.DISABLED) {
		properties.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
	}
}