org.apache.flink.streaming.connectors.kafka.config.OffsetCommitMode Java Examples

The following examples show how to use org.apache.flink.streaming.connectors.kafka.config.OffsetCommitMode. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
protected AbstractFetcher<T, ?> createFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> thisSubtaskPartitionsWithStartOffsets,
		SerializedValue<WatermarkStrategy<T>> watermarkStrategy,
		StreamingRuntimeContext runtimeContext,
		OffsetCommitMode offsetCommitMode,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {
	return new TestingFetcher<T, String>(
			sourceContext,
			thisSubtaskPartitionsWithStartOffsets,
			watermarkStrategy,
			runtimeContext.getProcessingTimeService(),
			0L,
			getClass().getClassLoader(),
			consumerMetricGroup,
			useMetrics);
}
Example #2
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testConfigureOnCheckpointsCommitMode() throws Exception {
	@SuppressWarnings("unchecked")
	// auto-commit enabled; this should be ignored in this case
	final DummyFlinkKafkaConsumer<String> consumer = new DummyFlinkKafkaConsumer<>(true);

	setupConsumer(
		consumer,
		false,
		null,
		true, // enable checkpointing; auto commit should be ignored
		0,
		1);

	assertEquals(OffsetCommitMode.ON_CHECKPOINTS, consumer.getOffsetCommitMode());
}
Example #3
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testConfigureDisableOffsetCommitWithCheckpointing() throws Exception {
	@SuppressWarnings("unchecked")
	// auto-commit enabled; this should be ignored in this case
	final DummyFlinkKafkaConsumer<String> consumer = new DummyFlinkKafkaConsumer<>(true);
	consumer.setCommitOffsetsOnCheckpoints(false); // disabling offset committing should override everything

	setupConsumer(
		consumer,
		false,
		null,
		true, // enable checkpointing; auto commit should be ignored
		0,
		1);

	assertEquals(OffsetCommitMode.DISABLED, consumer.getOffsetCommitMode());
}
Example #4
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testConfigureOnCheckpointsCommitMode() throws Exception {
	@SuppressWarnings("unchecked")
	// auto-commit enabled; this should be ignored in this case
	final DummyFlinkKafkaConsumer<String> consumer = new DummyFlinkKafkaConsumer<>(true);

	setupConsumer(
		consumer,
		false,
		null,
		true, // enable checkpointing; auto commit should be ignored
		0,
		1);

	assertEquals(OffsetCommitMode.ON_CHECKPOINTS, consumer.getOffsetCommitMode());
}
Example #5
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testConfigureDisableOffsetCommitWithCheckpointing() throws Exception {
	@SuppressWarnings("unchecked")
	// auto-commit enabled; this should be ignored in this case
	final DummyFlinkKafkaConsumer<String> consumer = new DummyFlinkKafkaConsumer<>(true);
	consumer.setCommitOffsetsOnCheckpoints(false); // disabling offset committing should override everything

	setupConsumer(
		consumer,
		false,
		null,
		true, // enable checkpointing; auto commit should be ignored
		0,
		1);

	assertEquals(OffsetCommitMode.DISABLED, consumer.getOffsetCommitMode());
}
Example #6
Source File: FlinkKafkaConsumerBaseTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testConfigureOnCheckpointsCommitMode() throws Exception {
	@SuppressWarnings("unchecked")
	// auto-commit enabled; this should be ignored in this case
	final DummyFlinkKafkaConsumer<String> consumer = new DummyFlinkKafkaConsumer<>(true);

	setupConsumer(
		consumer,
		false,
		null,
		true, // enable checkpointing; auto commit should be ignored
		0,
		1);

	assertEquals(OffsetCommitMode.ON_CHECKPOINTS, consumer.getOffsetCommitMode());
}
Example #7
Source File: FlinkKafkaConsumerBaseTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testConfigureDisableOffsetCommitWithCheckpointing() throws Exception {
	@SuppressWarnings("unchecked")
	// auto-commit enabled; this should be ignored in this case
	final DummyFlinkKafkaConsumer<String> consumer = new DummyFlinkKafkaConsumer<>(true);
	consumer.setCommitOffsetsOnCheckpoints(false); // disabling offset committing should override everything

	setupConsumer(
		consumer,
		false,
		null,
		true, // enable checkpointing; auto commit should be ignored
		0,
		1);

	assertEquals(OffsetCommitMode.DISABLED, consumer.getOffsetCommitMode());
}
Example #8
Source File: FlinkKafkaConsumerBaseMigrationTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
protected AbstractFetcher<T, ?> createFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> thisSubtaskPartitionsWithStartOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		StreamingRuntimeContext runtimeContext,
		OffsetCommitMode offsetCommitMode,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {
	return fetcher;
}
Example #9
Source File: FlinkKafkaConsumer08.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
protected AbstractFetcher<T, ?> createFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> assignedPartitionsWithInitialOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		StreamingRuntimeContext runtimeContext,
		OffsetCommitMode offsetCommitMode,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {

	long autoCommitInterval = (offsetCommitMode == OffsetCommitMode.KAFKA_PERIODIC)
			? PropertiesUtil.getLong(kafkaProperties, "auto.commit.interval.ms", 60000)
			: -1; // this disables the periodic offset committer thread in the fetcher

	return new Kafka08Fetcher<>(
			sourceContext,
			assignedPartitionsWithInitialOffsets,
			watermarksPeriodic,
			watermarksPunctuated,
			runtimeContext,
			deserializer,
			kafkaProperties,
			autoCommitInterval,
			consumerMetricGroup,
			useMetrics);
}
Example #10
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testConfigureAutoCommitMode() throws Exception {
	@SuppressWarnings("unchecked")
	final DummyFlinkKafkaConsumer<String> consumer = new DummyFlinkKafkaConsumer<>(true);

	setupConsumer(consumer);

	assertEquals(OffsetCommitMode.KAFKA_PERIODIC, consumer.getOffsetCommitMode());
}
Example #11
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testConfigureDisableOffsetCommitWithoutCheckpointing() throws Exception {
	@SuppressWarnings("unchecked")
	final DummyFlinkKafkaConsumer<String> consumer = new DummyFlinkKafkaConsumer<>(false);

	setupConsumer(consumer);

	assertEquals(OffsetCommitMode.DISABLED, consumer.getOffsetCommitMode());
}
Example #12
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
@SuppressWarnings("unchecked")
protected AbstractFetcher<T, ?> createFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> thisSubtaskPartitionsWithStartOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		StreamingRuntimeContext runtimeContext,
		OffsetCommitMode offsetCommitMode,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {
	return testFetcherSupplier.get();
}
Example #13
Source File: FlinkKafkaShuffleConsumer.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
protected AbstractFetcher<T, ?> createFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> assignedPartitionsWithInitialOffsets,
		SerializedValue<WatermarkStrategy<T>> watermarkStrategy,
		StreamingRuntimeContext runtimeContext,
		OffsetCommitMode offsetCommitMode,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {
	// make sure that auto commit is disabled when our offset commit mode is ON_CHECKPOINTS;
	// this overwrites whatever setting the user configured in the properties
	adjustAutoCommitConfig(properties, offsetCommitMode);

	return new KafkaShuffleFetcher<>(
		sourceContext,
		assignedPartitionsWithInitialOffsets,
		watermarkStrategy,
		runtimeContext.getProcessingTimeService(),
		runtimeContext.getExecutionConfig().getAutoWatermarkInterval(),
		runtimeContext.getUserCodeClassLoader(),
		runtimeContext.getTaskNameWithSubtasks(),
		deserializer,
		properties,
		pollTimeout,
		runtimeContext.getMetricGroup(),
		consumerMetricGroup,
		useMetrics,
		typeSerializer,
		producerParallelism);
}
Example #14
Source File: FlinkKafkaConsumer.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
protected AbstractFetcher<T, ?> createFetcher(
	SourceContext<T> sourceContext,
	Map<KafkaTopicPartition, Long> assignedPartitionsWithInitialOffsets,
	SerializedValue<WatermarkStrategy<T>> watermarkStrategy,
	StreamingRuntimeContext runtimeContext,
	OffsetCommitMode offsetCommitMode,
	MetricGroup consumerMetricGroup,
	boolean useMetrics) throws Exception {

	// make sure that auto commit is disabled when our offset commit mode is ON_CHECKPOINTS;
	// this overwrites whatever setting the user configured in the properties
	adjustAutoCommitConfig(properties, offsetCommitMode);

	return new KafkaFetcher<>(
		sourceContext,
		assignedPartitionsWithInitialOffsets,
		watermarkStrategy,
		runtimeContext.getProcessingTimeService(),
		runtimeContext.getExecutionConfig().getAutoWatermarkInterval(),
		runtimeContext.getUserCodeClassLoader(),
		runtimeContext.getTaskNameWithSubtasks(),
		deserializer,
		properties,
		pollTimeout,
		runtimeContext.getMetricGroup(),
		consumerMetricGroup,
		useMetrics);
}
Example #15
Source File: FlinkKafkaConsumer010.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
protected AbstractFetcher<T, ?> createFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> assignedPartitionsWithInitialOffsets,
		SerializedValue<WatermarkStrategy<T>> watermarkStrategy,
		StreamingRuntimeContext runtimeContext,
		OffsetCommitMode offsetCommitMode,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {

	// make sure that auto commit is disabled when our offset commit mode is ON_CHECKPOINTS;
	// this overwrites whatever setting the user configured in the properties
	adjustAutoCommitConfig(properties, offsetCommitMode);

	// If a rateLimiter is set, then call rateLimiter.open() with the runtime context.
	if (rateLimiter != null) {
		rateLimiter.open(runtimeContext);
	}

	return new Kafka010Fetcher<>(
			sourceContext,
			assignedPartitionsWithInitialOffsets,
			watermarkStrategy,
			runtimeContext.getProcessingTimeService(),
			runtimeContext.getExecutionConfig().getAutoWatermarkInterval(),
			runtimeContext.getUserCodeClassLoader(),
			runtimeContext.getTaskNameWithSubtasks(),
			deserializer,
			properties,
			pollTimeout,
			runtimeContext.getMetricGroup(),
			consumerMetricGroup,
			useMetrics,
			rateLimiter);
}
Example #16
Source File: FlinkKafkaConsumerBaseMigrationTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
protected AbstractFetcher<T, ?> createFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> thisSubtaskPartitionsWithStartOffsets,
		SerializedValue<WatermarkStrategy<T>> watermarkStrategy,
		StreamingRuntimeContext runtimeContext,
		OffsetCommitMode offsetCommitMode,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {
	return fetcher;
}
Example #17
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testConfigureAutoCommitMode() throws Exception {
	@SuppressWarnings("unchecked")
	final DummyFlinkKafkaConsumer<String> consumer = new DummyFlinkKafkaConsumer<>(true);

	setupConsumer(consumer);

	assertEquals(OffsetCommitMode.KAFKA_PERIODIC, consumer.getOffsetCommitMode());
}
Example #18
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testConfigureDisableOffsetCommitWithoutCheckpointing() throws Exception {
	@SuppressWarnings("unchecked")
	final DummyFlinkKafkaConsumer<String> consumer = new DummyFlinkKafkaConsumer<>(false);

	setupConsumer(consumer);

	assertEquals(OffsetCommitMode.DISABLED, consumer.getOffsetCommitMode());
}
Example #19
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
protected AbstractFetcher<T, ?> createFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> thisSubtaskPartitionsWithStartOffsets,
		SerializedValue<WatermarkStrategy<T>> watermarkStrategy,
		StreamingRuntimeContext runtimeContext,
		OffsetCommitMode offsetCommitMode,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {
	return testFetcherSupplier.get();
}
Example #20
Source File: FlinkKafkaConsumerBaseMigrationTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
protected AbstractFetcher<T, ?> createFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> thisSubtaskPartitionsWithStartOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		StreamingRuntimeContext runtimeContext,
		OffsetCommitMode offsetCommitMode,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {
	return fetcher;
}
Example #21
Source File: FlinkKafkaConsumer09.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
protected AbstractFetcher<T, ?> createFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> assignedPartitionsWithInitialOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		StreamingRuntimeContext runtimeContext,
		OffsetCommitMode offsetCommitMode,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {

	// make sure that auto commit is disabled when our offset commit mode is ON_CHECKPOINTS;
	// this overwrites whatever setting the user configured in the properties
	adjustAutoCommitConfig(properties, offsetCommitMode);

	// If a rateLimiter is set, then call rateLimiter.open() with the runtime context.
	if (rateLimiter != null) {
		rateLimiter.open(runtimeContext);
	}

	return new Kafka09Fetcher<>(
			sourceContext,
			assignedPartitionsWithInitialOffsets,
			watermarksPeriodic,
			watermarksPunctuated,
			runtimeContext.getProcessingTimeService(),
			runtimeContext.getExecutionConfig().getAutoWatermarkInterval(),
			runtimeContext.getUserCodeClassLoader(),
			runtimeContext.getTaskNameWithSubtasks(),
			deserializer,
			properties,
			pollTimeout,
			runtimeContext.getMetricGroup(),
			consumerMetricGroup,
			useMetrics,
			rateLimiter);
}
Example #22
Source File: FlinkKafkaConsumer.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
protected AbstractFetcher<T, ?> createFetcher(
	SourceContext<T> sourceContext,
	Map<KafkaTopicPartition, Long> assignedPartitionsWithInitialOffsets,
	SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
	SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
	StreamingRuntimeContext runtimeContext,
	OffsetCommitMode offsetCommitMode,
	MetricGroup consumerMetricGroup,
	boolean useMetrics) throws Exception {

	// make sure that auto commit is disabled when our offset commit mode is ON_CHECKPOINTS;
	// this overwrites whatever setting the user configured in the properties
	adjustAutoCommitConfig(properties, offsetCommitMode);

	return new KafkaFetcher<>(
		sourceContext,
		assignedPartitionsWithInitialOffsets,
		watermarksPeriodic,
		watermarksPunctuated,
		runtimeContext.getProcessingTimeService(),
		runtimeContext.getExecutionConfig().getAutoWatermarkInterval(),
		runtimeContext.getUserCodeClassLoader(),
		runtimeContext.getTaskNameWithSubtasks(),
		deserializer,
		properties,
		pollTimeout,
		runtimeContext.getMetricGroup(),
		consumerMetricGroup,
		useMetrics);
}
Example #23
Source File: FlinkKafkaConsumer010.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
protected AbstractFetcher<T, ?> createFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> assignedPartitionsWithInitialOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		StreamingRuntimeContext runtimeContext,
		OffsetCommitMode offsetCommitMode,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {

	// make sure that auto commit is disabled when our offset commit mode is ON_CHECKPOINTS;
	// this overwrites whatever setting the user configured in the properties
	adjustAutoCommitConfig(properties, offsetCommitMode);

	FlinkConnectorRateLimiter rateLimiter = super.getRateLimiter();
	// If a rateLimiter is set, then call rateLimiter.open() with the runtime context.
	if (rateLimiter != null) {
		rateLimiter.open(runtimeContext);
	}

	return new Kafka010Fetcher<>(
			sourceContext,
			assignedPartitionsWithInitialOffsets,
			watermarksPeriodic,
			watermarksPunctuated,
			runtimeContext.getProcessingTimeService(),
			runtimeContext.getExecutionConfig().getAutoWatermarkInterval(),
			runtimeContext.getUserCodeClassLoader(),
			runtimeContext.getTaskNameWithSubtasks(),
			deserializer,
			properties,
			pollTimeout,
			runtimeContext.getMetricGroup(),
			consumerMetricGroup,
			useMetrics,
			rateLimiter);
}
Example #24
Source File: FlinkKafkaConsumer010.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
protected AbstractFetcher<T, ?> createFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> assignedPartitionsWithInitialOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		StreamingRuntimeContext runtimeContext,
		OffsetCommitMode offsetCommitMode,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {

	// make sure that auto commit is disabled when our offset commit mode is ON_CHECKPOINTS;
	// this overwrites whatever setting the user configured in the properties
	adjustAutoCommitConfig(properties, offsetCommitMode);

	FlinkConnectorRateLimiter rateLimiter = super.getRateLimiter();
	// If a rateLimiter is set, then call rateLimiter.open() with the runtime context.
	if (rateLimiter != null) {
		rateLimiter.open(runtimeContext);
	}

	return new Kafka010Fetcher<>(
			sourceContext,
			assignedPartitionsWithInitialOffsets,
			watermarksPeriodic,
			watermarksPunctuated,
			runtimeContext.getProcessingTimeService(),
			runtimeContext.getExecutionConfig().getAutoWatermarkInterval(),
			runtimeContext.getUserCodeClassLoader(),
			runtimeContext.getTaskNameWithSubtasks(),
			deserializer,
			properties,
			pollTimeout,
			runtimeContext.getMetricGroup(),
			consumerMetricGroup,
			useMetrics,
			rateLimiter);
}
Example #25
Source File: FlinkKafkaConsumerBaseTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Test
public void testConfigureAutoCommitMode() throws Exception {
	@SuppressWarnings("unchecked")
	final DummyFlinkKafkaConsumer<String> consumer = new DummyFlinkKafkaConsumer<>(true);

	setupConsumer(consumer);

	assertEquals(OffsetCommitMode.KAFKA_PERIODIC, consumer.getOffsetCommitMode());
}
Example #26
Source File: FlinkKafkaConsumerBaseTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Test
public void testConfigureDisableOffsetCommitWithoutCheckpointing() throws Exception {
	@SuppressWarnings("unchecked")
	final DummyFlinkKafkaConsumer<String> consumer = new DummyFlinkKafkaConsumer<>(false);

	setupConsumer(consumer);

	assertEquals(OffsetCommitMode.DISABLED, consumer.getOffsetCommitMode());
}
Example #27
Source File: FlinkKafkaConsumerBaseTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
@SuppressWarnings("unchecked")
protected AbstractFetcher<T, ?> createFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> thisSubtaskPartitionsWithStartOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		StreamingRuntimeContext runtimeContext,
		OffsetCommitMode offsetCommitMode,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {
	return testFetcherSupplier.get();
}
Example #28
Source File: FlinkKafkaConsumer09.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
protected AbstractFetcher<T, ?> createFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> assignedPartitionsWithInitialOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		StreamingRuntimeContext runtimeContext,
		OffsetCommitMode offsetCommitMode,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {

	// make sure that auto commit is disabled when our offset commit mode is ON_CHECKPOINTS;
	// this overwrites whatever setting the user configured in the properties
	adjustAutoCommitConfig(properties, offsetCommitMode);

	// If a rateLimiter is set, then call rateLimiter.open() with the runtime context.
	if (rateLimiter != null) {
		rateLimiter.open(runtimeContext);
	}

	return new Kafka09Fetcher<>(
			sourceContext,
			assignedPartitionsWithInitialOffsets,
			watermarksPeriodic,
			watermarksPunctuated,
			runtimeContext.getProcessingTimeService(),
			runtimeContext.getExecutionConfig().getAutoWatermarkInterval(),
			runtimeContext.getUserCodeClassLoader(),
			runtimeContext.getTaskNameWithSubtasks(),
			deserializer,
			properties,
			pollTimeout,
			runtimeContext.getMetricGroup(),
			consumerMetricGroup,
			useMetrics,
			rateLimiter);
}
Example #29
Source File: FlinkKafkaConsumer08.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
protected AbstractFetcher<T, ?> createFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> assignedPartitionsWithInitialOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		StreamingRuntimeContext runtimeContext,
		OffsetCommitMode offsetCommitMode,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {

	long autoCommitInterval = (offsetCommitMode == OffsetCommitMode.KAFKA_PERIODIC)
			? PropertiesUtil.getLong(kafkaProperties, "auto.commit.interval.ms", 60000)
			: -1; // this disables the periodic offset committer thread in the fetcher

	return new Kafka08Fetcher<>(
			sourceContext,
			assignedPartitionsWithInitialOffsets,
			watermarksPeriodic,
			watermarksPunctuated,
			runtimeContext,
			deserializer,
			kafkaProperties,
			autoCommitInterval,
			consumerMetricGroup,
			useMetrics);
}
Example #30
Source File: FlinkKafkaConsumer.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
protected AbstractFetcher<T, ?> createFetcher(
	SourceContext<T> sourceContext,
	Map<KafkaTopicPartition, Long> assignedPartitionsWithInitialOffsets,
	SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
	SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
	StreamingRuntimeContext runtimeContext,
	OffsetCommitMode offsetCommitMode,
	MetricGroup consumerMetricGroup,
	boolean useMetrics) throws Exception {

	// make sure that auto commit is disabled when our offset commit mode is ON_CHECKPOINTS;
	// this overwrites whatever setting the user configured in the properties
	adjustAutoCommitConfig(properties, offsetCommitMode);

	return new KafkaFetcher<>(
		sourceContext,
		assignedPartitionsWithInitialOffsets,
		watermarksPeriodic,
		watermarksPunctuated,
		runtimeContext.getProcessingTimeService(),
		runtimeContext.getExecutionConfig().getAutoWatermarkInterval(),
		runtimeContext.getUserCodeClassLoader(),
		runtimeContext.getTaskNameWithSubtasks(),
		deserializer,
		properties,
		pollTimeout,
		runtimeContext.getMetricGroup(),
		consumerMetricGroup,
		useMetrics);
}