org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory Java Examples

The following examples show how to use org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: EventApisFactory.java    From eventapis with Apache License 2.0 6 votes vote down vote up
@Bean({"eventsKafkaListenerContainerFactory", "kafkaListenerContainerFactory"})
public ConcurrentKafkaListenerContainerFactory<String, PublishedEventWrapper> eventsKafkaListenerContainerFactory(
        EventMessageConverter eventMessageConverter, ConsumerFactory<String, PublishedEventWrapper> consumerFactory) {

    ConcurrentKafkaListenerContainerFactory<String, PublishedEventWrapper> factory = new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConsumerFactory(consumerFactory);
    factory.setConcurrency(eventApisConfiguration.getEventBus().getConsumer().getEventConcurrency());
    factory.setMessageConverter(eventMessageConverter);
    factory.getContainerProperties().setPollTimeout(3000);
    ThreadPoolTaskScheduler scheduler = new ThreadPoolTaskScheduler();
    scheduler.setPoolSize(eventApisConfiguration.getEventBus().getConsumer().getEventSchedulerPoolSize());
    scheduler.setBeanName("EventsFactory-Scheduler");
    scheduler.initialize();

    factory.getContainerProperties().setScheduler(scheduler);
    factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.RECORD);
    return factory;
}
 
Example #2
Source File: CommonConfiguration.java    From grussell-spring-kafka with Apache License 2.0 6 votes vote down vote up
@Bean
public ConcurrentKafkaListenerContainerFactory<String, String> retryKafkaListenerContainerFactory() {
	ConcurrentKafkaListenerContainerFactory<String, String> factory =
			new ConcurrentKafkaListenerContainerFactory<>();
	factory.setConsumerFactory(consumerFactory());
	factory.setRetryTemplate(new RetryTemplate());
	factory.setRecordFilterStrategy(new RecordFilterStrategy<String, String>() {

		@Override
		public boolean filter(ConsumerRecord<String, String> consumerRecord) {
			return consumerRecord.value().equals("bar");
		}

	});
	return factory;
}
 
Example #3
Source File: ErrorConfiguration.java    From faster-framework-project with Apache License 2.0 6 votes vote down vote up
/**
 * @param kafkaTemplate                           操作类
 * @param concurrentKafkaListenerContainerFactory concurrentKafkaListenerContainerFactory
 * @return 死信批量处理器
 */
@Bean
@ConditionalOnProperty(prefix = "app.kafka.error", name = "dead-letter", havingValue = "true")
@ConditionalOnMissingBean
public GenericErrorHandler kafkaDeadLetterBatchErrorHandler(KafkaTemplate<Object, Object> kafkaTemplate,
                                                            ConcurrentKafkaListenerContainerFactory concurrentKafkaListenerContainerFactory) {
    //此处之所以要获取bean而非获取配置文件进行判断,因为spring-kafka允许注册自定义factory并且设置batchListener为true,此时配置文件参数可为空。
    if (concurrentKafkaListenerContainerFactory.isBatchListener() != null && concurrentKafkaListenerContainerFactory.isBatchListener()) {
        BatchErrorHandler batchErrorHandler = new KafkaDeadLetterBatchErrorHandler(kafkaTemplate);
        concurrentKafkaListenerContainerFactory.setBatchErrorHandler(batchErrorHandler);
        return batchErrorHandler;
    } else {
        ErrorHandler errorHandler = new KafkaDeadLetterErrorHandler(kafkaTemplate);
        concurrentKafkaListenerContainerFactory.setErrorHandler(errorHandler);
        return errorHandler;
    }
}
 
Example #4
Source File: UmcReceiveAutoConfiguration.java    From super-cloudops with Apache License 2.0 6 votes vote down vote up
@Bean(BEAN_KAFKA_BATCH_FACTORY)
@EnableKafkaCollectReceiver
@SuppressWarnings({ "unchecked", "rawtypes" })
public KafkaListenerContainerFactory<?> batchFactory(ReceiverProperties conf) {
	// Create consumer factory.
	Properties properties = conf.getKafka().getProperties();
	Map<String, Object> config = (Map) properties;
	ConsumerFactory<String, String> cf = new DefaultKafkaConsumerFactory<>(config);

	// Create concurrent consumer container factory.
	ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
	factory.setConsumerFactory(cf);
	factory.setConcurrency(conf.getKafka().getConcurrency());
	factory.setBatchListener(true);

	// Spring kafka container properties.
	ContainerProperties containerProps = factory.getContainerProperties();
	containerProps.setPollTimeout(conf.getKafka().getPollTimeout());
	// Bulk consumption change buffer queue size.
	containerProps.setQueueDepth(conf.getKafka().getQueueDepth());
	containerProps.setAckMode(AckMode.MANUAL_IMMEDIATE);
	return factory;
}
 
Example #5
Source File: KafkaConsumerConfiguration.java    From ZTuoExchange_framework with MIT License 6 votes vote down vote up
@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
	ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
	factory.setConsumerFactory(consumerFactory());
	factory.setConcurrency(concurrency);
	factory.getContainerProperties().setPollTimeout(1500);
	factory.setBatchListener(true);
	return factory;
}
 
Example #6
Source File: KafkaConsumerConfig.java    From stateful-functions with Apache License 2.0 6 votes vote down vote up
@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>>
    kafkaListenerContainerFactory(ConsumerFactory<String, String> consumerFactory) {
  ConcurrentKafkaListenerContainerFactory<String, String> factory =
      new ConcurrentKafkaListenerContainerFactory<>();
  factory.setConsumerFactory(consumerFactory);
  return factory;
}
 
Example #7
Source File: KafKaConfig.java    From gpmall with Apache License 2.0 6 votes vote down vote up
/**
 * 消费则的监听工厂
 * @return
 */
@Bean
public KafkaListenerContainerFactory userRegisterSuccKafkaListenerContainerFactory(){
    ConcurrentKafkaListenerContainerFactory conFactory = new ConcurrentKafkaListenerContainerFactory<>();
    conFactory.setConsumerFactory(kafKaRegisterSuccConsumerFactory());
    conFactory.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL);//  设置消费者消费消息后的提交方式为手动提交
    return conFactory;
}
 
Example #8
Source File: KafkaConfiguration.java    From ad with Apache License 2.0 6 votes vote down vote up
@Bean
KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
    ConcurrentKafkaListenerContainerFactory<String, String> containerFactory = new ConcurrentKafkaListenerContainerFactory<>();
    containerFactory.setConcurrency(concurrency);

    Map<String, Object> config = Maps.newHashMap();
    config.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
    config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
    config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
    // 由于课程原版实现中广告的索引数据是存在于ConcurrentHashMap中, 即每个索引服务实例的jvm中。
    // 所以当每一个索引实例监听kafka topic数据时, 需要保证每个实例都处于不同的消费者组
    // 即各个实例之间需要各不相同的groupId, 保证在部署多实例时, 每个实例都可以加载到完整的索引数据

    // 但在本实现中由于将索引数据单独提出, 存放到了Redis数据库中, 所以应该让所有实例属于同一个消费者组
    // 共同消费kafka topic下的数据, 保证索引数据不会被重复消费。

    // 综上, 若索引数据的存放如果为各个实例自身的jvm, 应该考虑加上以下代码(或自行编写其他实现)保证各实例处于不同的消费者组
    // 若索引数据存放的位置, 是所有检索实例共享的位置, 应该将以下配置取消(或直接删除本类)
    config.put(ConsumerConfig.GROUP_ID_CONFIG, UUID.randomUUID().toString());
    DefaultKafkaConsumerFactory<String, String> consumerFactory = new DefaultKafkaConsumerFactory<String, String>(config);
    containerFactory.setConsumerFactory(consumerFactory);
    return containerFactory;
}
 
Example #9
Source File: KafkaConfiguration.java    From spring-examples with GNU General Public License v3.0 5 votes vote down vote up
@Bean
public ConcurrentKafkaListenerContainerFactory<String, KMessage> kafkaListenerContainerFactory() {
    ConcurrentKafkaListenerContainerFactory<String, KMessage> factory =
            new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConsumerFactory(consumerFactory());
    return factory;
}
 
Example #10
Source File: ReceiverConfig.java    From spring-kafka with MIT License 5 votes vote down vote up
@Bean
public ConcurrentKafkaListenerContainerFactory<String, String> kafkaListenerContainerFactory() {
  ConcurrentKafkaListenerContainerFactory<String, String> factory =
      new ConcurrentKafkaListenerContainerFactory<>();
  factory.setConsumerFactory(consumerFactory());
  factory.setMessageConverter(new StringJsonMessageConverter());

  return factory;
}
 
Example #11
Source File: ReceiverConfig.java    From spring-kafka with MIT License 5 votes vote down vote up
@Bean
public ConcurrentKafkaListenerContainerFactory<String, String> kafkaListenerContainerFactory() {
  ConcurrentKafkaListenerContainerFactory<String, String> factory =
      new ConcurrentKafkaListenerContainerFactory<>();
  factory.setConsumerFactory(consumerFactory());
  // enable batch listening
  factory.setBatchListener(true);

  return factory;
}
 
Example #12
Source File: EventRegistryKafkaConfiguration.java    From flowable-engine with Apache License 2.0 5 votes vote down vote up
@Bean(name = "kafkaListenerContainerFactory")
public ConcurrentKafkaListenerContainerFactory<?, ?> kafkaListenerContainerFactory(ConsumerFactory<Object, Object> kafkaConsumerFactory) {
    ConcurrentKafkaListenerContainerFactory<Object, Object> factory = new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConsumerFactory(kafkaConsumerFactory);

    ContainerProperties containerProperties = factory.getContainerProperties();
    containerProperties.setMissingTopicsFatal(true);
    return factory;
}
 
Example #13
Source File: ReceiverConfig.java    From spring-kafka with MIT License 5 votes vote down vote up
@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
  ConcurrentKafkaListenerContainerFactory<String, String> factory =
      new ConcurrentKafkaListenerContainerFactory<>();
  factory.setConsumerFactory(consumerFactory());

  return factory;
}
 
Example #14
Source File: ReceiverConfig.java    From spring-kafka with MIT License 5 votes vote down vote up
@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
  ConcurrentKafkaListenerContainerFactory<String, String> factory =
      new ConcurrentKafkaListenerContainerFactory<>();
  factory.setConsumerFactory(consumerFactory());

  return factory;
}
 
Example #15
Source File: Application.java    From spring-cloud-contract-samples with Apache License 2.0 5 votes vote down vote up
@Bean
public ConcurrentKafkaListenerContainerFactory<?, ?> kafkaListenerContainerFactory(
		ConcurrentKafkaListenerContainerFactoryConfigurer configurer,
		ConsumerFactory<Object, Object> kafkaConsumerFactory,
		KafkaTemplate<Object, Object> template) {
	ConcurrentKafkaListenerContainerFactory<Object, Object> factory = new ConcurrentKafkaListenerContainerFactory<>();
	configurer.configure(factory, kafkaConsumerFactory);
	factory.setErrorHandler(new SeekToCurrentErrorHandler(
			new DeadLetterPublishingRecoverer(template), 3)); // dead-letter after 3 tries
	return factory;
}
 
Example #16
Source File: ReceiverConfig.java    From spring-kafka with MIT License 5 votes vote down vote up
@Bean
public ConcurrentKafkaListenerContainerFactory<String, Car> kafkaListenerContainerFactory() {
  ConcurrentKafkaListenerContainerFactory<String, Car> factory =
      new ConcurrentKafkaListenerContainerFactory<>();
  factory.setConsumerFactory(consumerFactory());

  return factory;
}
 
Example #17
Source File: AConsumerConfig.java    From SO with BSD 2-Clause "Simplified" License 5 votes vote down vote up
/**
 * return KafkaListenerContainerFactory.<BR/>
 *
 * @return KafkaListenerContainerFactory
 */
@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
    log.debug("kafkaListenerContainerFactory()");
    ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConsumerFactory(consumerFactory());
    factory.setConcurrency(Settings2.HANDLER_COUNT);
    factory.getContainerProperties().setPollTimeout(Settings2.getPollTimeout());
    return factory;
}
 
Example #18
Source File: KafkaConsumerConfig.java    From tutorials with MIT License 5 votes vote down vote up
@Bean
public ConcurrentKafkaListenerContainerFactory<String, String> filterKafkaListenerContainerFactory() {
    ConcurrentKafkaListenerContainerFactory<String, String> factory = kafkaListenerContainerFactory("filter");
    factory.setRecordFilterStrategy(record -> record.value()
        .contains("World"));
    return factory;
}
 
Example #19
Source File: JsonConfiguration.java    From grussell-spring-kafka with Apache License 2.0 5 votes vote down vote up
@Bean
public ConcurrentKafkaListenerContainerFactory<String, Foo> kafkaListenerContainerFactory() {
	ConcurrentKafkaListenerContainerFactory<String, Foo> factory =
			new ConcurrentKafkaListenerContainerFactory<>();
	factory.setConsumerFactory(consumerFactory());
	return factory;
}
 
Example #20
Source File: ReceiverConfig.java    From spring-kafka with MIT License 5 votes vote down vote up
@Bean
public ConcurrentKafkaListenerContainerFactory<String, User> kafkaListenerContainerFactory() {
  ConcurrentKafkaListenerContainerFactory<String, User> factory =
      new ConcurrentKafkaListenerContainerFactory<>();
  factory.setConsumerFactory(consumerFactory());

  return factory;
}
 
Example #21
Source File: CommonConfiguration.java    From grussell-spring-kafka with Apache License 2.0 5 votes vote down vote up
@Bean
public ConcurrentKafkaListenerContainerFactory<String, String> jsonKafkaListenerContainerFactory() {
	ConcurrentKafkaListenerContainerFactory<String, String> factory =
			new ConcurrentKafkaListenerContainerFactory<>();
	factory.setConsumerFactory(consumerFactory());
	factory.setMessageConverter(new StringJsonMessageConverter());
	return factory;
}
 
Example #22
Source File: CommonConfiguration.java    From grussell-spring-kafka with Apache License 2.0 5 votes vote down vote up
@Bean
public ConcurrentKafkaListenerContainerFactory<String, String> kafkaListenerContainerFactory() {
	ConcurrentKafkaListenerContainerFactory<String, String> factory =
			new ConcurrentKafkaListenerContainerFactory<>();
	factory.setConsumerFactory(consumerFactory());
	return factory;
}
 
Example #23
Source File: AConsumerConfig.java    From SO with BSD 2-Clause "Simplified" License 5 votes vote down vote up
/**
 * return KafkaListenerContainerFactory.<BR/>
 *
 * @return KafkaListenerContainerFactory
 */
@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
    log.debug("kafkaListenerContainerFactory()");
    ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConsumerFactory(consumerFactory());
    factory.setConcurrency(Settings2.HANDLER_COUNT);
    factory.getContainerProperties().setPollTimeout(Settings2.getPollTimeout());
    return factory;
}
 
Example #24
Source File: KafkaConfig.java    From spring-boot-demo with MIT License 5 votes vote down vote up
@Bean("ackContainerFactory")
public ConcurrentKafkaListenerContainerFactory<String, String> ackContainerFactory() {
    ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConsumerFactory(consumerFactory());
    factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL_IMMEDIATE);
    factory.setConcurrency(KafkaConsts.DEFAULT_PARTITION_NUM);
    return factory;
}
 
Example #25
Source File: KafkaConfig.java    From spring-boot-demo with MIT License 5 votes vote down vote up
@Bean
public ConcurrentKafkaListenerContainerFactory<String, String> kafkaListenerContainerFactory() {
    ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConsumerFactory(consumerFactory());
    factory.setConcurrency(KafkaConsts.DEFAULT_PARTITION_NUM);
    factory.setBatchListener(true);
    factory.getContainerProperties().setPollTimeout(3000);
    return factory;
}
 
Example #26
Source File: LocKafkaAutoConfiguration.java    From loc-framework with MIT License 5 votes vote down vote up
@Bean(name = "kafkaListenerContainerFactory")
public ConcurrentKafkaListenerContainerFactory<?, ?> kafkaListenerContainerFactory(
    ConcurrentKafkaListenerContainerFactoryConfigurer configurer,
    ConsumerFactory<Object, Object> kafkaConsumerFactory) {
  ConcurrentKafkaListenerContainerFactory<Object, Object> factory = new ConcurrentKafkaListenerContainerFactory<>();
  ContainerProperties containerProperties = factory.getContainerProperties();
  factory.setRecordFilterStrategy(locMessageFilterStrategy());
  factory.setErrorHandler(new LocKafkaConsumerErrorHandler());
  factory.setMessageConverter(recordMessageConverter());
  configurer.configure(factory, kafkaConsumerFactory);
  return factory;
}
 
Example #27
Source File: KafkaConfig.java    From gae with MIT License 5 votes vote down vote up
@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
    ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConsumerFactory(consumerFactory());
    factory.setConcurrency(3);
    factory.getContainerProperties().setPollTimeout(3000);
    return factory;
}
 
Example #28
Source File: KafkaConsumerConfig.java    From springboot_cwenao with MIT License 5 votes vote down vote up
@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {

    ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<String, String>();
    factory.setConsumerFactory(consumerFactory());
    factory.setConcurrency(4);
    factory.getContainerProperties().setPollTimeout(4000);

    return factory;
}
 
Example #29
Source File: ProductConsumerConfiguration.java    From integration-patterns with MIT License 5 votes vote down vote up
@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
    ConcurrentKafkaListenerContainerFactory<String, String> factory =
            new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConsumerFactory(consumerFactory());
    factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL_IMMEDIATE);

    return factory;
}
 
Example #30
Source File: EventApisFactory.java    From eventapis with Apache License 2.0 5 votes vote down vote up
@Bean("operationsKafkaListenerContainerFactory")
    public ConcurrentKafkaListenerContainerFactory<String, Operation> operationsKafkaListenerContainerFactory(
            ConsumerFactory<String, Operation> consumerFactory) {
        ConcurrentKafkaListenerContainerFactory<String, Operation> factory
                = new ConcurrentKafkaListenerContainerFactory<>();
        factory.setConsumerFactory(consumerFactory);
        RetryTemplate retryTemplate = new RetryTemplate();
        factory.setRetryTemplate(retryTemplate);

        factory.setConcurrency(eventApisConfiguration.getEventBus().getConsumer().getOperationSchedulerPoolSize());
        ThreadPoolTaskScheduler scheduler = new ThreadPoolTaskScheduler();
        scheduler.setPoolSize(eventApisConfiguration.getEventBus().getConsumer().getOperationSchedulerPoolSize());
        scheduler.setBeanName("OperationsFactory-Scheduler");
        scheduler.initialize();
        factory.getContainerProperties().setScheduler(scheduler);
        ThreadPoolTaskScheduler consumerScheduler = new ThreadPoolTaskScheduler();
        consumerScheduler.setPoolSize(eventApisConfiguration.getEventBus().getConsumer().getOperationSchedulerPoolSize());
        consumerScheduler.setBeanName("OperationsFactory-ConsumerScheduler");
        consumerScheduler.initialize();

        factory.getContainerProperties().setPollTimeout(3000L);
        factory.getContainerProperties().setAckOnError(false);
        factory.getContainerProperties().setConsumerTaskExecutor(consumerScheduler);
        factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.RECORD);
        /**
         * This is Fix for Spring Kafka versions which does not have ConsumerAwareErrorHandler handler till 2.0
         * When Listener faces with error, it retries snapshot operation
         * See https://github.com/kloiasoft/eventapis/issues/44
         */
        factory.getContainerProperties().setTransactionManager(new EmptyTransactionManager());
//        factory.getContainerProperties().setTransactionManager(platformTransactionManager);
        return factory;
    }