org.springframework.kafka.config.KafkaListenerContainerFactory Java Examples

The following examples show how to use org.springframework.kafka.config.KafkaListenerContainerFactory. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KafkaConfiguration.java    From ad with Apache License 2.0 6 votes vote down vote up
@Bean
KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
    ConcurrentKafkaListenerContainerFactory<String, String> containerFactory = new ConcurrentKafkaListenerContainerFactory<>();
    containerFactory.setConcurrency(concurrency);

    Map<String, Object> config = Maps.newHashMap();
    config.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
    config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
    config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
    // 由于课程原版实现中广告的索引数据是存在于ConcurrentHashMap中, 即每个索引服务实例的jvm中。
    // 所以当每一个索引实例监听kafka topic数据时, 需要保证每个实例都处于不同的消费者组
    // 即各个实例之间需要各不相同的groupId, 保证在部署多实例时, 每个实例都可以加载到完整的索引数据

    // 但在本实现中由于将索引数据单独提出, 存放到了Redis数据库中, 所以应该让所有实例属于同一个消费者组
    // 共同消费kafka topic下的数据, 保证索引数据不会被重复消费。

    // 综上, 若索引数据的存放如果为各个实例自身的jvm, 应该考虑加上以下代码(或自行编写其他实现)保证各实例处于不同的消费者组
    // 若索引数据存放的位置, 是所有检索实例共享的位置, 应该将以下配置取消(或直接删除本类)
    config.put(ConsumerConfig.GROUP_ID_CONFIG, UUID.randomUUID().toString());
    DefaultKafkaConsumerFactory<String, String> consumerFactory = new DefaultKafkaConsumerFactory<String, String>(config);
    containerFactory.setConsumerFactory(consumerFactory);
    return containerFactory;
}
 
Example #2
Source File: KafKaConfig.java    From gpmall with Apache License 2.0 6 votes vote down vote up
/**
 * 消费则的监听工厂
 * @return
 */
@Bean
public KafkaListenerContainerFactory userRegisterSuccKafkaListenerContainerFactory(){
    ConcurrentKafkaListenerContainerFactory conFactory = new ConcurrentKafkaListenerContainerFactory<>();
    conFactory.setConsumerFactory(kafKaRegisterSuccConsumerFactory());
    conFactory.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL);//  设置消费者消费消息后的提交方式为手动提交
    return conFactory;
}
 
Example #3
Source File: KafkaConsumerConfig.java    From stateful-functions with Apache License 2.0 6 votes vote down vote up
@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>>
    kafkaListenerContainerFactory(ConsumerFactory<String, String> consumerFactory) {
  ConcurrentKafkaListenerContainerFactory<String, String> factory =
      new ConcurrentKafkaListenerContainerFactory<>();
  factory.setConsumerFactory(consumerFactory);
  return factory;
}
 
Example #4
Source File: KafkaConsumerConfiguration.java    From ZTuoExchange_framework with MIT License 6 votes vote down vote up
@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
	ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
	factory.setConsumerFactory(consumerFactory());
	factory.setConcurrency(concurrency);
	factory.getContainerProperties().setPollTimeout(1500);
	factory.setBatchListener(true);
	return factory;
}
 
Example #5
Source File: KafkaChannelDefinitionProcessor.java    From flowable-engine with Apache License 2.0 6 votes vote down vote up
protected KafkaListenerContainerFactory<?> resolveContainerFactory(KafkaListenerEndpoint endpoint, KafkaListenerContainerFactory<?> containerFactory) {
    if (containerFactory != null) {
        return containerFactory;
    } else if (this.containerFactory != null) {
        return this.containerFactory;
    } else if (containerFactoryBeanName != null) {
        Assert.state(beanFactory != null, "BeanFactory must be set to obtain container factory by bean name");
        // Consider changing this if live change of the factory is required...
        this.containerFactory = beanFactory.getBean(containerFactoryBeanName, KafkaListenerContainerFactory.class);
        return this.containerFactory;
    } else {
        throw new IllegalStateException("Could not resolve the " +
            KafkaListenerContainerFactory.class.getSimpleName() + " to use for [" +
            endpoint + "] no factory was given and no default is set.");
    }
}
 
Example #6
Source File: UmcReceiveAutoConfiguration.java    From super-cloudops with Apache License 2.0 6 votes vote down vote up
@Bean(BEAN_KAFKA_BATCH_FACTORY)
@EnableKafkaCollectReceiver
@SuppressWarnings({ "unchecked", "rawtypes" })
public KafkaListenerContainerFactory<?> batchFactory(ReceiverProperties conf) {
	// Create consumer factory.
	Properties properties = conf.getKafka().getProperties();
	Map<String, Object> config = (Map) properties;
	ConsumerFactory<String, String> cf = new DefaultKafkaConsumerFactory<>(config);

	// Create concurrent consumer container factory.
	ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
	factory.setConsumerFactory(cf);
	factory.setConcurrency(conf.getKafka().getConcurrency());
	factory.setBatchListener(true);

	// Spring kafka container properties.
	ContainerProperties containerProps = factory.getContainerProperties();
	containerProps.setPollTimeout(conf.getKafka().getPollTimeout());
	// Bulk consumption change buffer queue size.
	containerProps.setQueueDepth(conf.getKafka().getQueueDepth());
	containerProps.setAckMode(AckMode.MANUAL_IMMEDIATE);
	return factory;
}
 
Example #7
Source File: KafkaConsumerConfig.java    From MicroCommunity with Apache License 2.0 5 votes vote down vote up
@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
    ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<String, String>();
    factory.setConsumerFactory(consumerFactory());
    factory.setConcurrency(concurrency);
    factory.getContainerProperties().setPollTimeout(1500);
    return factory;
}
 
Example #8
Source File: AConsumerConfig.java    From SO with BSD 2-Clause "Simplified" License 5 votes vote down vote up
/**
 * return KafkaListenerContainerFactory.<BR/>
 *
 * @return KafkaListenerContainerFactory
 */
@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
    log.debug("kafkaListenerContainerFactory()");
    ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConsumerFactory(consumerFactory());
    factory.setConcurrency(Settings2.HANDLER_COUNT);
    factory.getContainerProperties().setPollTimeout(Settings2.getPollTimeout());
    return factory;
}
 
Example #9
Source File: ReceiverConfig.java    From spring-kafka with MIT License 5 votes vote down vote up
@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
  ConcurrentKafkaListenerContainerFactory<String, String> factory =
      new ConcurrentKafkaListenerContainerFactory<>();
  factory.setConsumerFactory(consumerFactory());

  return factory;
}
 
Example #10
Source File: ReceiverConfig.java    From spring-kafka with MIT License 5 votes vote down vote up
@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
  ConcurrentKafkaListenerContainerFactory<String, String> factory =
      new ConcurrentKafkaListenerContainerFactory<>();
  factory.setConsumerFactory(consumerFactory());

  return factory;
}
 
Example #11
Source File: KafkaChannelDefinitionProcessor.java    From flowable-engine with Apache License 2.0 5 votes vote down vote up
/**
 * Register a new {@link KafkaListenerEndpoint} alongside the
 * {@link KafkaListenerContainerFactory} to use to create the underlying container.
 * <p>The {@code factory} may be {@code null} if the default factory has to be
 * used for that endpoint.
 */
protected void registerEndpoint(KafkaListenerEndpoint endpoint, KafkaListenerContainerFactory<?> factory) {
    Assert.notNull(endpoint, "Endpoint must not be null");
    Assert.hasText(endpoint.getId(), "Endpoint id must be set");

    Assert.state(this.endpointRegistry != null, "No KafkaListenerEndpointRegistry set");
    endpointRegistry.registerListenerContainer(endpoint, resolveContainerFactory(endpoint, factory), true);
}
 
Example #12
Source File: AConsumerConfig.java    From SO with BSD 2-Clause "Simplified" License 5 votes vote down vote up
/**
 * return KafkaListenerContainerFactory.<BR/>
 *
 * @return KafkaListenerContainerFactory
 */
@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
    log.debug("kafkaListenerContainerFactory()");
    ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConsumerFactory(consumerFactory());
    factory.setConcurrency(Settings2.HANDLER_COUNT);
    factory.getContainerProperties().setPollTimeout(Settings2.getPollTimeout());
    return factory;
}
 
Example #13
Source File: KafkaConfig.java    From gae with MIT License 5 votes vote down vote up
@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
    ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConsumerFactory(consumerFactory());
    factory.setConcurrency(3);
    factory.getContainerProperties().setPollTimeout(3000);
    return factory;
}
 
Example #14
Source File: KafkaConsumerConfig.java    From springboot_cwenao with MIT License 5 votes vote down vote up
@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {

    ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<String, String>();
    factory.setConsumerFactory(consumerFactory());
    factory.setConcurrency(4);
    factory.getContainerProperties().setPollTimeout(4000);

    return factory;
}
 
Example #15
Source File: ProductConsumerConfiguration.java    From integration-patterns with MIT License 5 votes vote down vote up
@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
    ConcurrentKafkaListenerContainerFactory<String, String> factory =
            new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConsumerFactory(consumerFactory());
    factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL_IMMEDIATE);

    return factory;
}
 
Example #16
Source File: KafkaConsumerConfig.java    From javatech with Creative Commons Attribution Share Alike 4.0 International 5 votes vote down vote up
@Bean(name = "kafkaListenerContainerFactory1")
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory1() {
	ConcurrentKafkaListenerContainerFactory<String, String> factory =
		new ConcurrentKafkaListenerContainerFactory<>();
	factory.setConsumerFactory(consumerFactory("groupB"));
	factory.setConcurrency(3);
	factory.getContainerProperties().setPollTimeout(3000);
	return factory;
}
 
Example #17
Source File: KafkaConsumerConfig.java    From javatech with Creative Commons Attribution Share Alike 4.0 International 5 votes vote down vote up
@Bean(name = "kafkaListenerContainerFactory")
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
	ConcurrentKafkaListenerContainerFactory<String, String> factory =
		new ConcurrentKafkaListenerContainerFactory<>();
	factory.setConsumerFactory(consumerFactory("groupA"));
	factory.setConcurrency(3);
	factory.getContainerProperties().setPollTimeout(3000);
	return factory;
}
 
Example #18
Source File: KafkaConsumerConfiguration.java    From ZTuoExchange_framework with MIT License 5 votes vote down vote up
@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
	ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
	factory.setConsumerFactory(consumerFactory());
	factory.setConcurrency(concurrency);
	factory.getContainerProperties().setPollTimeout(1500);
	factory.setBatchListener(true);
	return factory;
}
 
Example #19
Source File: KafkaConsumerConfig.java    From kafka-with-springboot with Apache License 2.0 5 votes vote down vote up
@Bean(name = "kafkaListenerContainerFactoryForBatchConsumer")
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactoryForBatchConsumer() {
    ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConcurrency(1);
    factory.setBatchListener(true);
    factory.setConsumerFactory(consumerFactory());
    return factory;
}
 
Example #20
Source File: KafkaConsumerConfig.java    From kafka-with-springboot with Apache License 2.0 5 votes vote down vote up
@Bean(name = "kafkaListenerContainerFactoryWith6Consumer")
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactoryWith3Consumer() {
    ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConcurrency(6); //3 partition -> 6 thread in parallel in a single consumer
    factory.setConsumerFactory(consumerFactory());
    return factory;
}
 
Example #21
Source File: KafkaConsumerConfig.java    From kafka-with-springboot with Apache License 2.0 5 votes vote down vote up
@Bean
@Primary
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
    ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConsumerFactory(consumerFactory());

    return factory;
}
 
Example #22
Source File: KafkaConsumerConfig.java    From flink-statefun with Apache License 2.0 5 votes vote down vote up
@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>>
    kafkaListenerContainerFactory(ConsumerFactory<String, String> consumerFactory) {
  ConcurrentKafkaListenerContainerFactory<String, String> factory =
      new ConcurrentKafkaListenerContainerFactory<>();
  factory.setConsumerFactory(consumerFactory);
  return factory;
}
 
Example #23
Source File: KafkaConsumerConfiguration.java    From ZTuoExchange_framework with MIT License 5 votes vote down vote up
@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
	ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
	factory.setConsumerFactory(consumerFactory());
	factory.setConcurrency(concurrency);
	factory.getContainerProperties().setPollTimeout(1500);
	factory.setBatchListener(true);
	return factory;
}
 
Example #24
Source File: KafkaConsumerConfig.java    From ext-opensource-netty with Mozilla Public License 2.0 5 votes vote down vote up
@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
    ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConsumerFactory(consumerFactory());
    factory.getContainerProperties().setPollTimeout(1500);
    return factory;
}
 
Example #25
Source File: KafkaConsumerConfiguration.java    From ZTuoExchange_framework with MIT License 5 votes vote down vote up
@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
	ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
	factory.setConsumerFactory(consumerFactory());
	factory.setConcurrency(concurrency);
	factory.getContainerProperties().setPollTimeout(1500);
	factory.setBatchListener(true);
	return factory;
}
 
Example #26
Source File: KafkaMessageBrokerConfiguration.java    From piper with Apache License 2.0 4 votes vote down vote up
private KafkaListenerContainerFactory createContainerFactory(int aConcurrency) {
  ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory();
  factory.setConcurrency(aConcurrency);

  return factory;
}
 
Example #27
Source File: KafkaChannelDefinitionProcessor.java    From flowable-engine with Apache License 2.0 4 votes vote down vote up
public KafkaListenerContainerFactory<?> getContainerFactory() {
    return containerFactory;
}
 
Example #28
Source File: KafkaChannelDefinitionProcessor.java    From flowable-engine with Apache License 2.0 4 votes vote down vote up
public void setContainerFactory(KafkaListenerContainerFactory<?> containerFactory) {
    this.containerFactory = containerFactory;
}
 
Example #29
Source File: BinlogConsumer.java    From ad with Apache License 2.0 4 votes vote down vote up
public BinlogConsumer(ISender iSender, KafkaListenerContainerFactory containerFactory) {
    this.iSender = iSender;
    this.containerFactory = containerFactory;
}