org.springframework.kafka.listener.ContainerProperties Java Examples

The following examples show how to use org.springframework.kafka.listener.ContainerProperties. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: StubRunnerKafkaConfiguration.java    From spring-cloud-contract with Apache License 2.0 6 votes vote down vote up
private void registerContainers(ConfigurableListableBeanFactory beanFactory,
		List<Contract> matchingContracts, String flowName,
		StubRunnerKafkaRouter listener) {
	// listener's container
	ConsumerFactory consumerFactory = beanFactory.getBean(ConsumerFactory.class);
	for (Contract matchingContract : matchingContracts) {
		if (matchingContract.getInput() == null) {
			continue;
		}
		String destination = MapConverter.getStubSideValuesForNonBody(
				matchingContract.getInput().getMessageFrom()).toString();
		ContainerProperties containerProperties = new ContainerProperties(
				destination);
		KafkaMessageListenerContainer container = listenerContainer(consumerFactory,
				containerProperties, listener);
		String containerName = flowName + ".container";
		Object initializedContainer = beanFactory.initializeBean(container,
				containerName);
		beanFactory.registerSingleton(containerName, initializedContainer);
		if (log.isDebugEnabled()) {
			log.debug(
					"Initialized kafka message container with name [" + containerName
							+ "] listening to destination [" + destination + "]");
		}
	}
}
 
Example #2
Source File: KafKaConfig.java    From gpmall with Apache License 2.0 6 votes vote down vote up
/**
 * 消费则的监听工厂
 * @return
 */
@Bean
public KafkaListenerContainerFactory userRegisterSuccKafkaListenerContainerFactory(){
    ConcurrentKafkaListenerContainerFactory conFactory = new ConcurrentKafkaListenerContainerFactory<>();
    conFactory.setConsumerFactory(kafKaRegisterSuccConsumerFactory());
    conFactory.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL);//  设置消费者消费消息后的提交方式为手动提交
    return conFactory;
}
 
Example #3
Source File: SpringKafkaTest.java    From spring-boot-tutorial with Creative Commons Attribution Share Alike 4.0 International 6 votes vote down vote up
@Test
public void test() throws InterruptedException {
    log.info("Start auto");
    ContainerProperties containerProps = new ContainerProperties("topic1", "topic2");
    final CountDownLatch latch = new CountDownLatch(4);
    containerProps.setMessageListener((MessageListener<Integer, String>) message -> {
        log.info("received: " + message);
        latch.countDown();
    });
    KafkaMessageListenerContainer<Integer, String> container = createContainer(containerProps);
    container.setBeanName("testAuto");
    container.start();
    Thread.sleep(1000); // wait a bit for the container to start
    KafkaTemplate<Integer, String> template = createTemplate();
    template.setDefaultTopic("zptest");
    template.sendDefault(0, "foo");
    template.sendDefault(2, "bar");
    template.sendDefault(0, "baz");
    template.sendDefault(2, "qux");
    template.flush();
    assertThat(latch.await(60, TimeUnit.SECONDS)).isTrue();
    container.stop();
    log.info("Stop auto");
}
 
Example #4
Source File: EventApisFactory.java    From eventapis with Apache License 2.0 6 votes vote down vote up
@Bean({"eventsKafkaListenerContainerFactory", "kafkaListenerContainerFactory"})
public ConcurrentKafkaListenerContainerFactory<String, PublishedEventWrapper> eventsKafkaListenerContainerFactory(
        EventMessageConverter eventMessageConverter, ConsumerFactory<String, PublishedEventWrapper> consumerFactory) {

    ConcurrentKafkaListenerContainerFactory<String, PublishedEventWrapper> factory = new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConsumerFactory(consumerFactory);
    factory.setConcurrency(eventApisConfiguration.getEventBus().getConsumer().getEventConcurrency());
    factory.setMessageConverter(eventMessageConverter);
    factory.getContainerProperties().setPollTimeout(3000);
    ThreadPoolTaskScheduler scheduler = new ThreadPoolTaskScheduler();
    scheduler.setPoolSize(eventApisConfiguration.getEventBus().getConsumer().getEventSchedulerPoolSize());
    scheduler.setBeanName("EventsFactory-Scheduler");
    scheduler.initialize();

    factory.getContainerProperties().setScheduler(scheduler);
    factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.RECORD);
    return factory;
}
 
Example #5
Source File: EnodeTestKafkaConfig.java    From enode with MIT License 5 votes vote down vote up
@Bean
public KafkaMessageListenerContainer<String, String> domainEventListenerContainer(KafkaDomainEventListener domainEventListener, ConsumerFactory<String, String> consumerFactory) {
    ContainerProperties properties = new ContainerProperties(eventTopic);
    properties.setGroupId(Constants.DEFAULT_PRODUCER_GROUP);
    properties.setMessageListener(domainEventListener);
    properties.setMissingTopicsFatal(false);
    properties.setAckMode(ContainerProperties.AckMode.MANUAL);
    return new KafkaMessageListenerContainer<>(consumerFactory, properties);
}
 
Example #6
Source File: EventRegistryKafkaConfiguration.java    From flowable-engine with Apache License 2.0 5 votes vote down vote up
@Bean(name = "kafkaListenerContainerFactory")
public ConcurrentKafkaListenerContainerFactory<?, ?> kafkaListenerContainerFactory(ConsumerFactory<Object, Object> kafkaConsumerFactory) {
    ConcurrentKafkaListenerContainerFactory<Object, Object> factory = new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConsumerFactory(kafkaConsumerFactory);

    ContainerProperties containerProperties = factory.getContainerProperties();
    containerProperties.setMissingTopicsFatal(true);
    return factory;
}
 
Example #7
Source File: StubRunnerKafkaConfiguration.java    From spring-cloud-contract with Apache License 2.0 5 votes vote down vote up
private KafkaMessageListenerContainer listenerContainer(
		ConsumerFactory consumerFactory, ContainerProperties containerProperties,
		GenericMessageListener listener) {
	KafkaMessageListenerContainer container = new KafkaMessageListenerContainer(
			consumerFactory, containerProperties);
	container.setupMessageListener(listener);
	return container;
}
 
Example #8
Source File: KafkaConfig.java    From spring-boot-demo with MIT License 5 votes vote down vote up
@Bean("ackContainerFactory")
public ConcurrentKafkaListenerContainerFactory<String, String> ackContainerFactory() {
    ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConsumerFactory(consumerFactory());
    factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL_IMMEDIATE);
    factory.setConcurrency(KafkaConsts.DEFAULT_PARTITION_NUM);
    return factory;
}
 
Example #9
Source File: LocKafkaAutoConfiguration.java    From loc-framework with MIT License 5 votes vote down vote up
@Bean(name = "kafkaListenerContainerFactory")
public ConcurrentKafkaListenerContainerFactory<?, ?> kafkaListenerContainerFactory(
    ConcurrentKafkaListenerContainerFactoryConfigurer configurer,
    ConsumerFactory<Object, Object> kafkaConsumerFactory) {
  ConcurrentKafkaListenerContainerFactory<Object, Object> factory = new ConcurrentKafkaListenerContainerFactory<>();
  ContainerProperties containerProperties = factory.getContainerProperties();
  factory.setRecordFilterStrategy(locMessageFilterStrategy());
  factory.setErrorHandler(new LocKafkaConsumerErrorHandler());
  factory.setMessageConverter(recordMessageConverter());
  configurer.configure(factory, kafkaConsumerFactory);
  return factory;
}
 
Example #10
Source File: ProductConsumerConfiguration.java    From integration-patterns with MIT License 5 votes vote down vote up
@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
    ConcurrentKafkaListenerContainerFactory<String, String> factory =
            new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConsumerFactory(consumerFactory());
    factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL_IMMEDIATE);

    return factory;
}
 
Example #11
Source File: EventApisFactory.java    From eventapis with Apache License 2.0 5 votes vote down vote up
@Bean("operationsKafkaListenerContainerFactory")
    public ConcurrentKafkaListenerContainerFactory<String, Operation> operationsKafkaListenerContainerFactory(
            ConsumerFactory<String, Operation> consumerFactory) {
        ConcurrentKafkaListenerContainerFactory<String, Operation> factory
                = new ConcurrentKafkaListenerContainerFactory<>();
        factory.setConsumerFactory(consumerFactory);
        RetryTemplate retryTemplate = new RetryTemplate();
        factory.setRetryTemplate(retryTemplate);

        factory.setConcurrency(eventApisConfiguration.getEventBus().getConsumer().getOperationSchedulerPoolSize());
        ThreadPoolTaskScheduler scheduler = new ThreadPoolTaskScheduler();
        scheduler.setPoolSize(eventApisConfiguration.getEventBus().getConsumer().getOperationSchedulerPoolSize());
        scheduler.setBeanName("OperationsFactory-Scheduler");
        scheduler.initialize();
        factory.getContainerProperties().setScheduler(scheduler);
        ThreadPoolTaskScheduler consumerScheduler = new ThreadPoolTaskScheduler();
        consumerScheduler.setPoolSize(eventApisConfiguration.getEventBus().getConsumer().getOperationSchedulerPoolSize());
        consumerScheduler.setBeanName("OperationsFactory-ConsumerScheduler");
        consumerScheduler.initialize();

        factory.getContainerProperties().setPollTimeout(3000L);
        factory.getContainerProperties().setAckOnError(false);
        factory.getContainerProperties().setConsumerTaskExecutor(consumerScheduler);
        factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.RECORD);
        /**
         * This is Fix for Spring Kafka versions which does not have ConsumerAwareErrorHandler handler till 2.0
         * When Listener faces with error, it retries snapshot operation
         * See https://github.com/kloiasoft/eventapis/issues/44
         */
        factory.getContainerProperties().setTransactionManager(new EmptyTransactionManager());
//        factory.getContainerProperties().setTransactionManager(platformTransactionManager);
        return factory;
    }
 
Example #12
Source File: SpringKafkaTest.java    From spring-boot-tutorial with Creative Commons Attribution Share Alike 4.0 International 5 votes vote down vote up
private KafkaMessageListenerContainer<Integer, String> createContainer(
    ContainerProperties containerProps) {
    Map<String, Object> props = consumerProps();
    DefaultKafkaConsumerFactory<Integer, String> cf =
        new DefaultKafkaConsumerFactory<>(props);
    KafkaMessageListenerContainer<Integer, String> container =
        new KafkaMessageListenerContainer<>(cf, containerProps);
    return container;
}
 
Example #13
Source File: KafkaEventConfig.java    From enode with MIT License 5 votes vote down vote up
@Bean
public KafkaMessageListenerContainer publishableExceptionListenerContainer(KafkaPublishableExceptionListener publishableExceptionListener, RetryTemplate retryTemplate) {
    ContainerProperties properties = new ContainerProperties(QueueProperties.EXCEPTION_TOPIC);
    properties.setGroupId(QueueProperties.DEFAULT_PRODUCER_GROUP);
    RetryingMessageListenerAdapter listenerAdapter = new RetryingMessageListenerAdapter(publishableExceptionListener, retryTemplate);
    properties.setMessageListener(listenerAdapter);
    properties.setMissingTopicsFatal(false);
    properties.setAckMode(ContainerProperties.AckMode.MANUAL);
    return new KafkaMessageListenerContainer<>(consumerFactory(), properties);
}
 
Example #14
Source File: KafkaEventConfig.java    From enode with MIT License 5 votes vote down vote up
@Bean
public KafkaMessageListenerContainer applicationMessageListenerContainer(KafkaApplicationMessageListener applicationMessageListener, RetryTemplate retryTemplate) {
    ContainerProperties properties = new ContainerProperties(QueueProperties.APPLICATION_TOPIC);
    properties.setGroupId(QueueProperties.DEFAULT_PRODUCER_GROUP);
    RetryingMessageListenerAdapter listenerAdapter = new RetryingMessageListenerAdapter(applicationMessageListener, retryTemplate);
    properties.setMessageListener(listenerAdapter);
    properties.setMissingTopicsFatal(false);
    properties.setAckMode(ContainerProperties.AckMode.MANUAL);
    return new KafkaMessageListenerContainer<>(consumerFactory(), properties);
}
 
Example #15
Source File: KafkaEventConfig.java    From enode with MIT License 5 votes vote down vote up
@Bean
public KafkaMessageListenerContainer domainEventListenerContainer(KafkaDomainEventListener domainEventListener, RetryTemplate retryTemplate) {
    ContainerProperties properties = new ContainerProperties(QueueProperties.EVENT_TOPIC);
    properties.setGroupId(QueueProperties.DEFAULT_PRODUCER_GROUP);
    RetryingMessageListenerAdapter listenerAdapter = new RetryingMessageListenerAdapter(domainEventListener, retryTemplate);
    properties.setMessageListener(listenerAdapter);
    properties.setMissingTopicsFatal(false);
    properties.setAckMode(ContainerProperties.AckMode.MANUAL);
    return new KafkaMessageListenerContainer<>(consumerFactory(), properties);
}
 
Example #16
Source File: KafkaCommandConfig.java    From enode with MIT License 5 votes vote down vote up
@Bean
public KafkaMessageListenerContainer kafkaMessageListenerContainer(KafkaCommandListener commandListener, RetryTemplate retryTemplate) {
    ContainerProperties properties = new ContainerProperties(COMMAND_TOPIC);
    properties.setGroupId(DEFAULT_CONSUMER_GROUP);
    RetryingMessageListenerAdapter listenerAdapter = new RetryingMessageListenerAdapter(commandListener, retryTemplate);
    properties.setMessageListener(listenerAdapter);
    properties.setMissingTopicsFatal(false);
    properties.setAckMode(ContainerProperties.AckMode.MANUAL);
    return new KafkaMessageListenerContainer<>(consumerFactory(), properties);
}
 
Example #17
Source File: EnodeTestKafkaConfig.java    From enode with MIT License 5 votes vote down vote up
@Bean
public KafkaMessageListenerContainer<String, String> publishableExceptionListenerContainer(KafkaPublishableExceptionListener publishableExceptionListener, ConsumerFactory<String, String> consumerFactory) {
    ContainerProperties properties = new ContainerProperties(exceptionTopic);
    properties.setGroupId(Constants.DEFAULT_PRODUCER_GROUP);
    properties.setMessageListener(publishableExceptionListener);
    properties.setMissingTopicsFatal(false);
    properties.setAckMode(ContainerProperties.AckMode.MANUAL);
    return new KafkaMessageListenerContainer<>(consumerFactory, properties);
}
 
Example #18
Source File: EnodeTestKafkaConfig.java    From enode with MIT License 5 votes vote down vote up
@Bean
public KafkaMessageListenerContainer<String, String> applicationMessageListenerContainer(KafkaApplicationMessageListener applicationMessageListener, ConsumerFactory<String, String> consumerFactory) {
    ContainerProperties properties = new ContainerProperties(applicationTopic);
    properties.setGroupId(Constants.DEFAULT_PRODUCER_GROUP);
    properties.setMessageListener(applicationMessageListener);
    properties.setMissingTopicsFatal(false);
    properties.setAckMode(ContainerProperties.AckMode.MANUAL);
    return new KafkaMessageListenerContainer<>(consumerFactory, properties);
}
 
Example #19
Source File: EnodeTestKafkaConfig.java    From enode with MIT License 5 votes vote down vote up
@Bean
public KafkaMessageListenerContainer<String, String> commandListenerContainer(KafkaCommandListener commandListener, ConsumerFactory<String, String> consumerFactory) {
    ContainerProperties properties = new ContainerProperties(commandTopic);
    properties.setGroupId(Constants.DEFAULT_CONSUMER_GROUP);
    properties.setMessageListener(commandListener);
    properties.setMissingTopicsFatal(false);
    return new KafkaMessageListenerContainer<>(consumerFactory, properties);
}
 
Example #20
Source File: ReplyConfiguration.java    From faster-framework-project with Apache License 2.0 4 votes vote down vote up
/**
 * @return replyContainer
 */
public KafkaMessageListenerContainer<Object, Object> replyContainer() {
    ContainerProperties containerProperties = new ContainerProperties(properties.getProducer().getReplyTopic());
    return new KafkaMessageListenerContainer<>(consumerFactory, containerProperties);
}
 
Example #21
Source File: KafkaBinderTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 4 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testManualAckIsNotPossibleWhenAutoCommitOffsetIsEnabledOnTheBinder()
		throws Exception {
	Binder binder = getBinder();

	DirectChannel moduleOutputChannel = createBindableChannel("output",
			createProducerBindingProperties(createProducerProperties()));
	QueueChannel moduleInputChannel = new QueueChannel();

	Binding<MessageChannel> producerBinding = binder.bindProducer(
			"testManualAckIsNotPossibleWhenAutoCommitOffsetIsEnabledOnTheBinder",
			moduleOutputChannel, createProducerProperties());

	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();

	Binding<MessageChannel> consumerBinding = binder.bindConsumer(
			"testManualAckIsNotPossibleWhenAutoCommitOffsetIsEnabledOnTheBinder",
			"test", moduleInputChannel, consumerProperties);

	AbstractMessageListenerContainer<?, ?> container = TestUtils.getPropertyValue(
			consumerBinding, "lifecycle.messageListenerContainer",
			AbstractMessageListenerContainer.class);
	assertThat(container.getContainerProperties().getAckMode())
			.isEqualTo(ContainerProperties.AckMode.BATCH);

	String testPayload1 = "foo" + UUID.randomUUID().toString();
	Message<?> message1 = org.springframework.integration.support.MessageBuilder
			.withPayload(testPayload1.getBytes()).build();

	// Let the consumer actually bind to the producer before sending a msg
	binderBindUnbindLatency();
	moduleOutputChannel.send(message1);

	Message<?> receivedMessage = receive(moduleInputChannel);
	assertThat(receivedMessage).isNotNull();
	assertThat(receivedMessage.getHeaders().get(KafkaHeaders.ACKNOWLEDGMENT))
			.isNull();

	producerBinding.unbind();
	consumerBinding.unbind();
}
 
Example #22
Source File: KafkaBinderTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 4 votes vote down vote up
@Test
@Override
@SuppressWarnings("unchecked")
public void testTwoRequiredGroups() throws Exception {
	Binder binder = getBinder();
	ExtendedProducerProperties<KafkaProducerProperties> producerProperties = createProducerProperties();

	DirectChannel output = createBindableChannel("output",
			createProducerBindingProperties(producerProperties));

	String testDestination = "testDestination"
			+ UUID.randomUUID().toString().replace("-", "");

	producerProperties.setRequiredGroups("test1", "test2");
	Binding<MessageChannel> producerBinding = binder.bindProducer(testDestination,
			output, producerProperties);

	String testPayload = "foo-" + UUID.randomUUID().toString();
	output.send(new GenericMessage<>(testPayload.getBytes()));

	QueueChannel inbound1 = new QueueChannel();
	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	consumerProperties.getExtension().setAutoRebalanceEnabled(false);
	consumerProperties.getExtension().setAckEachRecord(true);
	Binding<MessageChannel> consumerBinding1 = binder.bindConsumer(testDestination,
			"test1", inbound1, consumerProperties);
	QueueChannel inbound2 = new QueueChannel();
	Binding<MessageChannel> consumerBinding2 = binder.bindConsumer(testDestination,
			"test2", inbound2, consumerProperties);

	AbstractMessageListenerContainer<?, ?> container = TestUtils.getPropertyValue(
			consumerBinding2, "lifecycle.messageListenerContainer",
			AbstractMessageListenerContainer.class);
	assertThat(container.getContainerProperties().getAckMode())
			.isEqualTo(ContainerProperties.AckMode.RECORD);

	Message<?> receivedMessage1 = receive(inbound1);
	assertThat(receivedMessage1).isNotNull();
	assertThat(new String((byte[]) receivedMessage1.getPayload(),
			StandardCharsets.UTF_8)).isEqualTo(testPayload);
	Message<?> receivedMessage2 = receive(inbound2);
	assertThat(receivedMessage2).isNotNull();
	assertThat(new String((byte[]) receivedMessage2.getPayload(),
			StandardCharsets.UTF_8)).isEqualTo(testPayload);

	consumerBinding1.unbind();
	consumerBinding2.unbind();
	producerBinding.unbind();
}
 
Example #23
Source File: KafkaBinderTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 4 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testAutoAddPartitionsDisabledSucceedsIfTopicPartitionedCorrectly()
		throws Throwable {
	Binding<?> binding = null;
	try {
		KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties();

		String testTopicName = "existing" + System.currentTimeMillis();
		invokeCreateTopic(testTopicName, 6, 1);
		configurationProperties.setAutoAddPartitions(false);
		Binder binder = getBinder(configurationProperties);
		ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();

		DirectChannel input = createBindableChannel("input",
				createConsumerBindingProperties(consumerProperties));

		// this consumer must consume from partition 2
		consumerProperties.setInstanceCount(3);
		consumerProperties.setInstanceIndex(2);
		consumerProperties.getExtension().setAutoRebalanceEnabled(false);

		binding = binder.bindConsumer(testTopicName, "test-x", input,
				consumerProperties);

		ContainerProperties containerProps = TestUtils.getPropertyValue(
				binding,
				"lifecycle.messageListenerContainer.containerProperties",
				ContainerProperties.class);
		TopicPartitionOffset[] listenedPartitions = containerProps.getTopicPartitionsToAssign();
		assertThat(listenedPartitions).hasSize(2);
		assertThat(listenedPartitions).contains(
				new TopicPartitionOffset(testTopicName, 2),
				new TopicPartitionOffset(testTopicName, 5));
		int partitions = invokePartitionSize(testTopicName);
		assertThat(partitions).isEqualTo(6);
	}
	finally {
		if (binding != null) {
			binding.unbind();
		}
	}
}
 
Example #24
Source File: SpringKafkaSenderTest.java    From spring-kafka with MIT License 4 votes vote down vote up
@Before
public void setUp() throws Exception {
  // set up the Kafka consumer properties
  Map<String, Object> consumerProperties =
      KafkaTestUtils.consumerProps("sender", "false",
          embeddedKafka.getEmbeddedKafka());

  // create a Kafka consumer factory
  DefaultKafkaConsumerFactory<String, String> consumerFactory =
      new DefaultKafkaConsumerFactory<String, String>(
          consumerProperties);

  // set the topic that needs to be consumed
  ContainerProperties containerProperties =
      new ContainerProperties(SENDER_TOPIC);

  // create a Kafka MessageListenerContainer
  container = new KafkaMessageListenerContainer<>(consumerFactory,
      containerProperties);

  // create a thread safe queue to store the received message
  records = new LinkedBlockingQueue<>();

  // setup a Kafka message listener
  container
      .setupMessageListener(new MessageListener<String, String>() {
        @Override
        public void onMessage(
            ConsumerRecord<String, String> record) {
          LOGGER.debug("test-listener received message='{}'",
              record.toString());
          records.add(record);
        }
      });

  // start the container and underlying message listener
  container.start();

  // wait until the container has the required number of assigned partitions
  ContainerTestUtils.waitForAssignment(container,
      embeddedKafka.getEmbeddedKafka().getPartitionsPerTopic());
}