org.springframework.cloud.stream.binder.ExtendedConsumerProperties Java Examples

The following examples show how to use org.springframework.cloud.stream.binder.ExtendedConsumerProperties. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RabbitBinderTests.java    From spring-cloud-stream-binder-rabbit with Apache License 2.0 6 votes vote down vote up
@Test
public void testMultiplexOnPartitionedConsumer() throws Exception {
	final ExtendedConsumerProperties<RabbitConsumerProperties> consumerProperties = createConsumerProperties();
	RabbitTestSupport.RabbitProxy proxy = new RabbitTestSupport.RabbitProxy();
	CachingConnectionFactory cf = new CachingConnectionFactory("localhost",
			proxy.getPort());

	final RabbitExchangeQueueProvisioner rabbitExchangeQueueProvisioner = new RabbitExchangeQueueProvisioner(cf);

	consumerProperties.setMultiplex(true);
	consumerProperties.setPartitioned(true);
	consumerProperties.setInstanceIndexList(Arrays.asList(1, 2, 3));

	final ConsumerDestination consumerDestination = rabbitExchangeQueueProvisioner.provisionConsumerDestination("foo", "boo", consumerProperties);

	final String name = consumerDestination.getName();

	assertThat(name).isEqualTo("foo.boo-1,foo.boo-2,foo.boo-3");
}
 
Example #2
Source File: KafkaMessageChannelBinder.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 6 votes vote down vote up
private TopicPartitionOffset[] getTopicPartitionOffsets(
		Collection<PartitionInfo> listenedPartitions,
		ExtendedConsumerProperties<KafkaConsumerProperties> extendedConsumerProperties,
		ConsumerFactory<?, ?> consumerFactory) {

	final TopicPartitionOffset[] TopicPartitionOffsets =
			new TopicPartitionOffset[listenedPartitions.size()];
	int i = 0;
	SeekPosition seekPosition = null;
	Object resetTo = checkReset(extendedConsumerProperties.getExtension().isResetOffsets(),
			consumerFactory.getConfigurationProperties().get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG));
	if (resetTo != null) {
		seekPosition = "earliest".equals(resetTo) ? SeekPosition.BEGINNING : SeekPosition.END;
	}
	for (PartitionInfo partition : listenedPartitions) {

		TopicPartitionOffsets[i++] = new TopicPartitionOffset(
				partition.topic(), partition.partition(), seekPosition);
	}
	return TopicPartitionOffsets;
}
 
Example #3
Source File: RabbitMessageChannelBinder.java    From spring-cloud-stream-binder-rabbit with Apache License 2.0 6 votes vote down vote up
private void setSMLCProperties(
		ExtendedConsumerProperties<RabbitConsumerProperties> properties,
		SimpleMessageListenerContainer listenerContainer, int concurrency) {

	listenerContainer.setConcurrentConsumers(concurrency);
	int maxConcurrency = properties.getExtension().getMaxConcurrency();
	if (maxConcurrency > concurrency) {
		listenerContainer.setMaxConcurrentConsumers(maxConcurrency);
	}
	listenerContainer.setDeBatchingEnabled(!properties.isBatchMode());
	listenerContainer.setBatchSize(properties.getExtension().getBatchSize());
	if (properties.getExtension().getQueueDeclarationRetries() != null) {
		listenerContainer.setDeclarationRetries(
				properties.getExtension().getQueueDeclarationRetries());
	}
}
 
Example #4
Source File: KStreamBinder.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 6 votes vote down vote up
@Override
protected Binding<KStream<Object, Object>> doBindConsumer(String name, String group,
		KStream<Object, Object> inputTarget,
		ExtendedConsumerProperties<KafkaStreamsConsumerProperties> properties) {

	KStream<Object, Object> delegate = ((KStreamBoundElementFactory.KStreamWrapperHandler)
			((Advised) inputTarget).getAdvisors()[0].getAdvice()).getDelegate();

	this.kafkaStreamsBindingInformationCatalogue.registerConsumerProperties(delegate, properties.getExtension());

	if (!StringUtils.hasText(group)) {
		group = properties.getExtension().getApplicationId();
	}
	KafkaStreamsBinderUtils.prepareConsumerBinding(name, group,
			getApplicationContext(), this.kafkaTopicProvisioner,
			this.binderConfigurationProperties, properties);

	return new DefaultBinding<>(name, group, inputTarget, null);
}
 
Example #5
Source File: RabbitBinderTests.java    From spring-cloud-stream-binder-rabbit with Apache License 2.0 6 votes vote down vote up
@Test
public void testAnonWithBuiltInExchangeCustomPrefix() throws Exception {
	RabbitTestBinder binder = getBinder();
	ExtendedConsumerProperties<RabbitConsumerProperties> properties = createConsumerProperties();
	properties.getExtension().setDeclareExchange(false);
	properties.getExtension().setQueueNameGroupOnly(true);
	properties.getExtension().setAnonymousGroupPrefix("customPrefix.");

	Binding<MessageChannel> consumerBinding = binder.bindConsumer("amq.topic", null,
			createBindableChannel("input", new BindingProperties()), properties);
	Lifecycle endpoint = extractEndpoint(consumerBinding);
	SimpleMessageListenerContainer container = TestUtils.getPropertyValue(endpoint,
			"messageListenerContainer", SimpleMessageListenerContainer.class);
	String queueName = container.getQueueNames()[0];
	assertThat(queueName).startsWith("customPrefix.");
	assertThat(container.isRunning()).isTrue();
	consumerBinding.unbind();
	assertThat(container.isRunning()).isFalse();
}
 
Example #6
Source File: KafkaMessageChannelBinder.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 6 votes vote down vote up
/**
 * Configure a {@link BackOff} for the after rollback processor, based on the consumer
 * retry properties. If retry is disabled, return a {@link BackOff} that disables
 * retry. Otherwise calculate the {@link ExponentialBackOff#setMaxElapsedTime(long)}
 * so that the {@link BackOff} stops after the configured
 * {@link ExtendedConsumerProperties#getMaxAttempts()}.
 * @param extendedConsumerProperties the properties.
 * @return the backoff.
 */
private BackOff createBackOff(
		final ExtendedConsumerProperties<KafkaConsumerProperties> extendedConsumerProperties) {

	int maxAttempts = extendedConsumerProperties.getMaxAttempts();
	if (maxAttempts < 2) {
		return new FixedBackOff(0L, 0L);
	}
	int initialInterval = extendedConsumerProperties.getBackOffInitialInterval();
	double multiplier = extendedConsumerProperties.getBackOffMultiplier();
	int maxInterval = extendedConsumerProperties.getBackOffMaxInterval();
	ExponentialBackOff backOff = new ExponentialBackOff(initialInterval, multiplier);
	backOff.setMaxInterval(maxInterval);
	long maxElapsed = extendedConsumerProperties.getBackOffInitialInterval();
	double accum = maxElapsed;
	for (int i = 1; i < maxAttempts - 1; i++) {
		accum = accum * multiplier;
		if (accum > maxInterval) {
			accum = maxInterval;
		}
		maxElapsed += accum;
	}
	backOff.setMaxElapsedTime(maxElapsed);
	return backOff;
}
 
Example #7
Source File: RepublishUnitTests.java    From spring-cloud-stream-binder-rabbit with Apache License 2.0 6 votes vote down vote up
@Test
public void testBadRepublishSetting() throws IOException {
	ConnectionFactory cf = mock(ConnectionFactory.class);
	Connection conn = mock(Connection.class);
	given(cf.createConnection()).willReturn(conn);
	Channel channel = mock(Channel.class);
	given(channel.isOpen()).willReturn(true);
	given(channel.exchangeDeclarePassive("DLX")).willThrow(new IOException());
	given(conn.createChannel(false)).willReturn(channel);
	RabbitProperties props = new RabbitProperties();
	RabbitMessageChannelBinder binder = new RabbitMessageChannelBinder(cf, props, null);
	RabbitConsumerProperties extension = new RabbitConsumerProperties();
	ExtendedConsumerProperties<RabbitConsumerProperties> bindingProps =
			new ExtendedConsumerProperties<RabbitConsumerProperties>(extension);
	MessageHandler handler = binder.getErrorMessageHandler(mock(ConsumerDestination.class), "foo", bindingProps);
	ErrorMessage message = new ErrorMessage(new RuntimeException("test"),
			Collections.singletonMap(IntegrationMessageHeaderAccessor.SOURCE_DATA,
					new Message("foo".getBytes(), new MessageProperties())));
	handler.handleMessage(message);
	handler.handleMessage(message);
	verify(channel, times(1)).exchangeDeclarePassive("DLX");
	verify(channel, never()).basicPublish(any(), any(), eq(false), any(), any());
}
 
Example #8
Source File: KafkaBinderTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 6 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testPartitionCountIncreasedIfAutoAddPartitionsSet() throws Throwable {
	KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties();

	String testTopicName = "existing" + System.currentTimeMillis();
	configurationProperties.setMinPartitionCount(6);
	configurationProperties.setAutoAddPartitions(true);
	Binder binder = getBinder(configurationProperties);
	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	DirectChannel input = createBindableChannel("input",
			createConsumerBindingProperties(consumerProperties));

	Binding<?> binding = binder.bindConsumer(testTopicName, "test", input,
			consumerProperties);
	binding.unbind();
	assertThat(invokePartitionSize(testTopicName)).isEqualTo(6);
}
 
Example #9
Source File: KafkaBinderTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 6 votes vote down vote up
@Test
public void testNoBrokerOverride() throws Exception {
	Binder binder = getBinder();
	ExtendedProducerProperties<KafkaProducerProperties> producerProperties = createProducerProperties();
	producerProperties.getExtension().getConfiguration().put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "foo");
	BindingProperties outputBindingProperties = createProducerBindingProperties(producerProperties);
	DirectChannel moduleOutputChannel = createBindableChannel("output", outputBindingProperties);
	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	consumerProperties.getExtension().getConfiguration().put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "foo");
	BindingProperties consumerBindingProperties = createConsumerBindingProperties(consumerProperties);
	DirectChannel moduleInputChannel = createBindableChannel("input", consumerBindingProperties);

	assertThatExceptionOfType(BinderException.class).isThrownBy(() -> binder.bindProducer("foo.bar",
				moduleOutputChannel, outputBindingProperties.getProducer()))
			.withCauseExactlyInstanceOf(IllegalStateException.class);
	assertThatExceptionOfType(BinderException.class).isThrownBy(() -> binder.bindConsumer("foo.bar",
				"testSendAndReceive", moduleInputChannel, consumerProperties))
			.withCauseExactlyInstanceOf(IllegalStateException.class);
}
 
Example #10
Source File: KafkaBinderTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 6 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testAutoAddPartitionsDisabledSucceedsIfTopicUnderPartitionedAndAutoRebalanceEnabled()
		throws Throwable {
	KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties();

	String testTopicName = "existing" + System.currentTimeMillis();
	invokeCreateTopic(testTopicName, 1, 1);
	configurationProperties.setAutoAddPartitions(false);
	Binder binder = getBinder(configurationProperties);
	GenericApplicationContext context = new GenericApplicationContext();
	context.refresh();

	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();

	DirectChannel input = createBindableChannel("input",
			createConsumerBindingProperties(consumerProperties));

	// this consumer must consume from partition 2
	consumerProperties.setInstanceCount(3);
	consumerProperties.setInstanceIndex(2);
	Binding binding = binder.bindConsumer(testTopicName, "test", input,
			consumerProperties);
	binding.unbind();
	assertThat(invokePartitionSize(testTopicName)).isEqualTo(1);
}
 
Example #11
Source File: RabbitMessageChannelBinder.java    From spring-cloud-stream-binder-rabbit with Apache License 2.0 6 votes vote down vote up
private void setDMLCProperties(
		ExtendedConsumerProperties<RabbitConsumerProperties> properties,
		DirectMessageListenerContainer listenerContainer, int concurrency) {

	listenerContainer.setConsumersPerQueue(concurrency);
	if (properties.getExtension().getMaxConcurrency() > concurrency) {
		this.logger
				.warn("maxConcurrency is not supported by the direct container type");
	}
	if (properties.getExtension().getBatchSize() > 1) {
		this.logger.warn("batchSize is not supported by the direct container type");
	}
	if (properties.getExtension().getQueueDeclarationRetries() != null) {
		this.logger.warn(
				"queueDeclarationRetries is not supported by the direct container type");
	}
}
 
Example #12
Source File: KafkaBinderTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 6 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testAutoConfigureTopicsDisabledSucceedsIfTopicExisting()
		throws Throwable {
	KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties();

	String testTopicName = "existing" + System.currentTimeMillis();
	invokeCreateTopic(testTopicName, 5, 1);
	configurationProperties.setAutoCreateTopics(false);
	Binder binder = getBinder(configurationProperties);

	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();

	DirectChannel input = createBindableChannel("input",
			createConsumerBindingProperties(consumerProperties));
	Binding<MessageChannel> binding = binder.bindConsumer(testTopicName, "test",
			input, consumerProperties);
	binding.unbind();
}
 
Example #13
Source File: KafkaBinderTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testDynamicKeyExpression() throws Exception {
	Binder binder = getBinder(createConfigurationProperties());
	QueueChannel moduleInputChannel = new QueueChannel();
	ExtendedProducerProperties<KafkaProducerProperties> producerProperties = createProducerProperties();
	producerProperties.getExtension().getConfiguration().put("key.serializer",
			StringSerializer.class.getName());
	producerProperties.getExtension().setMessageKeyExpression(
			spelExpressionParser.parseExpression("headers.key"));
	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	String uniqueBindingId = UUID.randomUUID().toString();
	DirectChannel moduleOutputChannel = createBindableChannel("output",
			createProducerBindingProperties(producerProperties));
	Binding<MessageChannel> producerBinding = binder.bindProducer(
			"foo" + uniqueBindingId + ".0", moduleOutputChannel, producerProperties);
	Binding<MessageChannel> consumerBinding = binder.bindConsumer(
			"foo" + uniqueBindingId + ".0", null, moduleInputChannel,
			consumerProperties);
	Thread.sleep(1000);
	Message<?> message = MessageBuilder.withPayload("somePayload")
			.setHeader("key", "myDynamicKey").build();
	// Let the consumer actually bind to the producer before sending a msg
	binderBindUnbindLatency();
	moduleOutputChannel.send(message);
	Message<?> inbound = receive(moduleInputChannel);
	assertThat(inbound).isNotNull();
	String receivedKey = new String(inbound.getHeaders()
			.get(KafkaHeaders.RECEIVED_MESSAGE_KEY, byte[].class));
	assertThat(receivedKey).isEqualTo("myDynamicKey");
	producerBinding.unbind();
	consumerBinding.unbind();
}
 
Example #14
Source File: RabbitTestBinder.java    From spring-cloud-stream-binder-rabbit with Apache License 2.0 5 votes vote down vote up
@Override
public Binding<PollableSource<MessageHandler>> bindPollableConsumer(String name,
		String group, PollableSource<MessageHandler> inboundBindTarget,
		ExtendedConsumerProperties<RabbitConsumerProperties> properties) {
	captureConsumerResources(name, group, properties);
	return super.bindPollableConsumer(name, group, inboundBindTarget, properties);
}
 
Example #15
Source File: KafkaBinderTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testCustomPartitionCountOverridesPartitioningIfLarger() throws Exception {
	byte[] testPayload = new byte[2048];
	Arrays.fill(testPayload, (byte) 65);
	KafkaBinderConfigurationProperties binderConfiguration = createConfigurationProperties();
	binderConfiguration.setMinPartitionCount(4);
	Binder binder = getBinder(binderConfiguration);

	QueueChannel moduleInputChannel = new QueueChannel();
	ExtendedProducerProperties<KafkaProducerProperties> producerProperties = createProducerProperties();
	producerProperties.setPartitionCount(5);
	producerProperties.setPartitionKeyExpression(
			spelExpressionParser.parseExpression("payload"));
	DirectChannel moduleOutputChannel = createBindableChannel("output",
			createProducerBindingProperties(producerProperties));
	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	long uniqueBindingId = System.currentTimeMillis();
	Binding<MessageChannel> producerBinding = binder.bindProducer(
			"foo" + uniqueBindingId + ".0", moduleOutputChannel, producerProperties);
	Binding<MessageChannel> consumerBinding = binder.bindConsumer(
			"foo" + uniqueBindingId + ".0", null, moduleInputChannel,
			consumerProperties);
	Message<?> message = org.springframework.integration.support.MessageBuilder
			.withPayload(testPayload).build();
	// Let the consumer actually bind to the producer before sending a msg
	binderBindUnbindLatency();
	moduleOutputChannel.send(message);
	Message<?> inbound = receive(moduleInputChannel);
	assertThat(inbound).isNotNull();
	assertThat((byte[]) inbound.getPayload()).containsExactly(testPayload);
	assertThat(partitionSize("foo" + uniqueBindingId + ".0")).isEqualTo(5);
	producerBinding.unbind();
	consumerBinding.unbind();
}
 
Example #16
Source File: KafkaMessageChannelBinder.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 5 votes vote down vote up
private MessagingMessageConverter getMessageConverter(
		final ExtendedConsumerProperties<KafkaConsumerProperties> extendedConsumerProperties) {
	MessagingMessageConverter messageConverter;
	if (extendedConsumerProperties.getExtension().getConverterBeanName() == null) {
		messageConverter = new MessagingMessageConverter();
		StandardHeaders standardHeaders = extendedConsumerProperties.getExtension()
				.getStandardHeaders();
		messageConverter
				.setGenerateMessageId(StandardHeaders.id.equals(standardHeaders)
						|| StandardHeaders.both.equals(standardHeaders));
		messageConverter.setGenerateTimestamp(
				StandardHeaders.timestamp.equals(standardHeaders)
						|| StandardHeaders.both.equals(standardHeaders));
	}
	else {
		try {
			messageConverter = getApplicationContext().getBean(
					extendedConsumerProperties.getExtension().getConverterBeanName(),
					MessagingMessageConverter.class);
		}
		catch (NoSuchBeanDefinitionException ex) {
			throw new IllegalStateException(
					"Converter bean not present in application context", ex);
		}
	}
	messageConverter.setHeaderMapper(getHeaderMapper(extendedConsumerProperties));
	return messageConverter;
}
 
Example #17
Source File: KafkaBinderTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testCustomPartitionCountDoesNotOverridePartitioningIfSmaller()
		throws Exception {
	byte[] testPayload = new byte[2048];
	Arrays.fill(testPayload, (byte) 65);
	KafkaBinderConfigurationProperties binderConfiguration = createConfigurationProperties();
	binderConfiguration.setMinPartitionCount(6);
	Binder binder = getBinder(binderConfiguration);
	QueueChannel moduleInputChannel = new QueueChannel();
	ExtendedProducerProperties<KafkaProducerProperties> producerProperties = createProducerProperties();
	producerProperties.setPartitionCount(5);
	producerProperties.setPartitionKeyExpression(
			spelExpressionParser.parseExpression("payload"));
	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	long uniqueBindingId = System.currentTimeMillis();
	DirectChannel moduleOutputChannel = createBindableChannel("output",
			createProducerBindingProperties(producerProperties));
	Binding<MessageChannel> producerBinding = binder.bindProducer(
			"foo" + uniqueBindingId + ".0", moduleOutputChannel, producerProperties);
	Binding<MessageChannel> consumerBinding = binder.bindConsumer(
			"foo" + uniqueBindingId + ".0", null, moduleInputChannel,
			consumerProperties);
	Thread.sleep(1000);
	Message<?> message = org.springframework.integration.support.MessageBuilder
			.withPayload(testPayload).build();
	// Let the consumer actually bind to the producer before sending a msg
	binderBindUnbindLatency();
	moduleOutputChannel.send(message);
	Message<?> inbound = receive(moduleInputChannel);
	assertThat(inbound).isNotNull();
	assertThat((byte[]) inbound.getPayload()).containsExactly(testPayload);

	assertThat(partitionSize("foo" + uniqueBindingId + ".0")).isEqualTo(6);
	producerBinding.unbind();
	consumerBinding.unbind();
}
 
Example #18
Source File: KafkaBinderTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testCustomPartitionCountOverridesDefaultIfLarger() throws Exception {
	byte[] testPayload = new byte[2048];
	Arrays.fill(testPayload, (byte) 65);
	KafkaBinderConfigurationProperties binderConfiguration = createConfigurationProperties();
	binderConfiguration.setMinPartitionCount(10);
	Binder binder = getBinder(binderConfiguration);
	QueueChannel moduleInputChannel = new QueueChannel();
	ExtendedProducerProperties<KafkaProducerProperties> producerProperties = createProducerProperties();
	producerProperties.setPartitionCount(10);
	producerProperties.setPartitionKeyExpression(new LiteralExpression("foo"));

	DirectChannel moduleOutputChannel = createBindableChannel("output",
			createProducerBindingProperties(producerProperties));

	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	long uniqueBindingId = System.currentTimeMillis();
	Binding<MessageChannel> producerBinding = binder.bindProducer(
			"foo" + uniqueBindingId + ".0", moduleOutputChannel, producerProperties);
	Binding<MessageChannel> consumerBinding = binder.bindConsumer(
			"foo" + uniqueBindingId + ".0", null, moduleInputChannel,
			consumerProperties);
	Message<?> message = org.springframework.integration.support.MessageBuilder
			.withPayload(testPayload).build();
	// Let the consumer actually bind to the producer before sending a msg
	binderBindUnbindLatency();
	moduleOutputChannel.send(message);
	Message<?> inbound = receive(moduleInputChannel);
	assertThat(inbound).isNotNull();
	assertThat((byte[]) inbound.getPayload()).containsExactly(testPayload);

	assertThat(partitionSize("foo" + uniqueBindingId + ".0")).isEqualTo(10);
	producerBinding.unbind();
	consumerBinding.unbind();
}
 
Example #19
Source File: RabbitMessageChannelBinder.java    From spring-cloud-stream-binder-rabbit with Apache License 2.0 5 votes vote down vote up
@Override
protected void afterUnbindConsumer(ConsumerDestination consumerDestination,
		String group,
		ExtendedConsumerProperties<RabbitConsumerProperties> consumerProperties) {
	provisioningProvider.cleanAutoDeclareContext(consumerDestination,
			consumerProperties);
}
 
Example #20
Source File: KafkaBinderTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 5 votes vote down vote up
@Override
protected ExtendedConsumerProperties<KafkaConsumerProperties> createConsumerProperties() {
	final ExtendedConsumerProperties<KafkaConsumerProperties> kafkaConsumerProperties = new ExtendedConsumerProperties<>(
			new KafkaConsumerProperties());
	// set the default values that would normally be propagated by Spring Cloud Stream
	kafkaConsumerProperties.setInstanceCount(1);
	kafkaConsumerProperties.setInstanceIndex(0);
	return kafkaConsumerProperties;
}
 
Example #21
Source File: RabbitBinderTests.java    From spring-cloud-stream-binder-rabbit with Apache License 2.0 5 votes vote down vote up
@Test
public void testConsumerPropertiesWithUserInfrastructureNoBind() throws Exception {
	RabbitAdmin admin = new RabbitAdmin(this.rabbitAvailableRule.getResource());
	Queue queue = new Queue("propsUser1.infra");
	admin.declareQueue(queue);
	DirectExchange exchange = new DirectExchange("propsUser1");
	admin.declareExchange(exchange);
	admin.declareBinding(BindingBuilder.bind(queue).to(exchange).with("foo"));

	RabbitTestBinder binder = getBinder();
	ExtendedConsumerProperties<RabbitConsumerProperties> properties = createConsumerProperties();
	properties.getExtension().setDeclareExchange(false);
	properties.getExtension().setBindQueue(false);

	Binding<MessageChannel> consumerBinding = binder.bindConsumer("propsUser1",
			"infra", createBindableChannel("input", new BindingProperties()),
			properties);
	Lifecycle endpoint = extractEndpoint(consumerBinding);
	SimpleMessageListenerContainer container = TestUtils.getPropertyValue(endpoint,
			"messageListenerContainer", SimpleMessageListenerContainer.class);
	assertThat(TestUtils.getPropertyValue(container, "missingQueuesFatal",
			Boolean.class)).isFalse();
	assertThat(container.isRunning()).isTrue();
	consumerBinding.unbind();
	assertThat(container.isRunning()).isFalse();
	Client client = new Client("http://guest:guest@localhost:15672/api/");
	List<?> bindings = client.getBindingsBySource("/", exchange.getName());
	assertThat(bindings.size()).isEqualTo(1);
}
 
Example #22
Source File: KafkaBinderTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testAutoCreateTopicsEnabledSucceeds() throws Exception {
	KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties();
	configurationProperties.setAutoCreateTopics(true);
	Binder binder = getBinder(configurationProperties);
	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	String testTopicName = "nonexisting" + System.currentTimeMillis();
	DirectChannel moduleInputChannel = createBindableChannel("input",
			createConsumerBindingProperties(consumerProperties));

	Binding<?> binding = binder.bindConsumer(testTopicName, "test",
			moduleInputChannel, consumerProperties);
	binding.unbind();
}
 
Example #23
Source File: KafkaBinderTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testAutoAddPartitionsDisabledFailsIfTopicUnderPartitionedAndAutoRebalanceDisabled()
		throws Throwable {
	KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties();

	String testTopicName = "existing" + System.currentTimeMillis();
	invokeCreateTopic(testTopicName, 1, 1);
	configurationProperties.setAutoAddPartitions(false);
	Binder binder = getBinder(configurationProperties);
	GenericApplicationContext context = new GenericApplicationContext();
	context.refresh();

	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	DirectChannel output = createBindableChannel("output",
			createConsumerBindingProperties(consumerProperties));
	// this consumer must consume from partition 2
	consumerProperties.setInstanceCount(3);
	consumerProperties.setInstanceIndex(2);
	consumerProperties.getExtension().setAutoRebalanceEnabled(false);
	expectedProvisioningException.expect(ProvisioningException.class);
	expectedProvisioningException.expectMessage(
			"The number of expected partitions was: 3, but 1 has been found instead");
	Binding binding = binder.bindConsumer(testTopicName, "test", output,
			consumerProperties);
	if (binding != null) {
		binding.unbind();
	}
}
 
Example #24
Source File: KafkaMessageChannelBinder.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 5 votes vote down vote up
private KafkaHeaderMapper getHeaderMapper(
		final ExtendedConsumerProperties<KafkaConsumerProperties> extendedConsumerProperties) {
	KafkaHeaderMapper mapper = null;
	if (this.configurationProperties.getHeaderMapperBeanName() != null) {
		mapper = getApplicationContext().getBean(
				this.configurationProperties.getHeaderMapperBeanName(),
				KafkaHeaderMapper.class);
	}
	if (mapper == null) {
		//First, try to see if there is a bean named headerMapper registered by other frameworks using the binder (for e.g. spring cloud sleuth)
		try {
			mapper = getApplicationContext().getBean("kafkaBinderHeaderMapper", KafkaHeaderMapper.class);
		}
		catch (BeansException be) {
			BinderHeaderMapper headerMapper = new BinderHeaderMapper() {

				@Override
				public void toHeaders(Headers source, Map<String, Object> headers) {
					super.toHeaders(source, headers);
					if (headers.size() > 0) {
						headers.put(BinderHeaders.NATIVE_HEADERS_PRESENT, Boolean.TRUE);
					}
				}

			};
			String[] trustedPackages = extendedConsumerProperties.getExtension()
					.getTrustedPackages();
			if (!StringUtils.isEmpty(trustedPackages)) {
				headerMapper.addTrustedPackages(trustedPackages);
			}
			mapper = headerMapper;
		}
	}
	return mapper;
}
 
Example #25
Source File: KafkaTestBinder.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings({ "rawtypes", "unchecked" })
KafkaTestBinder(KafkaBinderConfigurationProperties binderConfiguration,
		KafkaTopicProvisioner kafkaTopicProvisioner, DlqPartitionFunction dlqPartitionFunction) {

	try {
		KafkaMessageChannelBinder binder = new KafkaMessageChannelBinder(
				binderConfiguration, kafkaTopicProvisioner, null, null, null, dlqPartitionFunction) {

			/*
			 * Some tests use multiple instance indexes for the same topic; we need to
			 * make the error infrastructure beans unique.
			 */
			@Override
			protected String errorsBaseName(ConsumerDestination destination,
					String group,
					ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties) {
				return super.errorsBaseName(destination, group, consumerProperties)
						+ "-" + consumerProperties.getInstanceIndex();
			}

		};

		ProducerListener producerListener = new LoggingProducerListener();
		binder.setProducerListener(producerListener);
		AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(
				Config.class);
		setApplicationContext(context);
		binder.setApplicationContext(context);
		binder.afterPropertiesSet();
		this.setPollableConsumerBinder(binder);
	}
	catch (Exception e) {
		throw new RuntimeException(e);
	}
}
 
Example #26
Source File: KafkaMessageChannelBinder.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 5 votes vote down vote up
public Collection<PartitionInfo> processTopic(final String group,
		final ExtendedConsumerProperties<KafkaConsumerProperties> extendedConsumerProperties,
		final ConsumerFactory<?, ?> consumerFactory, int partitionCount,
		boolean usingPatterns, boolean groupManagement, String topic) {
	Collection<PartitionInfo> listenedPartitions;
	Collection<PartitionInfo> allPartitions = usingPatterns ? Collections.emptyList()
			: getPartitionInfo(topic, extendedConsumerProperties, consumerFactory,
					partitionCount);

	if (groupManagement || extendedConsumerProperties.getInstanceCount() == 1) {
		listenedPartitions = allPartitions;
	}
	else {
		listenedPartitions = new ArrayList<>();
		for (PartitionInfo partition : allPartitions) {
			// divide partitions across modules
			if ((partition.partition() % extendedConsumerProperties
					.getInstanceCount()) == extendedConsumerProperties
							.getInstanceIndex()) {
				listenedPartitions.add(partition);
			}
		}
	}
	this.topicsInUse.put(topic,
			new TopicInformation(group, listenedPartitions, usingPatterns));
	return listenedPartitions;
}
 
Example #27
Source File: KTableBinder.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 5 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
protected Binding<KTable<Object, Object>> doBindConsumer(String name, String group,
		KTable<Object, Object> inputTarget,
		// @checkstyle:off
		ExtendedConsumerProperties<KafkaStreamsConsumerProperties> properties) {
	// @checkstyle:on
	if (!StringUtils.hasText(group)) {
		group = properties.getExtension().getApplicationId();
	}
	KafkaStreamsBinderUtils.prepareConsumerBinding(name, group,
			getApplicationContext(), this.kafkaTopicProvisioner,
			this.binderConfigurationProperties, properties);
	return new DefaultBinding<>(name, group, inputTarget, null);
}
 
Example #28
Source File: GlobalKTableBinder.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 5 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
protected Binding<GlobalKTable<Object, Object>> doBindConsumer(String name,
		String group, GlobalKTable<Object, Object> inputTarget,
		ExtendedConsumerProperties<KafkaStreamsConsumerProperties> properties) {
	if (!StringUtils.hasText(group)) {
		group = properties.getExtension().getApplicationId();
	}
	KafkaStreamsBinderUtils.prepareConsumerBinding(name, group,
			getApplicationContext(), this.kafkaTopicProvisioner,
			this.binderConfigurationProperties, properties);
	return new DefaultBinding<>(name, group, inputTarget, null);
}
 
Example #29
Source File: KafkaTopicProvisioner.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 5 votes vote down vote up
private ConsumerDestination createDlqIfNeedBe(AdminClient adminClient, String name,
		String group, ExtendedConsumerProperties<KafkaConsumerProperties> properties,
		boolean anonymous, int partitions) {

	if (properties.getExtension().isEnableDlq() && !anonymous) {
		String dlqTopic = StringUtils.hasText(properties.getExtension().getDlqName())
				? properties.getExtension().getDlqName()
				: "error." + name + "." + group;
		int dlqPartitions = properties.getExtension().getDlqPartitions() == null
				? partitions
				: properties.getExtension().getDlqPartitions();
		try {
			createTopicAndPartitions(adminClient, dlqTopic, dlqPartitions,
					properties.getExtension().isAutoRebalanceEnabled(),
					properties.getExtension().getTopic());
		}
		catch (Throwable throwable) {
			if (throwable instanceof Error) {
				throw (Error) throwable;
			}
			else {
				throw new ProvisioningException("provisioning exception", throwable);
			}
		}
		return new KafkaConsumerDestination(name, partitions, dlqTopic);
	}
	return null;
}
 
Example #30
Source File: KafkaBinderTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testTopicPatterns() throws Exception {
	try (AdminClient admin = AdminClient.create(
			Collections.singletonMap(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG,
					embeddedKafka.getEmbeddedKafka().getBrokersAsString()))) {
		admin.createTopics(Collections
				.singletonList(new NewTopic("topicPatterns.1", 1, (short) 1))).all()
				.get();
		Binder binder = getBinder();
		ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
		consumerProperties.getExtension().setDestinationIsPattern(true);
		DirectChannel moduleInputChannel = createBindableChannel("input",
				createConsumerBindingProperties(consumerProperties));
		final CountDownLatch latch = new CountDownLatch(1);
		final AtomicReference<String> topic = new AtomicReference<>();
		moduleInputChannel.subscribe(m -> {
			topic.set(m.getHeaders().get(KafkaHeaders.RECEIVED_TOPIC, String.class));
			latch.countDown();
		});
		Binding<MessageChannel> consumerBinding = binder.bindConsumer(
				"topicPatterns\\..*", "testTopicPatterns", moduleInputChannel,
				consumerProperties);
		DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory(
				KafkaTestUtils.producerProps(embeddedKafka.getEmbeddedKafka()));
		KafkaTemplate template = new KafkaTemplate(pf);
		template.send("topicPatterns.1", "foo");
		assertThat(latch.await(10, TimeUnit.SECONDS)).isTrue();
		assertThat(topic.get()).isEqualTo("topicPatterns.1");
		consumerBinding.unbind();
		pf.destroy();
	}
}