Java Code Examples for org.springframework.cloud.stream.binder.ExtendedConsumerProperties#setInstanceCount()

The following examples show how to use org.springframework.cloud.stream.binder.ExtendedConsumerProperties#setInstanceCount() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: KafkaBinderTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 6 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testAutoAddPartitionsDisabledSucceedsIfTopicUnderPartitionedAndAutoRebalanceEnabled()
		throws Throwable {
	KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties();

	String testTopicName = "existing" + System.currentTimeMillis();
	invokeCreateTopic(testTopicName, 1, 1);
	configurationProperties.setAutoAddPartitions(false);
	Binder binder = getBinder(configurationProperties);
	GenericApplicationContext context = new GenericApplicationContext();
	context.refresh();

	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();

	DirectChannel input = createBindableChannel("input",
			createConsumerBindingProperties(consumerProperties));

	// this consumer must consume from partition 2
	consumerProperties.setInstanceCount(3);
	consumerProperties.setInstanceIndex(2);
	Binding binding = binder.bindConsumer(testTopicName, "test", input,
			consumerProperties);
	binding.unbind();
	assertThat(invokePartitionSize(testTopicName)).isEqualTo(1);
}
 
Example 2
Source File: KinesisStreamProvisionerTests.java    From spring-cloud-stream-binder-aws-kinesis with Apache License 2.0 5 votes vote down vote up
@Test
void testProvisionConsumerSuccessfulWithNewStream() {
	AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
	KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
	KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(
			amazonKinesisMock, binderProperties);
	int instanceCount = 1;
	int concurrency = 1;

	ExtendedConsumerProperties<KinesisConsumerProperties> extendedConsumerProperties =
			new ExtendedConsumerProperties<>(
			new KinesisConsumerProperties());
	extendedConsumerProperties.setInstanceCount(instanceCount);
	extendedConsumerProperties.setConcurrency(concurrency);

	String name = "test-stream";
	String group = "test-group";

	DescribeStreamResult describeStreamResult = describeStreamResultWithShards(
			Collections.singletonList(new Shard()));

	when(amazonKinesisMock.describeStream(any(DescribeStreamRequest.class)))
			.thenThrow(new ResourceNotFoundException("I got nothing"))
			.thenReturn(describeStreamResult);

	when(amazonKinesisMock.createStream(name, instanceCount * concurrency))
			.thenReturn(new CreateStreamResult());

	ConsumerDestination destination = provisioner.provisionConsumerDestination(name,
			group, extendedConsumerProperties);

	verify(amazonKinesisMock, times(2))
			.describeStream(any(DescribeStreamRequest.class));

	verify(amazonKinesisMock).createStream(name, instanceCount * concurrency);

	assertThat(destination.getName()).isEqualTo(name);
}
 
Example 3
Source File: KinesisStreamProvisionerTests.java    From spring-cloud-stream-binder-aws-kinesis with Apache License 2.0 5 votes vote down vote up
@Test
void testProvisionConsumerResourceNotFoundException() {
	AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
	KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
	binderProperties.setAutoCreateStream(false);
	KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(
			amazonKinesisMock, binderProperties);
	int instanceCount = 1;
	int concurrency = 1;

	ExtendedConsumerProperties<KinesisConsumerProperties> extendedConsumerProperties =
			new ExtendedConsumerProperties<>(
			new KinesisConsumerProperties());
	extendedConsumerProperties.setInstanceCount(instanceCount);
	extendedConsumerProperties.setConcurrency(concurrency);

	String name = "test-stream";
	String group = "test-group";

	when(amazonKinesisMock.describeStream(any(DescribeStreamRequest.class)))
			.thenThrow(new ResourceNotFoundException("Stream not found"));

	assertThatThrownBy(() -> provisioner.provisionConsumerDestination(name, group,
			extendedConsumerProperties))
			.isInstanceOf(ProvisioningException.class)
			.hasMessageContaining(
					"The stream [test-stream] was not found and auto creation is disabled.")
			.hasCauseInstanceOf(ResourceNotFoundException.class);

	verify(amazonKinesisMock, times(1))
			.describeStream(any(DescribeStreamRequest.class));

	verify(amazonKinesisMock, never()).createStream(name,
			instanceCount * concurrency);
}
 
Example 4
Source File: KinesisBinderTests.java    From spring-cloud-stream-binder-aws-kinesis with Apache License 2.0 5 votes vote down vote up
@Override
protected ExtendedConsumerProperties<KinesisConsumerProperties> createConsumerProperties() {
	ExtendedConsumerProperties<KinesisConsumerProperties> kinesisConsumerProperties = new ExtendedConsumerProperties<>(
			new KinesisConsumerProperties());
	// set the default values that would normally be propagated by Spring Cloud Stream
	kinesisConsumerProperties.setInstanceCount(1);
	kinesisConsumerProperties.setInstanceIndex(0);
	kinesisConsumerProperties.getExtension().setShardIteratorType(ShardIteratorType.TRIM_HORIZON.name());
	return kinesisConsumerProperties;
}
 
Example 5
Source File: KafkaBinderTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 5 votes vote down vote up
@Override
protected ExtendedConsumerProperties<KafkaConsumerProperties> createConsumerProperties() {
	final ExtendedConsumerProperties<KafkaConsumerProperties> kafkaConsumerProperties = new ExtendedConsumerProperties<>(
			new KafkaConsumerProperties());
	// set the default values that would normally be propagated by Spring Cloud Stream
	kafkaConsumerProperties.setInstanceCount(1);
	kafkaConsumerProperties.setInstanceIndex(0);
	return kafkaConsumerProperties;
}
 
Example 6
Source File: KafkaBinderTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testAutoAddPartitionsDisabledFailsIfTopicUnderPartitionedAndAutoRebalanceDisabled()
		throws Throwable {
	KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties();

	String testTopicName = "existing" + System.currentTimeMillis();
	invokeCreateTopic(testTopicName, 1, 1);
	configurationProperties.setAutoAddPartitions(false);
	Binder binder = getBinder(configurationProperties);
	GenericApplicationContext context = new GenericApplicationContext();
	context.refresh();

	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	DirectChannel output = createBindableChannel("output",
			createConsumerBindingProperties(consumerProperties));
	// this consumer must consume from partition 2
	consumerProperties.setInstanceCount(3);
	consumerProperties.setInstanceIndex(2);
	consumerProperties.getExtension().setAutoRebalanceEnabled(false);
	expectedProvisioningException.expect(ProvisioningException.class);
	expectedProvisioningException.expectMessage(
			"The number of expected partitions was: 3, but 1 has been found instead");
	Binding binding = binder.bindConsumer(testTopicName, "test", output,
			consumerProperties);
	if (binding != null) {
		binding.unbind();
	}
}
 
Example 7
Source File: KinesisBinderTests.java    From spring-cloud-stream-binder-aws-kinesis with Apache License 2.0 4 votes vote down vote up
@Test
@Override
@SuppressWarnings("unchecked")
public void testAnonymousGroup() throws Exception {
	KinesisTestBinder binder = getBinder();
	ExtendedProducerProperties<KinesisProducerProperties> producerProperties = createProducerProperties();
	DirectChannel output = createBindableChannel("output",
			createProducerBindingProperties(producerProperties));

	Binding<MessageChannel> producerBinding = binder.bindProducer(
			String.format("defaultGroup%s0", getDestinationNameDelimiter()), output,
			producerProperties);

	ExtendedConsumerProperties<KinesisConsumerProperties> consumerProperties = createConsumerProperties();
	consumerProperties.setConcurrency(2);
	consumerProperties.setInstanceCount(3);
	consumerProperties.setInstanceIndex(0);

	QueueChannel input1 = new QueueChannel();
	Binding<MessageChannel> binding1 = binder.bindConsumer(
			String.format("defaultGroup%s0", getDestinationNameDelimiter()), null,
			input1, consumerProperties);

	consumerProperties.setInstanceIndex(1);

	QueueChannel input2 = new QueueChannel();
	Binding<MessageChannel> binding2 = binder.bindConsumer(
			String.format("defaultGroup%s0", getDestinationNameDelimiter()), null,
			input2, consumerProperties);

	String testPayload1 = "foo-" + UUID.randomUUID().toString();
	output.send(MessageBuilder.withPayload(testPayload1)
			.setHeader(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.TEXT_PLAIN)
			.build());

	Message<byte[]> receivedMessage1 = (Message<byte[]>) receive(input1);
	assertThat(receivedMessage1).isNotNull();
	assertThat(new String(receivedMessage1.getPayload())).isEqualTo(testPayload1);

	Message<byte[]> receivedMessage2 = (Message<byte[]>) receive(input2);
	assertThat(receivedMessage2).isNotNull();
	assertThat(new String(receivedMessage2.getPayload())).isEqualTo(testPayload1);

	binding2.unbind();

	String testPayload2 = "foo-" + UUID.randomUUID().toString();
	output.send(MessageBuilder.withPayload(testPayload2)
			.setHeader(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.TEXT_PLAIN)
			.build());

	binding2 = binder.bindConsumer(
			String.format("defaultGroup%s0", getDestinationNameDelimiter()), null,
			input2, consumerProperties);
	String testPayload3 = "foo-" + UUID.randomUUID().toString();
	output.send(MessageBuilder.withPayload(testPayload3)
			.setHeader(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.TEXT_PLAIN)
			.build());

	receivedMessage1 = (Message<byte[]>) receive(input1);
	assertThat(receivedMessage1).isNotNull();
	assertThat(new String(receivedMessage1.getPayload())).isEqualTo(testPayload2);
	receivedMessage1 = (Message<byte[]>) receive(input1);
	assertThat(receivedMessage1).isNotNull();
	assertThat(new String(receivedMessage1.getPayload())).isNotNull();

	receivedMessage2 = (Message<byte[]>) receive(input2);
	assertThat(receivedMessage2).isNotNull();
	assertThat(new String(receivedMessage2.getPayload())).isEqualTo(testPayload1);

	receivedMessage2 = (Message<byte[]>) receive(input2);
	assertThat(receivedMessage2).isNotNull();
	assertThat(new String(receivedMessage2.getPayload())).isEqualTo(testPayload2);

	receivedMessage2 = (Message<byte[]>) receive(input2);
	assertThat(receivedMessage2).isNotNull();
	assertThat(new String(receivedMessage2.getPayload())).isEqualTo(testPayload3);

	producerBinding.unbind();
	binding1.unbind();
	binding2.unbind();
}
 
Example 8
Source File: KinesisBinderTests.java    From spring-cloud-stream-binder-aws-kinesis with Apache License 2.0 4 votes vote down vote up
@Test
@Override
public void testPartitionedModuleJava() throws Exception {
	KinesisTestBinder binder = getBinder();

	ExtendedConsumerProperties<KinesisConsumerProperties> consumerProperties = createConsumerProperties();
	consumerProperties.setConcurrency(2);
	consumerProperties.setInstanceCount(3);
	consumerProperties.setInstanceIndex(0);
	consumerProperties.setPartitioned(true);

	final List<Message<?>> results = new ArrayList<>();
	final CountDownLatch receiveLatch = new CountDownLatch(3);

	MessageHandler receivingHandler = (message) -> {
		results.add(message);
		receiveLatch.countDown();
	};

	DirectChannel input0 = createBindableChannelInternal("test.input0J", new BindingProperties(), true);
	input0.subscribe(receivingHandler);

	Binding<MessageChannel> input0Binding = binder.bindConsumer("partJ.0",
			"testPartitionedModuleJava", input0, consumerProperties);

	consumerProperties.setInstanceIndex(1);

	DirectChannel input1 = createBindableChannelInternal("test.input1J", new BindingProperties(), true);
	input1.subscribe(receivingHandler);

	Binding<MessageChannel> input1Binding = binder.bindConsumer("partJ.0",
			"testPartitionedModuleJava", input1, consumerProperties);

	consumerProperties.setInstanceIndex(2);

	DirectChannel input2 = createBindableChannelInternal("test.input2J", new BindingProperties(), true);
	input2.subscribe(receivingHandler);

	Binding<MessageChannel> input2Binding = binder.bindConsumer("partJ.0",
			"testPartitionedModuleJava", input2, consumerProperties);

	ExtendedProducerProperties<KinesisProducerProperties> producerProperties = createProducerProperties();

	producerProperties.setPartitionKeyExtractorName("partitionSupport");
	producerProperties.setPartitionSelectorName("partitionSupport");
	producerProperties.setPartitionCount(3);

	DirectChannel output = createBindableChannelInternal("test.output",
			createProducerBindingProperties(producerProperties), false);

	Binding<MessageChannel> outputBinding = binder.bindProducer("partJ.0", output,
			producerProperties);
	if (usesExplicitRouting()) {
		Object endpoint = extractEndpoint(outputBinding);
		assertThat(getEndpointRouting(endpoint))
				.contains(getExpectedRoutingBaseDestination("partJ.0",
						"testPartitionedModuleJava") + "-' + headers['"
						+ BinderHeaders.PARTITION_HEADER + "']");
	}

	output.send(new GenericMessage<>(2));
	output.send(new GenericMessage<>(1));
	output.send(new GenericMessage<>(0));

	assertThat(receiveLatch.await(20, TimeUnit.SECONDS)).isTrue();

	assertThat(results).extracting("payload").containsExactlyInAnyOrder(
			"0".getBytes(), "1".getBytes(), "2".getBytes());

	input0Binding.unbind();
	input1Binding.unbind();
	input2Binding.unbind();
	outputBinding.unbind();
}
 
Example 9
Source File: KafkaBinderTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 4 votes vote down vote up
@Test
@Override
@SuppressWarnings({ "unchecked", "rawtypes" })
public void testPartitionedModuleJava() throws Exception {
	Binder binder = getBinder();

	KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties();

	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	consumerProperties.setConcurrency(2);
	consumerProperties.setInstanceCount(4);
	consumerProperties.setInstanceIndex(0);
	consumerProperties.setPartitioned(true);
	consumerProperties.getExtension().setAutoRebalanceEnabled(false);
	QueueChannel input0 = new QueueChannel();
	input0.setBeanName("test.input0J");
	Binding<MessageChannel> input0Binding = binder.bindConsumer("partJ.0", "test",
			input0, consumerProperties);
	consumerProperties.setInstanceIndex(1);
	QueueChannel input1 = new QueueChannel();
	input1.setBeanName("test.input1J");
	Binding<MessageChannel> input1Binding = binder.bindConsumer("partJ.0", "test",
			input1, consumerProperties);
	consumerProperties.setInstanceIndex(2);
	QueueChannel input2 = new QueueChannel();
	input2.setBeanName("test.input2J");
	Binding<MessageChannel> input2Binding = binder.bindConsumer("partJ.0", "test",
			input2, consumerProperties);
	consumerProperties.setInstanceIndex(3);
	QueueChannel input3 = new QueueChannel();
	input3.setBeanName("test.input3J");
	Binding<MessageChannel> input3Binding = binder.bindConsumer("partJ.0", "test",
			input3, consumerProperties);

	ExtendedProducerProperties<KafkaProducerProperties> producerProperties = createProducerProperties();

	this.applicationContext.registerBean("pkExtractor",
			PartitionTestSupport.class, () -> new PartitionTestSupport());
	this.applicationContext.registerBean("pkSelector",
			PartitionTestSupport.class, () -> new PartitionTestSupport());
	producerProperties.setPartitionKeyExtractorName("pkExtractor");
	producerProperties.setPartitionSelectorName("pkSelector");
	producerProperties.setPartitionCount(3); // overridden to 8 on the actual topic
	DirectChannel output = createBindableChannel("output",
			createProducerBindingProperties(producerProperties));
	output.setBeanName("test.output");
	Binding<MessageChannel> outputBinding = binder.bindProducer("partJ.0", output,
			producerProperties);
	if (usesExplicitRouting()) {
		Object endpoint = extractEndpoint(outputBinding);
		assertThat(getEndpointRouting(endpoint))
				.contains(getExpectedRoutingBaseDestination("partJ.0", "test")
						+ "-' + headers['partition']");
	}

	output.send(new GenericMessage<>(2));
	output.send(new GenericMessage<>(1));
	output.send(new GenericMessage<>(0));
	output.send(new GenericMessage<>(3));

	Message<?> receive0 = receive(input0);
	assertThat(receive0).isNotNull();
	Message<?> receive1 = receive(input1);
	assertThat(receive1).isNotNull();
	Message<?> receive2 = receive(input2);
	assertThat(receive2).isNotNull();
	Message<?> receive3 = receive(input3);
	assertThat(receive3).isNotNull();
	ObjectMapper om = new ObjectMapper();

	assertThat(om.readValue((byte[]) receive0.getPayload(), Integer.class))
			.isEqualTo(0);
	assertThat(om.readValue((byte[]) receive1.getPayload(), Integer.class))
			.isEqualTo(1);
	assertThat(om.readValue((byte[]) receive2.getPayload(), Integer.class))
			.isEqualTo(2);
	assertThat(om.readValue((byte[]) receive3.getPayload(), Integer.class))
			.isEqualTo(3);

	input0Binding.unbind();
	input1Binding.unbind();
	input2Binding.unbind();
	input3Binding.unbind();
	outputBinding.unbind();
}
 
Example 10
Source File: KafkaBinderTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 4 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testPartitionedModuleJavaWithRawMode() throws Exception {
	Binder binder = getBinder();
	ExtendedProducerProperties<KafkaProducerProperties> properties = createProducerProperties();
	properties.setHeaderMode(HeaderMode.none);
	this.applicationContext.registerBean("pkExtractor",
			RawKafkaPartitionTestSupport.class, () -> new RawKafkaPartitionTestSupport());
	this.applicationContext.registerBean("pkSelector",
			RawKafkaPartitionTestSupport.class, () -> new RawKafkaPartitionTestSupport());
	properties.setPartitionKeyExtractorName("pkExtractor");
	properties.setPartitionSelectorName("pkSelector");
	properties.setPartitionCount(6);

	DirectChannel output = createBindableChannel("output",
			createProducerBindingProperties(properties));
	output.setBeanName("test.output");
	Binding<MessageChannel> outputBinding = binder.bindProducer("partJ.raw.0", output,
			properties);

	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	consumerProperties.setConcurrency(2);
	consumerProperties.setInstanceCount(3);
	consumerProperties.setInstanceIndex(0);
	consumerProperties.setPartitioned(true);
	consumerProperties.setHeaderMode(HeaderMode.none);
	consumerProperties.getExtension().setAutoRebalanceEnabled(false);
	QueueChannel input0 = new QueueChannel();
	input0.setBeanName("test.input0J");
	Binding<MessageChannel> input0Binding = binder.bindConsumer("partJ.raw.0", "test",
			input0, consumerProperties);
	consumerProperties.setInstanceIndex(1);
	QueueChannel input1 = new QueueChannel();
	input1.setBeanName("test.input1J");
	Binding<MessageChannel> input1Binding = binder.bindConsumer("partJ.raw.0", "test",
			input1, consumerProperties);
	consumerProperties.setInstanceIndex(2);
	QueueChannel input2 = new QueueChannel();
	input2.setBeanName("test.input2J");
	Binding<MessageChannel> input2Binding = binder.bindConsumer("partJ.raw.0", "test",
			input2, consumerProperties);

	output.send(new GenericMessage<>(new byte[] { (byte) 0 }));
	output.send(new GenericMessage<>(new byte[] { (byte) 1 }));
	output.send(new GenericMessage<>(new byte[] { (byte) 2 }));

	Message<?> receive0 = receive(input0);
	assertThat(receive0).isNotNull();
	Message<?> receive1 = receive(input1);
	assertThat(receive1).isNotNull();
	Message<?> receive2 = receive(input2);
	assertThat(receive2).isNotNull();

	assertThat(Arrays.asList(((byte[]) receive0.getPayload())[0],
			((byte[]) receive1.getPayload())[0], ((byte[]) receive2.getPayload())[0]))
					.containsExactlyInAnyOrder((byte) 0, (byte) 1, (byte) 2);

	input0Binding.unbind();
	input1Binding.unbind();
	input2Binding.unbind();
	outputBinding.unbind();
}
 
Example 11
Source File: KafkaBinderTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 4 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testPartitionedModuleSpELWithRawMode() throws Exception {
	Binder binder = getBinder();
	ExtendedProducerProperties<KafkaProducerProperties> properties = createProducerProperties();
	properties.setPartitionKeyExpression(
			spelExpressionParser.parseExpression("payload[0]"));
	properties.setPartitionSelectorExpression(
			spelExpressionParser.parseExpression("hashCode()"));
	properties.setPartitionCount(6);
	properties.setHeaderMode(HeaderMode.none);

	DirectChannel output = createBindableChannel("output",
			createProducerBindingProperties(properties));
	output.setBeanName("test.output");
	Binding<MessageChannel> outputBinding = binder.bindProducer("part.raw.0", output,
			properties);
	try {
		Object endpoint = extractEndpoint(outputBinding);
		assertThat(getEndpointRouting(endpoint))
				.contains(getExpectedRoutingBaseDestination("part.raw.0", "test")
						+ "-' + headers['partition']");
	}
	catch (UnsupportedOperationException ignored) {
	}

	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	consumerProperties.setConcurrency(2);
	consumerProperties.setInstanceIndex(0);
	consumerProperties.setInstanceCount(3);
	consumerProperties.setPartitioned(true);
	consumerProperties.setHeaderMode(HeaderMode.none);
	consumerProperties.getExtension().setAutoRebalanceEnabled(false);
	QueueChannel input0 = new QueueChannel();
	input0.setBeanName("test.input0S");
	Binding<MessageChannel> input0Binding = binder.bindConsumer("part.raw.0", "test",
			input0, consumerProperties);
	consumerProperties.setInstanceIndex(1);
	QueueChannel input1 = new QueueChannel();
	input1.setBeanName("test.input1S");
	Binding<MessageChannel> input1Binding = binder.bindConsumer("part.raw.0", "test",
			input1, consumerProperties);
	consumerProperties.setInstanceIndex(2);
	QueueChannel input2 = new QueueChannel();
	input2.setBeanName("test.input2S");
	Binding<MessageChannel> input2Binding = binder.bindConsumer("part.raw.0", "test",
			input2, consumerProperties);

	Message<byte[]> message2 = org.springframework.integration.support.MessageBuilder
			.withPayload(new byte[] { 2 })
			.setHeader(IntegrationMessageHeaderAccessor.CORRELATION_ID,
					"kafkaBinderTestCommonsDelegate")
			.setHeader(IntegrationMessageHeaderAccessor.SEQUENCE_NUMBER, 42)
			.setHeader(IntegrationMessageHeaderAccessor.SEQUENCE_SIZE, 43).build();
	output.send(message2);
	output.send(new GenericMessage<>(new byte[] { 1 }));
	output.send(new GenericMessage<>(new byte[] { 0 }));
	Message<?> receive0 = receive(input0);
	assertThat(receive0).isNotNull();
	Message<?> receive1 = receive(input1);
	assertThat(receive1).isNotNull();
	Message<?> receive2 = receive(input2);
	assertThat(receive2).isNotNull();
	assertThat(Arrays.asList(((byte[]) receive0.getPayload())[0],
			((byte[]) receive1.getPayload())[0], ((byte[]) receive2.getPayload())[0]))
					.containsExactlyInAnyOrder((byte) 0, (byte) 1, (byte) 2);
	input0Binding.unbind();
	input1Binding.unbind();
	input2Binding.unbind();
	outputBinding.unbind();
}
 
Example 12
Source File: KafkaBinderTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 4 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testAutoAddPartitionsDisabledSucceedsIfTopicPartitionedCorrectly()
		throws Throwable {
	Binding<?> binding = null;
	try {
		KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties();

		String testTopicName = "existing" + System.currentTimeMillis();
		invokeCreateTopic(testTopicName, 6, 1);
		configurationProperties.setAutoAddPartitions(false);
		Binder binder = getBinder(configurationProperties);
		ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();

		DirectChannel input = createBindableChannel("input",
				createConsumerBindingProperties(consumerProperties));

		// this consumer must consume from partition 2
		consumerProperties.setInstanceCount(3);
		consumerProperties.setInstanceIndex(2);
		consumerProperties.getExtension().setAutoRebalanceEnabled(false);

		binding = binder.bindConsumer(testTopicName, "test-x", input,
				consumerProperties);

		ContainerProperties containerProps = TestUtils.getPropertyValue(
				binding,
				"lifecycle.messageListenerContainer.containerProperties",
				ContainerProperties.class);
		TopicPartitionOffset[] listenedPartitions = containerProps.getTopicPartitionsToAssign();
		assertThat(listenedPartitions).hasSize(2);
		assertThat(listenedPartitions).contains(
				new TopicPartitionOffset(testTopicName, 2),
				new TopicPartitionOffset(testTopicName, 5));
		int partitions = invokePartitionSize(testTopicName);
		assertThat(partitions).isEqualTo(6);
	}
	finally {
		if (binding != null) {
			binding.unbind();
		}
	}
}
 
Example 13
Source File: KafkaBinderTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 4 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testResetOffsets() throws Exception {
	Binding<?> producerBinding = null;
	Binding<?> consumerBinding = null;
	try {
		String testPayload = "test";

		ExtendedProducerProperties<KafkaProducerProperties> producerProperties = createProducerProperties();

		DirectChannel moduleOutputChannel = createBindableChannel("output",
				createProducerBindingProperties(producerProperties));

		ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
		consumerProperties.setConcurrency(2);
		consumerProperties.setInstanceCount(5); // 10 partitions across 2 threads
		consumerProperties.getExtension().setResetOffsets(true);

		DirectChannel moduleInputChannel = createBindableChannel("input",
				createConsumerBindingProperties(consumerProperties));

		String testTopicName = "existing" + System.currentTimeMillis();
		KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties();
		configurationProperties.setAutoAddPartitions(true);
		Binder binder = getBinder(configurationProperties);
		producerBinding = binder.bindProducer(testTopicName, moduleOutputChannel,
				producerProperties);

		consumerBinding = binder.bindConsumer(testTopicName, "testReset",
				moduleInputChannel, consumerProperties);
		// Let the consumer actually bind to the producer before sending a msg
		binderBindUnbindLatency();
		IntStream.range(0, 10).forEach(i -> moduleOutputChannel.send(MessageBuilder.withPayload(testPayload)
				.setHeader(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.TEXT_PLAIN)
				.setHeader(KafkaHeaders.PARTITION_ID, i)
				.build()));
		CountDownLatch latch1 = new CountDownLatch(10);
		CountDownLatch latch2 = new CountDownLatch(20);
		AtomicReference<Message<byte[]>> inboundMessageRef = new AtomicReference<>();
		AtomicInteger received = new AtomicInteger();
		moduleInputChannel.subscribe(message1 -> {
			try {
				inboundMessageRef.set((Message<byte[]>) message1);
			}
			finally {
				received.incrementAndGet();
				latch1.countDown();
				latch2.countDown();
			}
		});
		assertThat(latch1.await(10, TimeUnit.SECONDS)).as("Failed to receive messages").isTrue();
		consumerBinding.unbind();
		consumerBinding = binder.bindConsumer(testTopicName, "testReset",
				moduleInputChannel, consumerProperties);
		assertThat(latch2.await(10, TimeUnit.SECONDS)).as("Failed to receive message").isTrue();
		binder.bindConsumer(testTopicName + "-x", "testReset",
				moduleInputChannel, consumerProperties).unbind(); // cause another rebalance
		assertThat(received.get()).as("Unexpected reset").isEqualTo(20);

		assertThat(inboundMessageRef.get()).isNotNull();
		assertThat(inboundMessageRef.get().getPayload()).isEqualTo("test".getBytes());
		assertThat(inboundMessageRef.get().getHeaders()).containsEntry("contentType",
				MimeTypeUtils.TEXT_PLAIN);
	}
	finally {
		if (producerBinding != null) {
			producerBinding.unbind();
		}
		if (consumerBinding != null) {
			consumerBinding.unbind();
		}
	}
}