org.springframework.kafka.config.StreamsBuilderFactoryBean Java Examples

The following examples show how to use org.springframework.kafka.config.StreamsBuilderFactoryBean. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: SleuthKafkaStreamsConfiguration.java From spring-cloud-sleuth with Apache License 2.0

6 votes

@Override
public Object postProcessAfterInitialization(Object bean, String beanName)
		throws BeansException {
	if (bean instanceof StreamsBuilderFactoryBean) {
		// KafkaStreamsTracing is created in SleuthKafkaStreamsConfiguration above, so
		// should not be null here
		KafkaStreamsTracing kafkaStreamsTracing = this.objectProvider
				.getIfAvailable();
		((StreamsBuilderFactoryBean) bean)
				.setClientSupplier(kafkaStreamsTracing.kafkaClientSupplier());
		if (log.isDebugEnabled()) {
			log.debug(
					"StreamsBuilderFactoryBean bean is auto-configured to enable tracing.");
		}
	}
	return bean;
}

Example #2

Source File: KafkaStreamsInteractiveQueryIntegrationTests.java From spring-cloud-stream-binder-kafka with Apache License 2.0

6 votes

@Test
public void testStateStoreRetrievalRetry() {

	StreamsBuilderFactoryBean mock = Mockito.mock(StreamsBuilderFactoryBean.class);
	KafkaStreams mockKafkaStreams = Mockito.mock(KafkaStreams.class);
	Mockito.when(mock.getKafkaStreams()).thenReturn(mockKafkaStreams);
	KafkaStreamsRegistry kafkaStreamsRegistry = new KafkaStreamsRegistry();
	kafkaStreamsRegistry.registerKafkaStreams(mock);
	KafkaStreamsBinderConfigurationProperties binderConfigurationProperties =
			new KafkaStreamsBinderConfigurationProperties(new KafkaProperties());
	binderConfigurationProperties.getStateStoreRetry().setMaxAttempts(3);
	InteractiveQueryService interactiveQueryService = new InteractiveQueryService(kafkaStreamsRegistry,
			binderConfigurationProperties);

	QueryableStoreType<ReadOnlyKeyValueStore<Object, Object>> storeType = QueryableStoreTypes.keyValueStore();
	try {
		interactiveQueryService.getQueryableStore("foo", storeType);
	}
	catch (Exception ignored) {

	}

	Mockito.verify(mockKafkaStreams, times(3)).store("foo", storeType);
}

Example #3

Source File: StreamsBuilderFactoryManager.java From spring-cloud-stream-binder-kafka with Apache License 2.0

6 votes

@Override
public synchronized void start() {
	if (!this.running) {
		try {
			Set<StreamsBuilderFactoryBean> streamsBuilderFactoryBeans = this.kafkaStreamsBindingInformationCatalogue
					.getStreamsBuilderFactoryBeans();
			int n = 0;
			for (StreamsBuilderFactoryBean streamsBuilderFactoryBean : streamsBuilderFactoryBeans) {
				streamsBuilderFactoryBean.start();
				this.kafkaStreamsRegistry.registerKafkaStreams(streamsBuilderFactoryBean);
				if (this.listener != null) {
					this.listener.streamsAdded("streams." + n++, streamsBuilderFactoryBean.getKafkaStreams());
				}
			}
			if (this.kafkaStreamsBinderMetrics != null) {
				this.kafkaStreamsBinderMetrics.addMetrics(streamsBuilderFactoryBeans);
			}
			this.running = true;
		}
		catch (Exception ex) {
			throw new KafkaException("Could not start stream: ", ex);
		}
	}
}

Example #4

Source File: StreamsBuilderFactoryManager.java From spring-cloud-stream-binder-kafka with Apache License 2.0

6 votes

@Override
public synchronized void stop() {
	if (this.running) {
		try {
			Set<StreamsBuilderFactoryBean> streamsBuilderFactoryBeans = this.kafkaStreamsBindingInformationCatalogue
					.getStreamsBuilderFactoryBeans();
			int n = 0;
			for (StreamsBuilderFactoryBean streamsBuilderFactoryBean : streamsBuilderFactoryBeans) {
				streamsBuilderFactoryBean.stop();
				if (this.listener != null) {
					this.listener.streamsRemoved("streams." + n++, streamsBuilderFactoryBean.getKafkaStreams());
				}
			}
		}
		catch (Exception ex) {
			throw new IllegalStateException(ex);
		}
		finally {
			this.running = false;
		}
	}
}

Example #5

Source File: MultiProcessorsWithSameNameAndBindingTests.java From spring-cloud-stream-binder-kafka with Apache License 2.0

6 votes

@Test
public void testBinderStartsSuccessfullyWhenTwoProcessorsWithSameNamesAndBindingsPresent() {
	SpringApplication app = new SpringApplication(
			MultiProcessorsWithSameNameAndBindingTests.WordCountProcessorApplication.class);
	app.setWebApplicationType(WebApplicationType.NONE);

	try (ConfigurableApplicationContext context = app.run("--server.port=0",
			"--spring.jmx.enabled=false",
			"--spring.cloud.stream.bindings.input.destination=words",
			"--spring.cloud.stream.bindings.input-1.destination=words",
			"--spring.cloud.stream.bindings.output.destination=counts",
			"--spring.cloud.stream.bindings.output.contentType=application/json",
			"--spring.cloud.stream.kafka.streams.binder.brokers="
					+ embeddedKafka.getBrokersAsString())) {
		StreamsBuilderFactoryBean streamsBuilderFactoryBean1 = context
				.getBean("&stream-builder-Foo-process", StreamsBuilderFactoryBean.class);
		assertThat(streamsBuilderFactoryBean1).isNotNull();
		StreamsBuilderFactoryBean streamsBuilderFactoryBean2 = context
				.getBean("&stream-builder-Bar-process", StreamsBuilderFactoryBean.class);
		assertThat(streamsBuilderFactoryBean2).isNotNull();
	}
}

Example #6

Source File: KafkaStreamsRegistry.java From spring-cloud-stream-binder-kafka with Apache License 2.0

5 votes

public StreamsBuilderFactoryBean streamsBuilderFactoryBean(String applicationId) {
	final Optional<StreamsBuilderFactoryBean> first = this.streamsBuilderFactoryBeanMap.values()
			.stream()
			.filter(streamsBuilderFactoryBean -> streamsBuilderFactoryBean
					.getStreamsConfiguration().getProperty(StreamsConfig.APPLICATION_ID_CONFIG)
					.equals(applicationId))
			.findFirst();
	return first.orElse(null);
}

Example #7

Source File: KafkaStreamsTopologyEndpoint.java From spring-cloud-stream-binder-kafka with Apache License 2.0

5 votes

@ReadOperation
public String kafkaStreamsTopology() {
	final List<StreamsBuilderFactoryBean> streamsBuilderFactoryBeans = this.kafkaStreamsRegistry.streamsBuilderFactoryBeans();
	final StringBuilder topologyDescription = new StringBuilder();
	streamsBuilderFactoryBeans.stream()
			.forEach(streamsBuilderFactoryBean ->
					topologyDescription.append(streamsBuilderFactoryBean.getTopology().describe().toString()));
	return topologyDescription.toString();
}

Example #8

Source File: KafkaStreamsTopologyEndpoint.java From spring-cloud-stream-binder-kafka with Apache License 2.0

5 votes

@ReadOperation
public String kafkaStreamsTopology(@Selector String applicationId) {
	if (!StringUtils.isEmpty(applicationId)) {
		final StreamsBuilderFactoryBean streamsBuilderFactoryBean = this.kafkaStreamsRegistry.streamsBuilderFactoryBean(applicationId);
		if (streamsBuilderFactoryBean != null) {
			return streamsBuilderFactoryBean.getTopology().describe().toString();
		}
		else {
			return NO_TOPOLOGY_FOUND_MSG;
		}
	}
	return NO_TOPOLOGY_FOUND_MSG;
}

Example #9

Source File: KafkastreamsBinderPojoInputStringOutputIntegrationTests.java From spring-cloud-stream-binder-kafka with Apache License 2.0

5 votes

@Test
public void testKstreamBinderWithPojoInputAndStringOuput() throws Exception {
	SpringApplication app = new SpringApplication(ProductCountApplication.class);
	app.setWebApplicationType(WebApplicationType.NONE);
	ConfigurableApplicationContext context = app.run("--server.port=0",
			"--spring.jmx.enabled=false",
			"--spring.cloud.stream.bindings.input.destination=foos",
			"--spring.cloud.stream.bindings.output.destination=counts-id",
			"--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000",
			"--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde"
					+ "=org.apache.kafka.common.serialization.Serdes$StringSerde",
			"--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde"
					+ "=org.apache.kafka.common.serialization.Serdes$StringSerde",
			"--spring.cloud.stream.kafka.streams.bindings.input.consumer.applicationId=ProductCountApplication-xyz",
			"--spring.cloud.stream.kafka.streams.binder.brokers="
					+ embeddedKafka.getBrokersAsString());
	try {
		receiveAndValidateFoo();
		// Assertions on StreamBuilderFactoryBean
		StreamsBuilderFactoryBean streamsBuilderFactoryBean = context
				.getBean("&stream-builder-ProductCountApplication-process", StreamsBuilderFactoryBean.class);
		CleanupConfig cleanup = TestUtils.getPropertyValue(streamsBuilderFactoryBean,
				"cleanupConfig", CleanupConfig.class);
		assertThat(cleanup.cleanupOnStart()).isFalse();
		assertThat(cleanup.cleanupOnStop()).isTrue();
	}
	finally {
		context.close();
	}
}

Example #10

Source File: SleuthKafkaStreamsConfigurationTest.java From spring-cloud-sleuth with Apache License 2.0

4 votes

@Bean
StreamsBuilderFactoryBean streamsBuilderFactoryBean() {
	TestTraceStreamsBuilderFactoryBean factoryBean = new TestTraceStreamsBuilderFactoryBean();
	factoryBean.setAutoStartup(false);
	return factoryBean;
}

Example #11

Source File: KafkaStreamsRegistry.java From spring-cloud-stream-binder-kafka with Apache License 2.0

4 votes

/**
 * Register the {@link KafkaStreams} object created in the application.
 * @param streamsBuilderFactoryBean {@link StreamsBuilderFactoryBean}
 */
void registerKafkaStreams(StreamsBuilderFactoryBean streamsBuilderFactoryBean) {
	final KafkaStreams kafkaStreams = streamsBuilderFactoryBean.getKafkaStreams();
	this.kafkaStreams.add(kafkaStreams);
	this.streamsBuilderFactoryBeanMap.put(kafkaStreams, streamsBuilderFactoryBean);
}

Example #12

Source File: SleuthKafkaStreamsConfigurationIntegrationTests.java From spring-cloud-sleuth with Apache License 2.0

4 votes

@Bean
StreamsBuilderFactoryBean streamsBuilderFactoryBean() {
	streamsBuilderFactoryBean = mock(StreamsBuilderFactoryBean.class);
	return UserConfig.streamsBuilderFactoryBean;
}

Example #13

Source File: MultipleFunctionsInSameAppTests.java From spring-cloud-stream-binder-kafka with Apache License 2.0

4 votes

@Test
public void testMultiFunctionsInSameAppWithMultiBinders() throws Exception {
	SpringApplication app = new SpringApplication(MultipleFunctionsInSameApp.class);
	app.setWebApplicationType(WebApplicationType.NONE);

	try (ConfigurableApplicationContext context = app.run(
			"--server.port=0",
			"--spring.jmx.enabled=false",
			"--spring.cloud.stream.function.definition=process;analyze",
			"--spring.cloud.stream.bindings.process-in-0.destination=purchases",
			"--spring.cloud.stream.kafka.streams.bindings.process-in-0.consumer.startOffset=latest",
			"--spring.cloud.stream.bindings.process-in-0.binder=kafka1",
			"--spring.cloud.stream.bindings.process-out-0.destination=coffee",
			"--spring.cloud.stream.bindings.process-out-0.binder=kafka1",
			"--spring.cloud.stream.bindings.process-out-1.destination=electronics",
			"--spring.cloud.stream.bindings.process-out-1.binder=kafka1",
			"--spring.cloud.stream.bindings.analyze-in-0.destination=coffee",
			"--spring.cloud.stream.bindings.analyze-in-0.binder=kafka2",
			"--spring.cloud.stream.bindings.analyze-in-1.destination=electronics",
			"--spring.cloud.stream.bindings.analyze-in-1.binder=kafka2",
			"--spring.cloud.stream.bindings.analyze-in-0.consumer.concurrency=2",
			"--spring.cloud.stream.binders.kafka1.type=kstream",
			"--spring.cloud.stream.binders.kafka1.environment.spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString(),
			"--spring.cloud.stream.binders.kafka1.environment.spring.cloud.stream.kafka.streams.binder.applicationId=my-app-1",
			"--spring.cloud.stream.binders.kafka1.environment.spring.cloud.stream.kafka.streams.binder.configuration.client.id=process-client",
			"--spring.cloud.stream.binders.kafka2.type=kstream",
			"--spring.cloud.stream.binders.kafka2.environment.spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString(),
			"--spring.cloud.stream.binders.kafka2.environment.spring.cloud.stream.kafka.streams.binder.applicationId=my-app-2",
			"--spring.cloud.stream.binders.kafka2.environment.spring.cloud.stream.kafka.streams.binder.configuration.client.id=analyze-client")) {

		Thread.sleep(1000);
		receiveAndValidate("purchases", "coffee", "electronics");

		StreamsBuilderFactoryBean processStreamsBuilderFactoryBean = context
				.getBean("&stream-builder-process", StreamsBuilderFactoryBean.class);

		StreamsBuilderFactoryBean analyzeStreamsBuilderFactoryBean = context
				.getBean("&stream-builder-analyze", StreamsBuilderFactoryBean.class);

		final Properties processStreamsConfiguration = processStreamsBuilderFactoryBean.getStreamsConfiguration();
		final Properties analyzeStreamsConfiguration = analyzeStreamsBuilderFactoryBean.getStreamsConfiguration();

		assertThat(processStreamsConfiguration.getProperty("application.id")).isEqualTo("my-app-1");
		assertThat(analyzeStreamsConfiguration.getProperty("application.id")).isEqualTo("my-app-2");
		assertThat(processStreamsConfiguration.getProperty("client.id")).isEqualTo("process-client");
		assertThat(analyzeStreamsConfiguration.getProperty("client.id")).isEqualTo("analyze-client");

		Integer concurrency = (Integer) analyzeStreamsConfiguration.get(StreamsConfig.NUM_STREAM_THREADS_CONFIG);
		assertThat(concurrency).isEqualTo(2);

		concurrency = (Integer) processStreamsConfiguration.get(StreamsConfig.NUM_STREAM_THREADS_CONFIG);
		assertThat(concurrency).isNull(); //thus default to 1 by Kafka Streams.
	}
}

Example #14

Source File: MultipleFunctionsInSameAppTests.java From spring-cloud-stream-binder-kafka with Apache License 2.0

4 votes

@Test
public void testMultiFunctionsInSameApp() throws InterruptedException {
	SpringApplication app = new SpringApplication(MultipleFunctionsInSameApp.class);
	app.setWebApplicationType(WebApplicationType.NONE);

	try (ConfigurableApplicationContext context = app.run(
			"--server.port=0",
			"--spring.jmx.enabled=false",
			"--spring.cloud.stream.function.definition=process;analyze;anotherProcess",
			"--spring.cloud.stream.bindings.process-in-0.destination=purchases",
			"--spring.cloud.stream.bindings.process-out-0.destination=coffee",
			"--spring.cloud.stream.bindings.process-out-1.destination=electronics",
			"--spring.cloud.stream.bindings.analyze-in-0.destination=coffee",
			"--spring.cloud.stream.bindings.analyze-in-1.destination=electronics",
			"--spring.cloud.stream.kafka.streams.binder.functions.analyze.applicationId=analyze-id-0",
			"--spring.cloud.stream.kafka.streams.binder.functions.process.applicationId=process-id-0",
			"--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000",
			"--spring.cloud.stream.bindings.process-in-0.consumer.concurrency=2",
			"--spring.cloud.stream.bindings.analyze-in-0.consumer.concurrency=1",
			"--spring.cloud.stream.kafka.streams.binder.configuration.num.stream.threads=3",
			"--spring.cloud.stream.kafka.streams.binder.functions.process.configuration.client.id=process-client",
			"--spring.cloud.stream.kafka.streams.binder.functions.analyze.configuration.client.id=analyze-client",
			"--spring.cloud.stream.kafka.streams.binder.functions.anotherProcess.configuration.client.id=anotherProcess-client",
			"--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) {
		receiveAndValidate("purchases", "coffee", "electronics");

		StreamsBuilderFactoryBean processStreamsBuilderFactoryBean = context
				.getBean("&stream-builder-process", StreamsBuilderFactoryBean.class);

		StreamsBuilderFactoryBean analyzeStreamsBuilderFactoryBean = context
				.getBean("&stream-builder-analyze", StreamsBuilderFactoryBean.class);

		StreamsBuilderFactoryBean anotherProcessStreamsBuilderFactoryBean = context
				.getBean("&stream-builder-anotherProcess", StreamsBuilderFactoryBean.class);

		final Properties processStreamsConfiguration = processStreamsBuilderFactoryBean.getStreamsConfiguration();
		final Properties analyzeStreamsConfiguration = analyzeStreamsBuilderFactoryBean.getStreamsConfiguration();
		final Properties anotherProcessStreamsConfiguration = anotherProcessStreamsBuilderFactoryBean.getStreamsConfiguration();

		assertThat(processStreamsConfiguration.getProperty("client.id")).isEqualTo("process-client");
		assertThat(analyzeStreamsConfiguration.getProperty("client.id")).isEqualTo("analyze-client");

		Integer concurrency = (Integer) processStreamsConfiguration.get(StreamsConfig.NUM_STREAM_THREADS_CONFIG);
		assertThat(concurrency).isEqualTo(2);
		concurrency = (Integer) analyzeStreamsConfiguration.get(StreamsConfig.NUM_STREAM_THREADS_CONFIG);
		assertThat(concurrency).isEqualTo(1);
		assertThat(anotherProcessStreamsConfiguration.get(StreamsConfig.NUM_STREAM_THREADS_CONFIG)).isEqualTo("3");
	}
}

Example #15

Source File: KafkaStreamsBinderWordCountIntegrationTests.java From spring-cloud-stream-binder-kafka with Apache License 2.0

4 votes

@Test
public void testSendToTombstone()
		throws Exception {
	SpringApplication app = new SpringApplication(
			WordCountProcessorApplication.class);
	app.setWebApplicationType(WebApplicationType.NONE);

	try (ConfigurableApplicationContext context = app.run("--server.port=0",
			"--spring.jmx.enabled=false",
			"--spring.cloud.stream.bindings.input.destination=words-1",
			"--spring.cloud.stream.bindings.output.destination=counts-1",
			"--spring.cloud.stream.kafka.streams.bindings.input.consumer.application-id=testKstreamWordCountWithInputBindingLevelApplicationId",
			"--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000",
			"--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde"
					+ "=org.apache.kafka.common.serialization.Serdes$StringSerde",
			"--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde"
					+ "=org.apache.kafka.common.serialization.Serdes$StringSerde",
			"--spring.cloud.stream.kafka.streams.bindings.output.producer.valueSerde=org.springframework.kafka.support.serializer.JsonSerde",
			"--spring.cloud.stream.kafka.streams.timeWindow.length=5000",
			"--spring.cloud.stream.kafka.streams.timeWindow.advanceBy=0",
			"--spring.cloud.stream.bindings.input.consumer.concurrency=2",
			"--spring.cloud.stream.kafka.streams.binder.brokers="
					+ embeddedKafka.getBrokersAsString())) {
		receiveAndValidate("words-1", "counts-1");
		// Assertions on StreamBuilderFactoryBean
		StreamsBuilderFactoryBean streamsBuilderFactoryBean = context
				.getBean("&stream-builder-WordCountProcessorApplication-process", StreamsBuilderFactoryBean.class);
		KafkaStreams kafkaStreams = streamsBuilderFactoryBean.getKafkaStreams();
		assertThat(kafkaStreams).isNotNull();
		// Ensure that concurrency settings are mapped to number of stream task
		// threads in Kafka Streams.
		final Properties streamsConfiguration = streamsBuilderFactoryBean.getStreamsConfiguration();
		final Integer concurrency = (Integer) streamsConfiguration
				.get(StreamsConfig.NUM_STREAM_THREADS_CONFIG);
		assertThat(concurrency).isEqualTo(2);

		sendTombStoneRecordsAndVerifyGracefulHandling();

		CleanupConfig cleanup = TestUtils.getPropertyValue(streamsBuilderFactoryBean,
				"cleanupConfig", CleanupConfig.class);
		assertThat(cleanup.cleanupOnStart()).isTrue();
		assertThat(cleanup.cleanupOnStop()).isFalse();
	}
}

Example #16

Source File: KafkaStreamsBindingInformationCatalogue.java From spring-cloud-stream-binder-kafka with Apache License 2.0

4 votes

Set<StreamsBuilderFactoryBean> getStreamsBuilderFactoryBeans() {
	return this.streamsBuilderFactoryBeans;
}

Example #17

Source File: KafkaStreamsBinderMetrics.java From spring-cloud-stream-binder-kafka with Apache License 2.0

4 votes

public void addMetrics(Set<StreamsBuilderFactoryBean> streamsBuilderFactoryBeans) {
	synchronized (KafkaStreamsBinderMetrics.this) {
		this.bindTo(streamsBuilderFactoryBeans, this.meterRegistry);
	}
}

Example #18

Source File: KafkaStreamsRegistry.java From spring-cloud-stream-binder-kafka with Apache License 2.0

4 votes

public List<StreamsBuilderFactoryBean> streamsBuilderFactoryBeans() {
	return new ArrayList<>(this.streamsBuilderFactoryBeanMap.values());
}

Example #19

Source File: KafkaStreamsBindingInformationCatalogue.java From spring-cloud-stream-binder-kafka with Apache License 2.0

2 votes

/**
 * Adds a mapping for KStream -> {@link StreamsBuilderFactoryBean}.
 * @param streamsBuilderFactoryBean provides the {@link StreamsBuilderFactoryBean}
 * mapped to the KStream
 */
void addStreamBuilderFactory(StreamsBuilderFactoryBean streamsBuilderFactoryBean) {
	this.streamsBuilderFactoryBeans.add(streamsBuilderFactoryBean);
}

Example #20

Source File: KafkaStreamsRegistry.java From spring-cloud-stream-binder-kafka with Apache License 2.0

2 votes

/**
 *
 * @param kafkaStreams {@link KafkaStreams} object
 * @return Corresponding {@link StreamsBuilderFactoryBean}.
 */
StreamsBuilderFactoryBean streamBuilderFactoryBean(KafkaStreams kafkaStreams) {
	return this.streamsBuilderFactoryBeanMap.get(kafkaStreams);
}