org.apache.kafka.streams.state.HostInfo Java Examples

private List<KeyValue<String, DepartmentAggregate>> readAllFromRemote(
    HostInfo hostInfo) throws IOException {

    List<KeyValue<String, DepartmentAggregate>> remoteResults =
        new ArrayList<>();

    String targetHost = String.format("http://%s:%d/dept/local",
        hostInfo.host(),
        hostInfo.port());

    String result = client.target(targetHost)
        .request(MediaType.APPLICATION_JSON).get(String.class);

    if (!result.equals(NO_RESULTS)) {
        remoteResults = objectMapper.readValue(result,
            remoteResults.getClass());
    }
    return remoteResults;
}
 

@RequestMapping("/charts/top-five")
@SuppressWarnings("unchecked")
public List<SongPlayCountBean> topFive(@RequestParam(value="genre") String genre) {

	HostInfo hostInfo = interactiveQueryService.getHostInfo(KafkaStreamsInteractiveQuerySample.TOP_FIVE_SONGS_STORE,
			KafkaStreamsInteractiveQuerySample.TOP_FIVE_KEY, new StringSerializer());

	if (interactiveQueryService.getCurrentHostInfo().equals(hostInfo)) {
		logger.info("Top Five songs request served from same host: " + hostInfo);
		return topFiveSongs(KafkaStreamsInteractiveQuerySample.TOP_FIVE_KEY, KafkaStreamsInteractiveQuerySample.TOP_FIVE_SONGS_STORE);
	}
	else {
		//find the store from the proper instance.
		logger.info("Top Five songs request served from different host: " + hostInfo);
		RestTemplate restTemplate = new RestTemplate();
		return restTemplate.postForObject(
				String.format("http://%s:%d/%s", hostInfo.host(),
						hostInfo.port(), "charts/top-five?genre=Punk"), "punk", List.class);
	}
}
 

/**
 * get Metrics for a machine
 * @param machine
 * @return 
 */
private Metrics getLocalMetrics(String machine) {
    LOGGER.log(Level.INFO, "Getting Metrics for machine {0}", machine);
    
    HostInfo thisInstance = GlobalAppState.getInstance().getHostPortInfo();
    KafkaStreams ks = GlobalAppState.getInstance().getKafkaStreams();

    String source = thisInstance.host() + ":" + thisInstance.port();
    Metrics localMetrics = new Metrics();

    ReadOnlyKeyValueStore<String, Double> averageStore = ks
            .store(storeName,
                    QueryableStoreTypes.<String, Double>keyValueStore());

    LOGGER.log(Level.INFO, "Entries in store {0}", averageStore.approximateNumEntries());

    localMetrics.add(source, machine, String.valueOf(averageStore.get(machine)));

    LOGGER.log(Level.INFO, "Metrics for machine {0} - {1}", new Object[]{machine, localMetrics});
    return localMetrics;
}
 

/**
 * Metrics for a machine
 *
 * @param machine
 * @return the metric
 */
@GET
@Path("{machine}")
@Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML})
public Response getMachineMetric(@PathParam("machine") String machine) {
    LOGGER.log(Level.INFO, "Fetching metrics for machine {0}", machine);

    KafkaStreams ks = GlobalAppState.getInstance().getKafkaStreams();
    HostInfo thisInstance = GlobalAppState.getInstance().getHostPortInfo();

    Metrics metrics = null;

    StreamsMetadata metadataForMachine = ks.metadataForKey(storeName, machine, new StringSerializer());

    if (metadataForMachine.host().equals(thisInstance.host()) && metadataForMachine.port() == thisInstance.port()) {
        LOGGER.log(Level.INFO, "Querying local store for machine {0}", machine);
        metrics = getLocalMetrics(machine);
    } else {
        //LOGGER.log(Level.INFO, "Querying remote store for machine {0}", machine);
        String url = "http://" + metadataForMachine.host() + ":" + metadataForMachine.port() + "/metrics/remote/" + machine;
        metrics = Utils.getRemoteStoreState(url, 2, TimeUnit.SECONDS);
        LOGGER.log(Level.INFO, "Metric from remote store at {0} == {1}", new Object[]{url, metrics});
    }

    return Response.ok(metrics).build();
}
 

/**
 * Query local state store to extract metrics
 *
 * @return local Metrics
 */
private Metrics getLocalMetrics() {
    HostInfo thisInstance = GlobalAppState.getInstance().getHostPortInfo();
    KafkaStreams ks = GlobalAppState.getInstance().getKafkaStreams();

    String source = thisInstance.host() + ":" + thisInstance.port();
    Metrics localMetrics = new Metrics();

    ReadOnlyKeyValueStore<String, Double> averageStore = ks
            .store(storeName,
                    QueryableStoreTypes.<String, Double>keyValueStore());

    LOGGER.log(Level.INFO, "Entries in store {0}", averageStore.approximateNumEntries());
    KeyValueIterator<String, Double> storeIterator = averageStore.all();

    while (storeIterator.hasNext()) {
        KeyValue<String, Double> kv = storeIterator.next();
        localMetrics.add(source, kv.key, String.valueOf(kv.value));

    }
    LOGGER.log(Level.INFO, "Local store state {0}", localMetrics);
    return localMetrics;
}
 

private String fetchRemote(HostInfo hostInfo, String path, Map<String, String> params) {
    String store = params.get(STORE_PARAM);
    String key = params.get(KEY_PARAM);
    String from = params.get(FROM_PARAM);
    String to = params.get(TO_PARAM);

    String url;

    if (from != null && to != null) {
        url = String.format("http://%s:%d/%s/%s/%s/%s/%s", hostInfo.host(), hostInfo.port(), path, store, key, from, to);
    } else {
        url = String.format("http://%s:%d/%s/%s/%s", hostInfo.host(), hostInfo.port(), path, store, key);
    }

    String remoteResponseValue = "";

    try {
        remoteResponseValue = client.target(url).request(MediaType.APPLICATION_JSON_TYPE).get(String.class);
    } catch (Exception e) {
        LOG.error("Problem connecting " + e.getMessage());
    }
    return remoteResponseValue;
}
 

/**
 * @param streams                The {@link KafkaStreams} application
 * @param localApplicationServer The {@link HostInfo} derived from the
 *                               {@link StreamsConfig#APPLICATION_SERVER_CONFIG application.server}
 *                               configuration property of local kafka streams node for the streams application.
 *                               This is used to identify requests for local store, bypassing gRPC calls
 * @param storeName              The name of the {@link ReadOnlyKeyValueStore} registered in the streams application
 * @param keySerde               The {@link Serde} for keys of the store
 * @param valSerde               The {@link Serde} for values of the store
 * @param grpcChannelProvider    A function that establishes gRPC {@link Channel} to a remote store service
 *                               for the given {@link HostInfo} parameter
 * @param parallel               {@code true} if lookups that need to query many stores in the cluster are
 *                               to be performed in parallel
 * @param filterPredicate        filter predicate to filter out keys and values
 */
public DistributedReadOnlyKeyValueStore(
    KafkaStreams streams,
    HostInfo localApplicationServer,
    String storeName,
    Serde<K> keySerde, Serde<V> valSerde,
    Function<? super HostInfo, ? extends Channel> grpcChannelProvider,
    boolean parallel,
    FilterPredicate<K, V> filterPredicate
) {
    super(
        streams,
        localApplicationServer,
        storeName,
        keySerde,
        valSerde,
        grpcChannelProvider,
        parallel
    );
    this.filterPredicate = filterPredicate;
}
 

private S serviceForHostInfo(HostInfo hostInfo) {
    return hostInfo2service.computeIfAbsent(
        hostInfo,
        hInfo -> {
            if (localApplicationServer.equals(hInfo)) {
                log.info("Obtaining local service '{}' for host info '{}'", storeName, hInfo);
                // use local store if host info is local host info
                return localService(storeName, streams);
            } else {
                log.info("Obtaining remote service '{}' for host info '{}'", storeName, hInfo);
                // connect to remote for other host info(s)
                return remoteServiceGrpcClient(storeName, grpcChannelProvider.apply(hInfo), keySerde);
            }
        }
    );
}
 

@Produces
@ApplicationScoped
public ReadOnlyKeyValueStore<Long, Str.TupleValue> globalIdKeyValueStore(
    KafkaStreams streams,
    HostInfo storageLocalHost,
    StreamsProperties properties
) {
    return new DistributedReadOnlyKeyValueStore<>(
        streams,
        storageLocalHost,
        properties.getGlobalIdStoreName(),
        Serdes.Long(), ProtoSerde.parsedWith(Str.TupleValue.parser()),
        new DefaultGrpcChannelProvider(),
        true,
        (filter, over, id, tuple) -> true
    );
}
 

@Produces
@ApplicationScoped
public ExtReadOnlyKeyValueStore<String, Str.Data> storageKeyValueStore(
    KafkaStreams streams,
    HostInfo storageLocalHost,
    StreamsProperties properties,
    FilterPredicate<String, Str.Data> filterPredicate
) {
    return new DistributedReadOnlyKeyValueStore<>(
        streams,
        storageLocalHost,
        properties.getStorageStoreName(),
        Serdes.String(), ProtoSerde.parsedWith(Str.Data.parser()),
        new DefaultGrpcChannelProvider(),
        true,
        filterPredicate
    );
}
 

/**
 * @param streams                The {@link KafkaStreams} application
 * @param localApplicationServer The {@link HostInfo} derived from the
 *                               {@link StreamsConfig#APPLICATION_SERVER_CONFIG application.server}
 *                               configuration property of local kafka streams node for the streams application.
 *                               This is used to identify requests for local service, bypassing gRPC calls
 * @param storeName              The name of the store registered in the streams application and used for distribution
 *                               of keys among kafka streams processing nodes.
 * @param keySerde               the {@link Serde} for keys of the service which are also the distribution keys of the
 *                               corresponding store.
 * @param grpcChannelProvider    A function that establishes gRPC {@link Channel} to a remote service
 *                               for the given {@link HostInfo} parameter
 * @param parallel               {@code true} if service calls that need to dispatch to many local services in
 *                               the cluster are to be performed in parallel
 */
public DistributedService(
    KafkaStreams streams,
    HostInfo localApplicationServer,
    String storeName,
    Serde<K> keySerde,
    Function<? super HostInfo, ? extends Channel> grpcChannelProvider,
    boolean parallel
) {
    this.streams = Objects.requireNonNull(streams, "streams");
    this.localApplicationServer = Objects.requireNonNull(localApplicationServer, "localApplicationServer");
    this.storeName = Objects.requireNonNull(storeName, "storeName");
    this.keySerde = Objects.requireNonNull(keySerde, "keySerde");
    this.grpcChannelProvider = Objects.requireNonNull(grpcChannelProvider, "grpcChannelProvider");
    this.parallel = parallel;
}
 

@Produces
@ApplicationScoped
@Current
public AsyncBiFunctionService<String, Long, Str.Data> waitForDataUpdateService(
    StreamsProperties properties,
    KafkaStreams streams,
    HostInfo storageLocalHost,
    LocalService<AsyncBiFunctionService.WithSerdes<String, Long, Str.Data>> localWaitForDataUpdateService
) {
    return new DistributedAsyncBiFunctionService<>(
        streams,
        storageLocalHost,
        properties.getStorageStoreName(),
        localWaitForDataUpdateService,
        new DefaultGrpcChannelProvider()
    );
}
 

@Produces
@ApplicationScoped
@Current
public AsyncBiFunctionService<Void, Void, KafkaStreams.State> stateService(
    KafkaStreams streams,
    HostInfo storageLocalHost,
    LocalService<AsyncBiFunctionService.WithSerdes<Void, Void, KafkaStreams.State>> localStateService
) {
    return new DistributedAsyncBiFunctionService<>(
        streams,
        storageLocalHost,
        "stateStore",
        localStateService,
        new DefaultGrpcChannelProvider()
    );
}
 

@Override
public Channel apply(HostInfo hostInfo) {
    return ManagedChannelBuilder
        .forAddress(hostInfo.host(), hostInfo.port())
        .usePlaintext()
        .build();
}
 

private String fetchFromSessionStore(Map<String, String> params) {
    String store = params.get(STORE_PARAM);
    String key = params.get(KEY_PARAM);

    HostInfo storeHostInfo = getHostInfo(store, key);

    if (storeHostInfo.host().equals("unknown")) {
        return STORES_NOT_ACCESSIBLE;
    }

    if (dataNotLocal(storeHostInfo)) {
        LOG.info("{} located in state store on another instance !!!!", key);
        return fetchRemote(storeHostInfo, "session", params);
    }

    ReadOnlySessionStore<String, CustomerTransactions> readOnlySessionStore = kafkaStreams.store(store, QueryableStoreTypes.sessionStore());
    List<String> results = new ArrayList<>();
    List<KeyValue<String, List<String>>> sessionResults = new ArrayList<>();
    try (KeyValueIterator<Windowed<String>, CustomerTransactions> iterator = readOnlySessionStore.fetch(key)) {
        while (iterator.hasNext()) {
            KeyValue<Windowed<String>, CustomerTransactions> windowed = iterator.next();
            CustomerTransactions transactions = windowed.value;
            LocalDateTime startSession = getLocalDateTime(windowed.key.window().start());
            LocalDateTime endSession = getLocalDateTime(windowed.key.window().end());
            transactions.setSessionInfo(String.format("Session Window{start=%s, end=%s}", startSession.toLocalTime().toString(), endSession.toLocalTime().toString()));
            results.add(transactions.toString());
        }
        sessionResults.add(new KeyValue<>(key, results));
    }

    return gson.toJson(sessionResults);
}
 

private void receiveAndValidateFoo(ConfigurableApplicationContext context) {
	Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka);
	DefaultKafkaProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<>(
			senderProps);
	KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf, true);
	template.setDefaultTopic("foos");
	template.sendDefault("{\"id\":\"123\"}");
	ConsumerRecord<String, String> cr = KafkaTestUtils.getSingleRecord(consumer,
			"counts-id");
	assertThat(cr.value().contains("Count for product with ID 123: 1")).isTrue();

	ProductCountApplication.Foo foo = context
			.getBean(ProductCountApplication.Foo.class);
	assertThat(foo.getProductStock(123).equals(1L));

	// perform assertions on HostInfo related methods in InteractiveQueryService
	InteractiveQueryService interactiveQueryService = context
			.getBean(InteractiveQueryService.class);
	HostInfo currentHostInfo = interactiveQueryService.getCurrentHostInfo();

	assertThat(currentHostInfo.host() + ":" + currentHostInfo.port())
			.isEqualTo(embeddedKafka.getBrokersAsString());

	HostInfo hostInfo = interactiveQueryService.getHostInfo("prod-id-count-store",
			123, new IntegerSerializer());
	assertThat(hostInfo.host() + ":" + hostInfo.port())
			.isEqualTo(embeddedKafka.getBrokersAsString());

	HostInfo hostInfoFoo = interactiveQueryService
			.getHostInfo("prod-id-count-store-foo", 123, new IntegerSerializer());
	assertThat(hostInfoFoo).isNull();

	final List<HostInfo> hostInfos = interactiveQueryService.getAllHostsInfo("prod-id-count-store");
	assertThat(hostInfos.size()).isEqualTo(1);
	final HostInfo hostInfo1 = hostInfos.get(0);
	assertThat(hostInfo1.host() + ":" + hostInfo1.port())
			.isEqualTo(embeddedKafka.getBrokersAsString());

}
 

/**
 * Gets the current {@link HostInfo} that the calling kafka streams application is
 * running on.
 *
 * Note that the end user applications must provide `applicaiton.server` as a
 * configuration property when calling this method. If this is not available, then
 * null is returned.
 * @return the current {@link HostInfo}
 */
public HostInfo getCurrentHostInfo() {
	Map<String, String> configuration = this.binderConfigurationProperties
			.getConfiguration();
	if (configuration.containsKey("application.server")) {

		String applicationServer = configuration.get("application.server");
		String[] splits = StringUtils.split(applicationServer, ":");

		return new HostInfo(splits[0], Integer.valueOf(splits[1]));
	}
	return null;
}
 

@Test
public void success(TestContext context){

    ShareableStreamsMetadataProvider mock = mock(ShareableStreamsMetadataProvider.class);
    StreamsMetadata host = new StreamsMetadata(new HostInfo("host", 1), Collections.emptySet(), Collections.emptySet());
    when(mock.metadataForKey(anyString(), any(), any(Serializer.class))).thenReturn(host);
    rule.vertx().sharedData().getLocalMap("metadata").put("metadata", mock);

    rule.vertx().eventBus().consumer(Config.WINDOWED_QUERY_ADDRESS_PREFIX + "host", msg -> {
        SortedMap<Long, String> result = new TreeMap<>();

        result.put(1L, "abc");
        result.put(2L, "def");
        msg.reply(new WindowedQueryResponse(result ));
    });

    rule.vertx().deployVerticle(new KeyBasedQueryFacadeVerticle<WindowedQuery, WindowedQueryResponse>(Config.WINDOWED_QUERY_FACADE_ADDRESS, Config.WINDOWED_QUERY_ADDRESS_PREFIX), context.asyncAssertSuccess(deployment->{

        WindowedQuery query = new WindowedQuery("store", Serdes.String().getClass().getName(), "key".getBytes(),Serdes.String().getClass().getName(),  1, 2);

        rule.vertx().eventBus().send(Config.WINDOWED_QUERY_FACADE_ADDRESS, query, context.asyncAssertSuccess(reply ->{

            context.assertTrue(reply.body() instanceof WindowedQueryResponse);
            WindowedQueryResponse response = (WindowedQueryResponse) reply.body();
            context.assertEquals(2, response.getValues().size());
            context.assertTrue(response.getValues().containsKey(1L));
            context.assertEquals("abc", response.getValues().get(1L));
            context.assertTrue(response.getValues().containsKey(2L));
            context.assertEquals("def", response.getValues().get(2L));

        }));


    }));

}
 

private String readKeyFromRemote(String searchKey, HostInfo hostInfo) {
    String result;
    String targetHost = String.format("http://%s:%d/kv/%s",
        hostInfo.host(),
        hostInfo.port(),
        searchKey);

    result = client.target(targetHost)
        .request(MediaType.APPLICATION_JSON).get(String.class);

    return (result == null) ? NO_RESULTS
        : (result.equals(APPLICATION_NOT_ACTIVE)) ? APPLICATION_NOT_ACTIVE
        : result;
}
 

@Produces
@Singleton
public HostInfo storageLocalHost(StreamsProperties props) {
    String appServer = props.getApplicationServer();
    String[] hostPort = appServer.split(":");
    log.info("Application server gRPC: '{}'", appServer);
    return new HostInfo(hostPort[0], Integer.parseInt(hostPort[1]));
}
 

/**
 * Local interface for fetching metrics
 *
 * @return Metrics from local and other remote stores (if needed)
 * @throws Exception
 */
@GET
@Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML})
public Response all_metrics() throws Exception {
    Response response = null;
    try {
        KafkaStreams ks = GlobalAppState.getInstance().getKafkaStreams();
        HostInfo thisInstance = GlobalAppState.getInstance().getHostPortInfo();

        LOGGER.info("Querying local store");
        Metrics metrics = getLocalMetrics();

        // LOGGER.info("Querying remote stores....");
        ks.allMetadataForStore(storeName)
                .stream()
                .filter(sm -> !(sm.host().equals(thisInstance.host()) && sm.port() == thisInstance.port())) //only query remote node stores
                .forEach(new Consumer<StreamsMetadata>() {
                    @Override
                    public void accept(StreamsMetadata t) {
                        String url = "http://" + t.host() + ":" + t.port() + "/metrics/remote";
                        //LOGGER.log(Level.INFO, "Fetching remote store at {0}", url);
                        Metrics remoteMetrics = Utils.getRemoteStoreState(url, 2, TimeUnit.SECONDS);
                        metrics.add(remoteMetrics);
                        LOGGER.log(Level.INFO, "Metric from remote store at {0} == {1}", new Object[]{url, remoteMetrics});
                    }

                });

        LOGGER.log(Level.INFO, "Complete store state {0}", metrics);
        response = Response.ok(metrics).build();
    } catch (Exception e) {
        LOGGER.log(Level.SEVERE, "Error - {0}", e.getMessage());
        e.printStackTrace();
    }

    return response;
}
 

public DistributedAsyncBiFunctionService(
    KafkaStreams streams,
    HostInfo localApplicationServer,
    String storeName, // used for distributing services by key (local service needs access to this local store)
    LocalService<? extends AsyncBiFunctionService.WithSerdes<K, REQ, RES>> localService,
    Function<? super HostInfo, ? extends Channel> grpcChannelProvider
) {
    super(streams, localApplicationServer, storeName, localService.getService().keySerde(), grpcChannelProvider, false);
    this.serviceName = localService.getServiceName();
    this.reqSerde = localService.getService().reqSerde();
    this.resSerde = localService.getService().resSerde();
    this.localService = localService.getService();
}
 

public HostInfo getHostPortInfo() {
    return this.hostInfo;
}
 

@Test
public void forwardingFailureDuringInstanceLookup(TestContext context){

    ShareableStreamsMetadataProvider mock = mock(ShareableStreamsMetadataProvider.class);
    StreamsMetadata host = new StreamsMetadata(new HostInfo("host", 1), Collections.emptySet(), Collections.emptySet());
    when(mock.metadataForKey(anyString(), any(), any(Serializer.class))).thenReturn(null);
    rule.vertx().sharedData().getLocalMap("metadata").put("metadata", mock);


    rule.vertx().deployVerticle(new KeyBasedQueryFacadeVerticle<KeyBasedQuery, ScalarKeyValueQueryResponse>(Config.KEY_VALUE_QUERY_FACADE_ADDRESS, Config.KEY_VALUE_QUERY_ADDRESS_PREFIX), context.asyncAssertSuccess(deployment->{

        KeyBasedQuery query = new KeyBasedQuery("store", Serdes.String().getClass().getName(), "key".getBytes(), Serdes.String().getClass().getName());

        rule.vertx().eventBus().send(Config.KEY_VALUE_QUERY_FACADE_ADDRESS, query, context.asyncAssertFailure(handler ->{

            context.assertTrue(handler instanceof ReplyException);
            ReplyException ex = (ReplyException) handler;
            context.assertEquals(404, ex.failureCode());

        }));
    }));

}
 

@Test
public void forwardingFailureDuringInstanceLookup(TestContext context){

    ShareableStreamsMetadataProvider mock = mock(ShareableStreamsMetadataProvider.class);
    StreamsMetadata host = new StreamsMetadata(new HostInfo("host", 1), Collections.emptySet(), Collections.emptySet());
    when(mock.metadataForKey(anyString(), any(), any(Serializer.class))).thenReturn(null);
    rule.vertx().sharedData().getLocalMap("metadata").put("metadata", mock);


    rule.vertx().deployVerticle(new KeyBasedQueryFacadeVerticle<WindowedQuery, WindowedQueryResponse>(Config.WINDOWED_QUERY_FACADE_ADDRESS, Config.WINDOWED_QUERY_ADDRESS_PREFIX), context.asyncAssertSuccess(deployment->{

        WindowedQuery query = new WindowedQuery("store", Serdes.String().getClass().getName(), "key".getBytes(),Serdes.String().getClass().getName(),  1, 2);

        rule.vertx().eventBus().send(Config.WINDOWED_QUERY_FACADE_ADDRESS, query, context.asyncAssertFailure(handler ->{

            context.assertTrue(handler instanceof ReplyException);
            ReplyException ex = (ReplyException) handler;
            context.assertEquals(404, ex.failureCode());

        }));
    }));

}
 

public static void main(String[] args) {

        if(args.length < 2){
            LOG.error("Need to specify host, port");
            System.exit(1);
        }

        String host = args[0];
        int port = Integer.parseInt(args[1]);
        final HostInfo hostInfo = new HostInfo(host, port);

        Properties properties = getProperties();
        properties.put(StreamsConfig.APPLICATION_SERVER_CONFIG, host+":"+port);

        StreamsConfig streamsConfig = new StreamsConfig(properties);
        Serde<String> stringSerde = Serdes.String();
        Serde<Long> longSerde = Serdes.Long();
        Serde<StockTransaction> stockTransactionSerde = StreamsSerdes.StockTransactionSerde();
        WindowedSerializer<String> windowedSerializer = new WindowedSerializer<>(stringSerde.serializer());
        WindowedDeserializer<String> windowedDeserializer = new WindowedDeserializer<>(stringSerde.deserializer());
        Serde<Windowed<String>> windowedSerde = Serdes.serdeFrom(windowedSerializer, windowedDeserializer);
        Serde<CustomerTransactions> customerTransactionsSerde = StreamsSerdes.CustomerTransactionsSerde();

        Aggregator<String, StockTransaction, Integer> sharesAggregator = (k, v, i) -> v.getShares() + i;

        StreamsBuilder builder = new StreamsBuilder();

        // data is already coming in keyed
        KStream<String, StockTransaction> stockTransactionKStream = builder.stream(MockDataProducer.STOCK_TRANSACTIONS_TOPIC, Consumed.with(stringSerde, stockTransactionSerde)
                .withOffsetResetPolicy(Topology.AutoOffsetReset.LATEST));


        stockTransactionKStream.map((k,v) -> KeyValue.pair(v.getSector(), v))
                .groupByKey(Serialized.with(stringSerde, stockTransactionSerde))
                .count(Materialized.as("TransactionsBySector"))
                .toStream()
                .peek((k,v) -> LOG.info("Transaction count for {} {}", k, v))
                .to("sector-transaction-counts", Produced.with(stringSerde, longSerde));
        
        stockTransactionKStream.map((k,v) -> KeyValue.pair(v.getCustomerId(), v))
                .groupByKey(Serialized.with(stringSerde, stockTransactionSerde))
                .windowedBy(SessionWindows.with(TimeUnit.MINUTES.toMillis(60)).until(TimeUnit.MINUTES.toMillis(120)))
                .aggregate(CustomerTransactions::new,(k, v, ct) -> ct.update(v),
                        (k, ct, other)-> ct.merge(other),
                        Materialized.<String, CustomerTransactions, SessionStore<Bytes, byte[]>>as("CustomerPurchaseSessions")
                                .withKeySerde(stringSerde).withValueSerde(customerTransactionsSerde))
                .toStream()
                .peek((k,v) -> LOG.info("Session info for {} {}", k, v))
                .to("session-transactions", Produced.with(windowedSerde, customerTransactionsSerde));


        stockTransactionKStream.groupByKey(Serialized.with(stringSerde, stockTransactionSerde))
                .windowedBy(TimeWindows.of(10000))
                .aggregate(() -> 0, sharesAggregator,
                        Materialized.<String, Integer, WindowStore<Bytes, byte[]>>as("NumberSharesPerPeriod")
                                .withKeySerde(stringSerde)
                                .withValueSerde(Serdes.Integer()))
                .toStream().peek((k,v)->LOG.info("key is {} value is {}", k, v))
                .to("transaction-count", Produced.with(windowedSerde,Serdes.Integer()));


        KafkaStreams kafkaStreams = new KafkaStreams(builder.build(), streamsConfig);
        InteractiveQueryServer queryServer = new InteractiveQueryServer(kafkaStreams, hostInfo);
        StateRestoreHttpReporter restoreReporter = new StateRestoreHttpReporter(queryServer);

        queryServer.init();

        kafkaStreams.setGlobalStateRestoreListener(restoreReporter);

        kafkaStreams.setStateListener(((newState, oldState) -> {
            if (newState == KafkaStreams.State.RUNNING && oldState == KafkaStreams.State.REBALANCING) {
                LOG.info("Setting the query server to ready");
                queryServer.setReady(true);
            } else if (newState != KafkaStreams.State.RUNNING) {
                LOG.info("State not RUNNING, disabling the query server");
                queryServer.setReady(false);
            }
        }));

        kafkaStreams.setUncaughtExceptionHandler((t, e) -> {
            LOG.error("Thread {} had a fatal error {}", t, e, e);
            shutdown(kafkaStreams, queryServer);
        });


        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
            shutdown(kafkaStreams, queryServer);
        }));

        LOG.info("Stock Analysis KStream Interactive Query App Started");
        kafkaStreams.cleanUp();
        kafkaStreams.start();
    }
 

public GlobalAppState hostPortInfo(String host, String port){
    hostInfo = new HostInfo(host, Integer.parseInt(port));
    return this;
}
 

protected WebClient httpClient(HostInfo hostInfo) {
  return WebClient.of(httpBaseUrl.apply(hostInfo.host(), hostInfo.port()));
}
 

public InteractiveQueryServer(final KafkaStreams kafkaStreams, final HostInfo hostInfo) {
    this.kafkaStreams = kafkaStreams;
    this.hostInfo = hostInfo;
    staticFiles.location("/webserver");
    port(hostInfo.port());
}
 

@Produces
@ApplicationScoped
public Lifecycle storageGrpcServer(
    HostInfo storageLocalHost,
    KeyValueStoreGrpc.KeyValueStoreImplBase storageStoreGrpcImpl,
    AsyncBiFunctionServiceGrpc.AsyncBiFunctionServiceImplBase storageAsyncBiFunctionServiceGrpcImpl
) {

    UnknownStatusDescriptionInterceptor unknownStatusDescriptionInterceptor =
        new UnknownStatusDescriptionInterceptor(
            ImmutableMap.of(
                IllegalArgumentException.class, Status.INVALID_ARGUMENT,
                IllegalStateException.class, Status.FAILED_PRECONDITION,
                InvalidStateStoreException.class, Status.FAILED_PRECONDITION,
                Throwable.class, Status.INTERNAL
            )
        );

    Server server = ServerBuilder
        .forPort(storageLocalHost.port())
        .addService(
            ServerInterceptors.intercept(
                storageStoreGrpcImpl,
                unknownStatusDescriptionInterceptor
            )
        )
        .addService(
            ServerInterceptors.intercept(
                storageAsyncBiFunctionServiceGrpcImpl,
                unknownStatusDescriptionInterceptor
            )
        )
        .build();

    return new Lifecycle() {
        @Override
        public void start() {
            try {
                server.start();
            } catch (IOException e) {
                throw new UncheckedIOException(e);
            }
        }

        @Override
        public void stop() {
            ConcurrentUtil
                .<Server>consumer(Server::awaitTermination)
                .accept(server.shutdown());
        }

        @Override
        public boolean isRunning() {
            return !(server.isShutdown() || server.isTerminated());
        }
    };
}