Java Code Examples for org.apache.kafka.streams.StreamsBuilder#table()

public Topology buildTopology(Properties envProps) {
    final StreamsBuilder builder = new StreamsBuilder();
    final String albumTopic = envProps.getProperty("album.topic.name");
    final String userTrackPurchaseTopic = envProps.getProperty("tracks.purchase.topic.name");
    final String musicInterestTopic = envProps.getProperty("music.interest.topic.name");

    final Serde<Long> longSerde = getPrimitiveAvroSerde(envProps, true);
    final Serde<MusicInterest> musicInterestSerde = getSpecificAvroSerde(envProps);
    final Serde<Album> albumSerde = getSpecificAvroSerde(envProps);
    final Serde<TrackPurchase> trackPurchaseSerde = getSpecificAvroSerde(envProps);

    final KTable<Long, Album> albums = builder.table(albumTopic, Consumed.with(longSerde, albumSerde));

    final KTable<Long, TrackPurchase> trackPurchases = builder.table(userTrackPurchaseTopic, Consumed.with(longSerde, trackPurchaseSerde));
    final MusicInterestJoiner trackJoiner = new MusicInterestJoiner();

    final KTable<Long, MusicInterest> musicInterestTable = trackPurchases.join(albums,
                                                                         TrackPurchase::getAlbumId,
                                                                         trackJoiner);

    musicInterestTable.toStream().to(musicInterestTopic, Produced.with(longSerde, musicInterestSerde));

    return builder.build();
}
 

public static <K, VV, EV> CompletableFuture<Map<TopicPartition, Long>> groupEdgesBySourceAndRepartition(
    StreamsBuilder builder,
    Properties streamsConfig,
    String initialVerticesTopic,
    String initialEdgesTopic,
    GraphSerialized<K, VV, EV> serialized,
    String verticesTopic,
    String edgesGroupedBySourceTopic,
    int numPartitions,
    short replicationFactor
) {
    KGraph<K, VV, EV> graph = new KGraph<>(
        builder.table(initialVerticesTopic, Consumed.with(serialized.keySerde(), serialized.vertexValueSerde())),
        builder.table(initialEdgesTopic, Consumed.with(new KryoSerde<>(), serialized.edgeValueSerde())),
        serialized);
    return groupEdgesBySourceAndRepartition(builder, streamsConfig, graph, verticesTopic, edgesGroupedBySourceTopic, numPartitions, replicationFactor);
}
 

public static <K, V> KTable<K, V> tableFromCollection(
    StreamsBuilder builder,
    Properties props,
    String topic,
    int numPartitions,
    short replicationFactor,
    Serde<K> keySerde,
    Serde<V> valueSerde,
    Collection<KeyValue<K, V>> values) {

    ClientUtils.createTopic(topic, numPartitions, replicationFactor, props);
    try (Producer<K, V> producer = new KafkaProducer<>(props, keySerde.serializer(), valueSerde.serializer())) {
        for (KeyValue<K, V> value : values) {
            ProducerRecord<K, V> producerRecord = new ProducerRecord<>(topic, value.key, value.value);
            producer.send(producerRecord);
        }
        producer.flush();
    }
    return builder.table(topic, Consumed.with(keySerde, valueSerde), Materialized.with(keySerde, valueSerde));
}
 

private KTable<?, ?> getKTable(KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties,
		StreamsBuilder streamsBuilder, Serde<?> keySerde,
		Serde<?> valueSerde, String materializedAs, String bindingDestination,
		Topology.AutoOffsetReset autoOffsetReset) {
	final Consumed<?, ?> consumed = getConsumed(kafkaStreamsConsumerProperties, keySerde, valueSerde, autoOffsetReset);
	return materializedAs != null
			? materializedAs(streamsBuilder, bindingDestination, materializedAs,
			keySerde, valueSerde, autoOffsetReset, kafkaStreamsConsumerProperties)
			: streamsBuilder.table(bindingDestination,
			consumed);
}
 

public Topology getTopology() {
    final StreamsBuilder builder = new StreamsBuilder();

    // Click Events
    final KStream<Integer, ClickEvent> clickEvents = builder.stream(this.clickInputTopic,
            Consumed.with(Serdes.Integer(), new JsonSerde<>(ClickEvent.class)));

    final KTable<Windowed<Integer>, Long> counts = clickEvents
            .selectKey(((key, value) -> value.getStatus()))
            .filter(((key, value) -> key >= 400))
            .groupByKey(Grouped.with(Serdes.Integer(), new JsonSerde<>(ClickEvent.class)))
            .windowedBy(TimeWindows.of(Duration.ofMinutes(1)))  // 1 Minute in ms
            .count();

    // Status codes
    final KTable<Integer, StatusCode> statusCodes = builder.table(this.statusInputTopic,
            Consumed.with(Serdes.Integer(), new JsonSerde<>(StatusCode.class)));

    // Join
    final KStream<Integer, ErrorOutput> errors = counts.toStream()
            .map((key, value) -> KeyValue.pair(
                    key.key(),
                    new ErrorOutput(key.key(), value, key.window().start(), null /*empty definition*/)))
            .join(statusCodes,
                    (countRecord, code) -> new ErrorOutput(
                            countRecord.getStatusCode(), countRecord.getCount(), countRecord.getTime(), code.getDefinition()),
                    Joined.valueSerde(new JsonSerde<>(ErrorOutput.class)));
    errors.to(this.errorOutputTopic);

    // Send alert if more than 5x a certain error code per minute
    errors.filter((key, errorOutput) -> errorOutput.getCount() > 5L).to(this.alertTopic);

    return builder.build();
}
 

private <K, V> KTable<K, V> materializedAs(StreamsBuilder streamsBuilder, String destination, String storeName,
		Serde<K> k, Serde<V> v, Topology.AutoOffsetReset autoOffsetReset, KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties) {

	final Consumed<K, V> consumed = getConsumed(kafkaStreamsConsumerProperties, k, v, autoOffsetReset);
	return streamsBuilder.table(this.bindingServiceProperties.getBindingDestination(destination),
			consumed, getMaterialized(storeName, k, v));
}
 

@Produces
public Topology buildTopology() {
    StreamsBuilder builder = new StreamsBuilder();

    ObjectMapperSerde<Category> categorySerde = new ObjectMapperSerde<>(Category.class);
    ObjectMapperSerde<Customer> customerSerde = new ObjectMapperSerde<>(Customer.class);
    ObjectMapperSerde<EnrichedCustomer> enrichedCustomerSerde = new ObjectMapperSerde<>(EnrichedCustomer.class);

    KTable<Integer, Category> categories = builder.table(
            "streams-test-categories",
            Consumed.with(Serdes.Integer(), categorySerde));

    KStream<Integer, EnrichedCustomer> customers = builder
            .stream("streams-test-customers", Consumed.with(Serdes.Integer(), customerSerde))
            .selectKey((id, customer) -> customer.category)
            .join(
                    categories,
                    (customer, category) -> {
                        return new EnrichedCustomer(customer.id, customer.name, category);
                    },
                    Joined.with(Serdes.Integer(), customerSerde, categorySerde));

    KeyValueBytesStoreSupplier storeSupplier = Stores.inMemoryKeyValueStore("countstore");
    customers.groupByKey()
            .count(Materialized.<Integer, Long> as(storeSupplier));

    customers.selectKey((categoryId, customer) -> customer.id)
            .to("streams-test-customers-processed", Produced.with(Serdes.Integer(), enrichedCustomerSerde));

    return builder.build();
}
 

public static void main(String[] args) {
    Properties props = new Properties();
    props.put(StreamsConfig.APPLICATION_ID_CONFIG, AppConfigs.applicationID);
    props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, AppConfigs.bootstrapServers);
    props.put(StreamsConfig.STATE_DIR_CONFIG, AppConfigs.stateStoreLocation);

    StreamsBuilder streamsBuilder = new StreamsBuilder();
    KTable<String, String> KT0 = streamsBuilder.table(
        AppConfigs.topicName,
        Consumed.with(Serdes.String(),Serdes.String()));

    KGroupedTable<String, String> KGT1 = KT0.groupBy(
        (person, age) -> KeyValue.pair(age, "1"),
        Serialized.with(Serdes.String(),Serdes.String())
    );

    KGT1.count()
        .toStream().peek((k, v) -> logger.info("Age=" + k + " Count=" + v));

    KafkaStreams myStream = new KafkaStreams(streamsBuilder.build(), props);
    myStream.start();

    Runtime.getRuntime().addShutdownHook(new Thread(() -> {
        logger.info("Stopping Stream");
        myStream.close();
    }));
}
 

public static void main(final String[] args) {
    final Properties props = new Properties();
    props.put(StreamsConfig.APPLICATION_ID_CONFIG, "StreamingTable");
    props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
    props.put(StreamsConfig.STATE_DIR_CONFIG, "state-store");
    props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG,
            Serdes.String().getClass().getName());
    props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG,
            Serdes.String().getClass().getName());
    props.put(StreamsConfig.DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG,
            WallclockTimestampExtractor.class.getName());

    //Uncomment to Enable record cache of size 10 MB.
    //props.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 10 * 1024 * 1024L);
    //Uncomment to Set commit interval to 1 second.
    //props.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000);

    StreamsBuilder streamBuilder = new StreamsBuilder();
    KTable<String, String> KT0 = streamBuilder.table("stock-tick");

    /*
    //Uncomment this block and comment next line to suppress
    KTable<String, String> KT1 = KT0.filter((key, value) -> key.contains("HDFCBANK"))
            .suppress(Suppressed.untilTimeLimit(
                    Duration.ofMinutes(5),
                    Suppressed.BufferConfig.maxBytes(1000000L).emitEarlyWhenFull())
            );
    */

    KTable<String, String> KT1 = KT0.filter((key, value) -> key.contains("HDFCBANK"));

    KStream<String, String> KS2 = KT1.toStream();
    KS2.peek((k, v) -> System.out.println("Key = " + k + " Value = " + v));

    KafkaStreams streams = new KafkaStreams(streamBuilder.build(), props);
    streams.start();
    Runtime.getRuntime().addShutdownHook(new Thread(streams::close));

}
 

public static void main(String[] args) {
    Properties properties = new Properties();
    properties.put(StreamsConfig.APPLICATION_ID_CONFIG,
        AppConfigs.applicationID);
    properties.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG,
        AppConfigs.bootstrapServers);
    properties.put(StreamsConfig.STATE_DIR_CONFIG,
        AppConfigs.stateStoreName);

    StreamsBuilder streamsBuilder = new StreamsBuilder();

    KTable<String, UserDetails> userDetailsKTable = streamsBuilder.table(
        AppConfigs.userMasterTopic,
        Consumed.with(AppSerdes.String(), AppSerdes.UserDetails())
    );

    KTable<String, UserLogin> userLoginKTable = streamsBuilder.table(
        AppConfigs.lastLoginTopic,
        Consumed.with(AppSerdes.String(), AppSerdes.UserLogin())
    );

    userLoginKTable.join(userDetailsKTable,
        (userLogin, userDetails) -> {
            userDetails.setLastLogin(userLogin.getCreatedTime());
            return userDetails;
        }
    ).toStream().to(AppConfigs.userMasterTopic,
        Produced.with(AppSerdes.String(), AppSerdes.UserDetails()));

    userDetailsKTable.toStream().foreach(
        (k, userDetails) -> logger.info(
            "Key = " + k + " Value = " + userDetails
        )
    );


    KafkaStreams streams = new KafkaStreams(streamsBuilder.build(), properties);
    streams.start();
    Runtime.getRuntime().addShutdownHook(new Thread(streams::close));
}
 

@Before
public void init() {
  functionRegistry = new FunctionRegistry();
  ksqlTable = (KsqlTable) metaStore.getSource("TEST2");
  StreamsBuilder builder = new StreamsBuilder();
  kTable = builder
          .table(ksqlTable.getKsqlTopic().getKafkaTopicName(), Consumed.with(Serdes.String()
              , ksqlTable.getKsqlTopic().getKsqlTopicSerDe().getGenericRowSerde(null, new
                  KsqlConfig(Collections.emptyMap()), false, new MockSchemaRegistryClient())));

}
 

public static void main(String[] args) {
    Properties props = new Properties();
    props.put(StreamsConfig.APPLICATION_ID_CONFIG, AppConfigs.applicationID);
    props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, AppConfigs.bootstrapServers);
    props.put(StreamsConfig.STATE_DIR_CONFIG, AppConfigs.stateStoreName);

    StreamsBuilder streamsBuilder = new StreamsBuilder();
    KTable<String, Employee> KS0 = streamsBuilder.table(AppConfigs.topicName,
        Consumed.with(AppSerdes.String(), AppSerdes.Employee()));

    KGroupedTable<String, Employee> KGS1 = KS0.groupBy(
        (k, v) -> KeyValue.pair(v.getDepartment(), v),
        Serialized.with(AppSerdes.String(), AppSerdes.Employee()));

    KTable<String, DepartmentAggregate> KT2 = KGS1.aggregate(
        //Initializer
        () -> new DepartmentAggregate()
            .withEmployeeCount(0)
            .withTotalSalary(0)
            .withAvgSalary(0D),
        //Adder
        (k, v, aggV) -> new DepartmentAggregate()
            .withEmployeeCount(aggV.getEmployeeCount() + 1)
            .withTotalSalary(aggV.getTotalSalary() + v.getSalary())
            .withAvgSalary((aggV.getTotalSalary() + v.getSalary())
                / (aggV.getEmployeeCount() + 1D)),
        //subtractor
        (k, v, aggV) -> new DepartmentAggregate()
            .withEmployeeCount(aggV.getEmployeeCount() - 1)
            .withTotalSalary(aggV.getTotalSalary() - v.getSalary())
            .withAvgSalary((aggV.getTotalSalary() - v.getSalary())
                / (aggV.getEmployeeCount() - 1D)),
        Materialized.<String, DepartmentAggregate, KeyValueStore<Bytes,
            byte[]>>as("dept-agg-store")
            .withValueSerde(AppSerdes.DepartmentAggregate())
    );

    KT2.toStream().foreach(
        (k, v) -> System.out.println("Key = " + k + " Value = " + v.toString()));

    KafkaStreams streams = new KafkaStreams(streamsBuilder.build(), props);
    streams.start();
    Runtime.getRuntime().addShutdownHook(new Thread(streams::close));
}
 

public static void main(String[] args) {

        if (args.length < 2) {
            System.out.println("Please provide command line arguments: hostname port");
            System.exit(-1);
        }
        String hostName = args[0];
        int portNumber = Integer.parseInt(args[1]);

        Properties props = new Properties();
        props.put(StreamsConfig.APPLICATION_ID_CONFIG, AppConfigs.applicationID);
        props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, AppConfigs.bootstrapServers);
        props.put(StreamsConfig.STATE_DIR_CONFIG, AppConfigs.stateStoreLocation);

        props.put(StreamsConfig.APPLICATION_SERVER_CONFIG, hostName
            + ":"
            + portNumber);

        StreamsBuilder streamsBuilder = new StreamsBuilder();
        KTable<String, Employee> KS0 = streamsBuilder.table(AppConfigs.topicName,
            Consumed.with(AppSerdes.String(), AppSerdes.Employee()));

        KGroupedTable<String, Employee> KGS1 = KS0.groupBy(
            (k, v) -> KeyValue.pair(v.getDepartment(), v),
            Grouped.with(AppSerdes.String(), AppSerdes.Employee()));

        KTable<String, DepartmentAggregate> KT2 = KGS1.aggregate(
            //Initializer
            () -> new DepartmentAggregate()
                .withEmployeeCount(0)
                .withTotalSalary(0)
                .withAvgSalary(0D),
            //Adder
            (k, v, aggV) -> new DepartmentAggregate()
                .withEmployeeCount(aggV.getEmployeeCount() + 1)
                .withTotalSalary(aggV.getTotalSalary() + v.getSalary())
                .withAvgSalary((aggV.getTotalSalary() + v.getSalary())
                    / (aggV.getEmployeeCount() + 1D)),
            //subtract
            (k, v, aggV) -> new DepartmentAggregate()
                .withEmployeeCount(aggV.getEmployeeCount() - 1)
                .withTotalSalary(aggV.getTotalSalary() - v.getSalary())
                .withAvgSalary((aggV.getTotalSalary() - v.getSalary())
                    / (aggV.getEmployeeCount() - 1D)),
            Materialized.<String, DepartmentAggregate, KeyValueStore<Bytes,
                byte[]>>as(AppConfigs.aggStateStoreName)
                .withKeySerde(AppSerdes.String())
                .withValueSerde(AppSerdes.DepartmentAggregate())
        );

        KT2.toStream().foreach(
            (k, v) -> System.out.println("Key = " + k + " Value = " + v.toString()));

        KafkaStreams streams = new KafkaStreams(
            streamsBuilder.build(), props);

        AppRestService queryServer = new AppRestService(
            streams,
            hostName,
            portNumber);

        streams.setStateListener((newState, oldState) -> {
            logger.info("State Changing to " + newState + " from " + oldState);
            queryServer.setActive(
                newState == KafkaStreams.State.RUNNING &&
                    oldState == KafkaStreams.State.REBALANCING
            );
        });

        streams.start();
        queryServer.start();

        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
            queryServer.stop();
            streams.close();
        }));
    }
 

public void prepare(StreamsBuilder builder, Properties streamsConfig) {
    Properties producerConfig = ClientUtils.producerConfig(
        bootstrapServers, serialized.keySerde().serializer().getClass(), KryoSerializer.class,
        streamsConfig != null ? streamsConfig : new Properties()
    );
    producerConfig.setProperty(ProducerConfig.CLIENT_ID_CONFIG, applicationId + "-producer");
    this.producer = new KafkaProducer<>(producerConfig);

    final StoreBuilder<KeyValueStore<Integer, Map<K, Map<K, List<Message>>>>> workSetStoreBuilder =
        Stores.keyValueStoreBuilder(Stores.persistentKeyValueStore(localworkSetStoreName),
            Serdes.Integer(), new KryoSerde<>()
        );
    builder.addStateStore(workSetStoreBuilder);

    final StoreBuilder<KeyValueStore<K, Tuple4<Integer, VV, Integer, VV>>> solutionSetStoreBuilder =
        Stores.keyValueStoreBuilder(Stores.persistentKeyValueStore(localSolutionSetStoreName),
            serialized.keySerde(), new KryoSerde<>()
        );
    builder.addStateStore(solutionSetStoreBuilder);

    this.vertices = builder
        .table(
            verticesTopic,
            Materialized.<K, VV, KeyValueStore<Bytes, byte[]>>as(verticesStoreName)
                .withKeySerde(serialized.keySerde()).withValueSerde(serialized.vertexValueSerde())
        );

    this.edgesGroupedBySource = builder
        .table(
            edgesGroupedBySourceTopic,
            Materialized.<K, Map<K, EV>, KeyValueStore<Bytes, byte[]>>as(edgesStoreName)
                .withKeySerde(serialized.keySerde()).withValueSerde(new KryoSerde<>())
        );

    this.solutionSet = builder
        .table(solutionSetTopic, Consumed.<K, Tuple4<Integer, VV, Integer, VV>>with(serialized.keySerde(), new KryoSerde<>()))
        .mapValues(v -> v._4, Materialized.as(solutionSetStore));

    // Initalize solution set
    this.vertices
        .toStream()
        .mapValues(v -> new Tuple4<>(-1, v, 0, v))
        .to(solutionSetTopic, Produced.with(serialized.keySerde(), new KryoSerde<>()));

    // Initialize workset
    this.vertices
        .toStream()
        .peek((k, v) -> {
            try {
                int partition = PregelComputation.vertexToPartition(k, serialized.keySerde().serializer(), numPartitions);
                ZKUtils.addChild(curator, applicationId, new PregelState(State.CREATED, 0, Stage.SEND), childPath(partition));
            } catch (Exception e) {
                throw toRuntimeException(e);
            }

        })
        .mapValues((k, v) -> new Tuple3<>(0, k, initialMessage.map(Collections::singletonList).orElse(Collections.emptyList())))
        .peek((k, v) -> log.trace("workset 0 before topic: (" + k + ", " + v + ")"))
        .<K, Tuple3<Integer, K, List<Message>>>to(workSetTopic, Produced.with(serialized.keySerde(), new KryoSerde<>()));

    this.workSet = builder
        .stream(workSetTopic, Consumed.with(serialized.keySerde(), new KryoSerde<Tuple3<Integer, K, List<Message>>>()))
        .peek((k, v) -> log.trace("workset 1 after topic: (" + k + ", " + v + ")"));

    KStream<K, Tuple2<Integer, Map<K, List<Message>>>> syncedWorkSet = workSet
        .transform(BarrierSync::new, localworkSetStoreName)
        .peek((k, v) -> log.trace("workset 2 after join: (" + k + ", " + v + ")"));

    KStream<K, Tuple3<Integer, Tuple4<Integer, VV, Integer, VV>, Map<K, List<Message>>>> superstepComputation =
        syncedWorkSet
            .transformValues(VertexComputeUdf::new, localSolutionSetStoreName, vertices.queryableStoreName(),
                edgesGroupedBySource.queryableStoreName());

    // Compute the solution set delta
    KStream<K, Tuple4<Integer, VV, Integer, VV>> solutionSetDelta = superstepComputation
        .flatMapValues(v -> v._2 != null ? Collections.singletonList(v._2) : Collections.emptyList())
        .peek((k, v) -> log.trace("solution set: (" + k + ", " + v + ")"));

    solutionSetDelta
        .to(solutionSetTopic, Produced.with(serialized.keySerde(), new KryoSerde<>()));

    // Compute the inbox of each vertex for the next step (new workset)
    KStream<K, Tuple2<Integer, Map<K, List<Message>>>> newworkSet = superstepComputation
        .mapValues(v -> new Tuple2<>(v._1, v._3))
        .peek((k, v) -> log.trace("workset new: (" + k + ", " + v + ")"));

    newworkSet.process(() -> new SendMessages(producer));
}
 

@Test
public void testVertexCountToValueFromFile() throws Exception {
    String suffix = "file";
    StreamsBuilder builder = new StreamsBuilder();

    Properties producerConfig = ClientUtils.producerConfig(CLUSTER.bootstrapServers(), KryoSerializer.class,
        FloatSerializer.class, new Properties()
    );
    GraphUtils.edgesToTopic(GraphUtils.class.getResourceAsStream("/ratings.txt"),
        Long::parseLong,
        Long::parseLong,
        s -> 0L,
        new LongSerializer(),
        producerConfig, "initEdges-" + suffix, 10, (short) 1);
    KTable<Edge<Long>, Long> edges =
        builder.table("initEdges-" + suffix, Consumed.with(new KryoSerde<>(), Serdes.Long()),
            Materialized.with(new KryoSerde<>(), Serdes.Long()));
    KGraph<Long, Long, Long> graph = KGraph.fromEdges(edges, new InitVertices(),
        GraphSerialized.with(Serdes.Long(), Serdes.Long(), Serdes.Long()));

    Properties props = ClientUtils.streamsConfig("prepare-" + suffix, "prepare-client-" + suffix,
        CLUSTER.bootstrapServers(), graph.keySerde().getClass(), graph.vertexValueSerde().getClass());
    CompletableFuture<Map<TopicPartition, Long>> state = GraphUtils.groupEdgesBySourceAndRepartition(
        builder, props, graph, "vertices-" + suffix, "edgesGroupedBySource-" + suffix, 10, (short) 1);
    Map<TopicPartition, Long> offsets = state.get();

    Thread.sleep(2000);

    algorithm =
        new PregelGraphAlgorithm<>(null, "run-" + suffix, CLUSTER.bootstrapServers(),
            CLUSTER.zKConnectString(), "vertices-" + suffix, "edgesGroupedBySource-" + suffix, offsets, graph.serialized(),
            "solutionSet-" + suffix, "solutionSetStore-" + suffix, "workSet-" + suffix, 10, (short) 1,
            Collections.emptyMap(), Optional.empty(), new VertexCountToValue<>());
    streamsConfiguration = ClientUtils.streamsConfig("run-" + suffix, "run-client-" + suffix,
        CLUSTER.bootstrapServers(), graph.keySerde().getClass(), KryoSerde.class);
    KafkaStreams streams = algorithm.configure(new StreamsBuilder(), streamsConfiguration).streams();
    GraphAlgorithmState<KTable<Long, Long>> paths = algorithm.run();
    paths.result().get();

    Map<Long, Long> map = StreamUtils.mapFromStore(paths.streams(), "solutionSetStore-" + suffix);
    log.info("result: {}", map);

    assertEquals(10L * 100000L + 10L * 10000L + 2071L, map.values().iterator().next().longValue());
}
 

@Test
public void testEdgeCountToValueFromFile() throws Exception {
    String suffix = "file";
    StreamsBuilder builder = new StreamsBuilder();

    Properties producerConfig = ClientUtils.producerConfig(CLUSTER.bootstrapServers(), KryoSerializer.class,
        FloatSerializer.class, new Properties()
    );
    GraphUtils.edgesToTopic(GraphUtils.class.getResourceAsStream("/ratings.txt"),
        Long::parseLong,
        Long::parseLong,
        s -> 0L,
        new LongSerializer(),
        producerConfig, "initEdges-" + suffix, 10, (short) 1);
    KTable<Edge<Long>, Long> edges =
        builder.table("initEdges-" + suffix, Consumed.with(new KryoSerde<>(), Serdes.Long()),
            Materialized.with(new KryoSerde<>(), Serdes.Long()));
    KGraph<Long, Long, Long> graph = KGraph.fromEdges(edges, new InitVertices(),
        GraphSerialized.with(Serdes.Long(), Serdes.Long(), Serdes.Long()));

    Properties props = ClientUtils.streamsConfig("prepare-" + suffix, "prepare-client-" + suffix,
        CLUSTER.bootstrapServers(), graph.keySerde().getClass(), graph.vertexValueSerde().getClass());
    CompletableFuture<Map<TopicPartition, Long>> state = GraphUtils.groupEdgesBySourceAndRepartition(
        builder, props, graph, "vertices-" + suffix, "edgesGroupedBySource-" + suffix, 10, (short) 1);
    Map<TopicPartition, Long> offsets = state.get();

    Thread.sleep(2000);

    algorithm =
        new PregelGraphAlgorithm<>(null, "run-" + suffix, CLUSTER.bootstrapServers(),
            CLUSTER.zKConnectString(), "vertices-" + suffix, "edgesGroupedBySource-" + suffix, offsets, graph.serialized(),
            "solutionSet-" + suffix, "solutionSetStore-" + suffix, "workSet-" + suffix, 10, (short) 1,
            Collections.emptyMap(), Optional.empty(), new EdgeCountToValue<>());
    streamsConfiguration = ClientUtils.streamsConfig("run-" + suffix, "run-client-" + suffix,
        CLUSTER.bootstrapServers(), graph.keySerde().getClass(), KryoSerde.class);
    KafkaStreams streams = algorithm.configure(new StreamsBuilder(), streamsConfiguration).streams();
    GraphAlgorithmState<KTable<Long, Long>> paths = algorithm.run();
    paths.result().get();

    Map<Long, Long> map = StreamUtils.mapFromStore(paths.streams(), "solutionSetStore-" + suffix);
    log.info("result: {}", map);

    assertEquals(35494L, map.values().iterator().next().longValue());
}
 

static <K, A> KTable<K, AggregateUpdate<A>> aggregateTable(TopologyContext<K, ?, ?, A> ctx, final StreamsBuilder builder) {
    return builder.table(ctx.topicName(AGGREGATE), Consumed.with(ctx.serdes().aggregateKey(), ctx.serdes().aggregateUpdate()));
}
 

public static void main(String[] args) throws Exception {


        StreamsConfig streamsConfig = new StreamsConfig(getProperties());

        Serde<String> stringSerde = Serdes.String();
        Serde<StockTransaction> transactionSerde = StreamsSerdes.StockTransactionSerde();
        Serde<TransactionSummary> transactionKeySerde = StreamsSerdes.TransactionSummarySerde();

        StreamsBuilder builder = new StreamsBuilder();
        long twentySeconds = 1000 * 20;
        long fifteenMinutes = 1000 * 60 * 15;
        long fiveSeconds = 1000 * 5;
        KTable<Windowed<TransactionSummary>, Long> customerTransactionCounts =
                 builder.stream(STOCK_TRANSACTIONS_TOPIC, Consumed.with(stringSerde, transactionSerde).withOffsetResetPolicy(LATEST))
                .groupBy((noKey, transaction) -> TransactionSummary.from(transaction),
                        Serialized.with(transactionKeySerde, transactionSerde))
                 // session window comment line below and uncomment another line below for a different window example
                .windowedBy(SessionWindows.with(twentySeconds).until(fifteenMinutes)).count();

                //The following are examples of different windows examples

                //Tumbling window with timeout 15 minutes
                //.windowedBy(TimeWindows.of(twentySeconds).until(fifteenMinutes)).count();

                //Tumbling window with default timeout 24 hours
                //.windowedBy(TimeWindows.of(twentySeconds)).count();

                //Hopping window 
                //.windowedBy(TimeWindows.of(twentySeconds).advanceBy(fiveSeconds).until(fifteenMinutes)).count();

        customerTransactionCounts.toStream().print(Printed.<Windowed<TransactionSummary>, Long>toSysOut().withLabel("Customer Transactions Counts"));

        KStream<String, TransactionSummary> countStream = customerTransactionCounts.toStream().map((window, count) -> {
                      TransactionSummary transactionSummary = window.key();
                      String newKey = transactionSummary.getIndustry();
                      transactionSummary.setSummaryCount(count);
                      return KeyValue.pair(newKey, transactionSummary);
        });

        KTable<String, String> financialNews = builder.table( "financial-news", Consumed.with(EARLIEST));


        ValueJoiner<TransactionSummary, String, String> valueJoiner = (txnct, news) ->
                String.format("%d shares purchased %s related news [%s]", txnct.getSummaryCount(), txnct.getStockTicker(), news);

        KStream<String,String> joined = countStream.leftJoin(financialNews, valueJoiner, Joined.with(stringSerde, transactionKeySerde, stringSerde));

        joined.print(Printed.<String, String>toSysOut().withLabel("Transactions and News"));



        KafkaStreams kafkaStreams = new KafkaStreams(builder.build(), streamsConfig);
        kafkaStreams.cleanUp();
        
        kafkaStreams.setUncaughtExceptionHandler((t, e) -> {
            LOG.error("had exception ", e);
        });
        CustomDateGenerator dateGenerator = CustomDateGenerator.withTimestampsIncreasingBy(Duration.ofMillis(750));
        
        DataGenerator.setTimestampGenerator(dateGenerator::get);
        
        MockDataProducer.produceStockTransactions(2, 5, 3, false);

        LOG.info("Starting CountingWindowing and KTableJoins Example");
        kafkaStreams.cleanUp();
        kafkaStreams.start();
        Thread.sleep(65000);
        LOG.info("Shutting down the CountingWindowing and KTableJoins Example Application now");
        kafkaStreams.close();
        MockDataProducer.shutdown();
    }
 

public static void main(String[] args) throws Exception {

        StreamsConfig streamsConfig = new StreamsConfig(getProperties());

        StreamsBuilder builder = new StreamsBuilder();


        KTable<String, StockTickerData> stockTickerTable = builder.table(STOCK_TICKER_TABLE_TOPIC);
        KStream<String, StockTickerData> stockTickerStream = builder.stream(STOCK_TICKER_STREAM_TOPIC);

        stockTickerTable.toStream().print(Printed.<String, StockTickerData>toSysOut().withLabel("Stocks-KTable"));
        stockTickerStream.print(Printed.<String, StockTickerData>toSysOut().withLabel( "Stocks-KStream"));

        int numberCompanies = 3;
        int iterations = 3;

        MockDataProducer.produceStockTickerData(numberCompanies, iterations);

        KafkaStreams kafkaStreams = new KafkaStreams(builder.build(), streamsConfig);
        LOG.info("KTable vs KStream output started");
        kafkaStreams.cleanUp();
        kafkaStreams.start();
        Thread.sleep(15000);
        LOG.info("Shutting down KTable vs KStream Application now");
        kafkaStreams.close();
        MockDataProducer.shutdown();

    }