Java Code Examples for org.apache.kafka.streams.kstream.KStream#join()

public Topology buildTopology(Properties envProps) {
    final StreamsBuilder builder = new StreamsBuilder();
    final String movieTopic = envProps.getProperty("movie.topic.name");
    final String rekeyedMovieTopic = envProps.getProperty("rekeyed.movie.topic.name");
    final String ratingTopic = envProps.getProperty("rating.topic.name");
    final String ratedMoviesTopic = envProps.getProperty("rated.movies.topic.name");
    final MovieRatingJoiner joiner = new MovieRatingJoiner();

    KStream<String, Movie> movieStream = builder.<String, Movie>stream(movieTopic)
            .map((key, movie) -> new KeyValue<>(movie.getId().toString(), movie));

    movieStream.to(rekeyedMovieTopic);

    KTable<String, Movie> movies = builder.table(rekeyedMovieTopic);

    KStream<String, Rating> ratings = builder.<String, Rating>stream(ratingTopic)
            .map((key, rating) -> new KeyValue<>(rating.getId().toString(), rating));

    KStream<String, RatedMovie> ratedMovie = ratings.join(movies, joiner);

    ratedMovie.to(ratedMoviesTopic, Produced.with(Serdes.String(), ratedMovieAvroSerde(envProps)));

    return builder.build();
}
 

@StreamListener
@SendTo("output")
public KStream<Long, EnrichedOrder> process(
		@Input("input") KStream<Long, Order> ordersStream,
		@Input("input-x") GlobalKTable<Long, Customer> customers,
		@Input("input-y") GlobalKTable<Long, Product> products) {

	KStream<Long, CustomerOrder> customerOrdersStream = ordersStream.join(
			customers, (orderId, order) -> order.getCustomerId(),
			(order, customer) -> new CustomerOrder(customer, order));

	return customerOrdersStream.join(products,
			(orderId, customerOrder) -> customerOrder.productId(),
			(customerOrder, product) -> {
				EnrichedOrder enrichedOrder = new EnrichedOrder();
				enrichedOrder.setProduct(product);
				enrichedOrder.setCustomer(customerOrder.customer);
				enrichedOrder.setOrder(customerOrder.order);
				return enrichedOrder;
			});
}
 

@StreamListener
public void testProcessor(
		@Input(BindingsForTwoKStreamJoinTest.INPUT_1) KStream<String, String> input1Stream,
		@Input(BindingsForTwoKStreamJoinTest.INPUT_2) KStream<String, String> input2Stream) {
	input1Stream
			.join(input2Stream,
					(event1, event2) -> null,
					JoinWindows.of(TimeUnit.MINUTES.toMillis(5)),
					Joined.with(
							Serdes.String(),
							Serdes.String(),
							Serdes.String()
					)
			);
}
 

public Topology buildTopology(Properties envProps) {
  final StreamsBuilder builder = new StreamsBuilder();
  final String inputTopic = envProps.getProperty("input.topic.name");
  final String outputTopic = envProps.getProperty("output.topic.name");
  final String joinTopic = envProps.getProperty("join.topic.name");

  final Serde<Long> longSerde = Serdes.Long();
  final Serde<String> stringSerde = Serdes.String();

  final boolean addFilter = Boolean.parseBoolean(envProps.getProperty("add.filter"));
  final boolean addNames = Boolean.parseBoolean(envProps.getProperty("add.names"));

  KStream<Long, String> inputStream = builder.stream(inputTopic, Consumed.with(longSerde, stringSerde))
                                                .selectKey((k, v) -> Long.parseLong(v.substring(0, 1)));
  if (addFilter) {
    inputStream = inputStream.filter((k, v) -> k != 100L);
  }

  final KStream<Long, String> joinedStream;
  final KStream<Long, Long> countStream;

  if (!addNames) {
       countStream = inputStream.groupByKey(Grouped.with(longSerde, stringSerde))
                                  .count()
                                  .toStream();

      joinedStream = inputStream.join(countStream, (v1, v2) -> v1 + v2.toString(),
                                                            JoinWindows.of(Duration.ofMillis(100)),
                                                            StreamJoined.with(longSerde, stringSerde, longSerde));
  } else {
      countStream = inputStream.groupByKey(Grouped.with("count", longSerde, stringSerde))
                                 .count(Materialized.as("the-counting-store"))
                                 .toStream();

      joinedStream = inputStream.join(countStream, (v1, v2) -> v1 + v2.toString(),
                                                            JoinWindows.of(Duration.ofMillis(100)),
                                                            StreamJoined.with(longSerde, stringSerde, longSerde)
                                                                        .withName("join").withStoreName("the-join-store"));
  }

  joinedStream.to(joinTopic, Produced.with(longSerde, stringSerde));
  countStream.map((k,v) -> KeyValue.pair(k.toString(), v.toString())).to(outputTopic, Produced.with(stringSerde, stringSerde));


  return builder.build();
}
 

public static void main(String[] args) throws Exception {

        StreamsConfig streamsConfig = new StreamsConfig(getProperties());
        StreamsBuilder builder = new StreamsBuilder();


        Serde<Purchase> purchaseSerde = StreamsSerdes.PurchaseSerde();
        Serde<String> stringSerde = Serdes.String();

        KeyValueMapper<String, Purchase, KeyValue<String,Purchase>> custIdCCMasking = (k, v) -> {
            Purchase masked = Purchase.builder(v).maskCreditCard().build();
            return new KeyValue<>(masked.getCustomerId(), masked);
        };


        Predicate<String, Purchase> coffeePurchase = (key, purchase) -> purchase.getDepartment().equalsIgnoreCase("coffee");
        Predicate<String, Purchase> electronicPurchase = (key, purchase) -> purchase.getDepartment().equalsIgnoreCase("electronics");

        int COFFEE_PURCHASE = 0;
        int ELECTRONICS_PURCHASE = 1;

        KStream<String, Purchase> transactionStream = builder.stream( "transactions", Consumed.with(Serdes.String(), purchaseSerde)).map(custIdCCMasking);

        KStream<String, Purchase>[] branchesStream = transactionStream.selectKey((k,v)-> v.getCustomerId()).branch(coffeePurchase, electronicPurchase);

        KStream<String, Purchase> coffeeStream = branchesStream[COFFEE_PURCHASE];
        KStream<String, Purchase> electronicsStream = branchesStream[ELECTRONICS_PURCHASE];

        ValueJoiner<Purchase, Purchase, CorrelatedPurchase> purchaseJoiner = new PurchaseJoiner();
        JoinWindows twentyMinuteWindow =  JoinWindows.of(60 * 1000 * 20);

        KStream<String, CorrelatedPurchase> joinedKStream = coffeeStream.join(electronicsStream,
                                                                              purchaseJoiner,
                                                                              twentyMinuteWindow,
                                                                              Joined.with(stringSerde,
                                                                                          purchaseSerde,
                                                                                          purchaseSerde));

        joinedKStream.print(Printed.<String, CorrelatedPurchase>toSysOut().withLabel("joined KStream"));

        // used only to produce data for this application, not typical usage
        MockDataProducer.producePurchaseData();
        
        LOG.info("Starting Join Examples");
        KafkaStreams kafkaStreams = new KafkaStreams(builder.build(), streamsConfig);
        kafkaStreams.start();
        Thread.sleep(65000);
        LOG.info("Shutting down the Join Examples now");
        kafkaStreams.close();
        MockDataProducer.shutdown();


    }