org.apache.kafka.streams.kstream.JoinWindows Java Examples

@Test
public void testKsDsl1() {
  final StreamsBuilder builder = new StreamsBuilder();
  final KStream<String, String> streamOne = builder.stream("input-topic-one");
  final KStream<String, String> streamTwo = builder.stream("input-topic-two");
  final KStream<String, String> streamOneNewKey = streamOne.selectKey((k, v) -> v.substring(0, 5));
  final KStream<String, String> streamTwoNewKey = streamTwo.selectKey((k, v) -> v.substring(4, 9));
  streamOneNewKey.join(streamTwoNewKey, (v1, v2) -> v1 + ":" + v2, JoinWindows.of(ofMinutes(5).toMillis())).to("joined-output");

  final Topology topology = builder.build();
  final String text = topology.describe().toString();
  System.out.println(text);

  final KStreamsTopologyDescriptionParser parsed = new KStreamsTopologyDescriptionParser(text);
  assertEquals(16, parsed.size());
  System.out.println(parsed.toString());
}
 

private void buildMatchStatistics(KStream<String, GoalScored> goalStream) {
    KStream<String, MatchStarted> matchStartedStream = builder
            .stream(MATCH_STARTED_TOPIC, with(String(), matchStartedSerde));

    KStream<String, MatchFinished> matchFinishedStream = builder
            .stream(MATCH_FINISHED_TOPIC, with(String(), matchFinishedSerde));

    KStream<String, MatchScore> scoreStream = matchStartedStream
            .leftJoin(goalStream, (match, goal) -> new MatchScore(match).goal(goal),
                JoinWindows.of(maxMatchDuration), with(String(), matchStartedSerde, goalScoredSerde)
    );

    KTable<String, MatchScore> scoreTable = scoreStream
            .groupByKey()
            .reduce(MatchScore::aggregate, materialized(MATCH_SCORES_STORE, matchScoreSerde));
    scoreTable.toStream().to(MATCH_SCORES_TOPIC, Produced.with(String(), matchScoreSerde));

    KStream<String, MatchScore> finalScoreStream = matchFinishedStream
            .leftJoin(scoreTable, (matchFinished, matchScore) -> matchScore,
                with(String(), matchFinishedSerde, matchScoreSerde)
    );

    // new key: clubId
    KStream<String, TeamRanking> rankingStream = finalScoreStream
            .flatMap((clubId, matchScore) -> {
                Collection<KeyValue<String, TeamRanking>> result = new ArrayList<>(2);
                result.add(pair(matchScore.getHomeClubId(), matchScore.homeRanking()));
                result.add(pair(matchScore.getAwayClubId(), matchScore.awayRanking()));
                return result;
            });

    KTable<String, TeamRanking> rankingTable = rankingStream
            .groupByKey(Serialized.with(String(), rankingSerde))
            .reduce(TeamRanking::aggregate, materialized(TEAM_RANKING_STORE, rankingSerde));

    // publish changes to a view topic
    rankingTable.toStream().to(TEAM_RANKING_TOPIC, Produced.with(String(), rankingSerde));
}
 

@StreamListener
public void testProcessor(
		@Input(BindingsForTwoKStreamJoinTest.INPUT_1) KStream<String, String> input1Stream,
		@Input(BindingsForTwoKStreamJoinTest.INPUT_2) KStream<String, String> input2Stream) {
	input1Stream
			.join(input2Stream,
					(event1, event2) -> null,
					JoinWindows.of(TimeUnit.MINUTES.toMillis(5)),
					Joined.with(
							Serdes.String(),
							Serdes.String(),
							Serdes.String()
					)
			);
}
 

public Topology buildTopology(Properties envProps) {
  final StreamsBuilder builder = new StreamsBuilder();
  final String inputTopic = envProps.getProperty("input.topic.name");
  final String outputTopic = envProps.getProperty("output.topic.name");
  final String joinTopic = envProps.getProperty("join.topic.name");

  final Serde<Long> longSerde = Serdes.Long();
  final Serde<String> stringSerde = Serdes.String();

  final boolean addFilter = Boolean.parseBoolean(envProps.getProperty("add.filter"));
  final boolean addNames = Boolean.parseBoolean(envProps.getProperty("add.names"));

  KStream<Long, String> inputStream = builder.stream(inputTopic, Consumed.with(longSerde, stringSerde))
                                                .selectKey((k, v) -> Long.parseLong(v.substring(0, 1)));
  if (addFilter) {
    inputStream = inputStream.filter((k, v) -> k != 100L);
  }

  final KStream<Long, String> joinedStream;
  final KStream<Long, Long> countStream;

  if (!addNames) {
       countStream = inputStream.groupByKey(Grouped.with(longSerde, stringSerde))
                                  .count()
                                  .toStream();

      joinedStream = inputStream.join(countStream, (v1, v2) -> v1 + v2.toString(),
                                                            JoinWindows.of(Duration.ofMillis(100)),
                                                            StreamJoined.with(longSerde, stringSerde, longSerde));
  } else {
      countStream = inputStream.groupByKey(Grouped.with("count", longSerde, stringSerde))
                                 .count(Materialized.as("the-counting-store"))
                                 .toStream();

      joinedStream = inputStream.join(countStream, (v1, v2) -> v1 + v2.toString(),
                                                            JoinWindows.of(Duration.ofMillis(100)),
                                                            StreamJoined.with(longSerde, stringSerde, longSerde)
                                                                        .withName("join").withStoreName("the-join-store"));
  }

  joinedStream.to(joinTopic, Produced.with(longSerde, stringSerde));
  countStream.map((k,v) -> KeyValue.pair(k.toString(), v.toString())).to(outputTopic, Produced.with(stringSerde, stringSerde));


  return builder.build();
}
 

public static void main(String[] args) throws Exception {

        StreamsConfig streamsConfig = new StreamsConfig(getProperties());
        StreamsBuilder builder = new StreamsBuilder();


        Serde<Purchase> purchaseSerde = StreamsSerdes.PurchaseSerde();
        Serde<String> stringSerde = Serdes.String();

        KeyValueMapper<String, Purchase, KeyValue<String,Purchase>> custIdCCMasking = (k, v) -> {
            Purchase masked = Purchase.builder(v).maskCreditCard().build();
            return new KeyValue<>(masked.getCustomerId(), masked);
        };


        Predicate<String, Purchase> coffeePurchase = (key, purchase) -> purchase.getDepartment().equalsIgnoreCase("coffee");
        Predicate<String, Purchase> electronicPurchase = (key, purchase) -> purchase.getDepartment().equalsIgnoreCase("electronics");

        int COFFEE_PURCHASE = 0;
        int ELECTRONICS_PURCHASE = 1;

        KStream<String, Purchase> transactionStream = builder.stream( "transactions", Consumed.with(Serdes.String(), purchaseSerde)).map(custIdCCMasking);

        KStream<String, Purchase>[] branchesStream = transactionStream.selectKey((k,v)-> v.getCustomerId()).branch(coffeePurchase, electronicPurchase);

        KStream<String, Purchase> coffeeStream = branchesStream[COFFEE_PURCHASE];
        KStream<String, Purchase> electronicsStream = branchesStream[ELECTRONICS_PURCHASE];

        ValueJoiner<Purchase, Purchase, CorrelatedPurchase> purchaseJoiner = new PurchaseJoiner();
        JoinWindows twentyMinuteWindow =  JoinWindows.of(60 * 1000 * 20);

        KStream<String, CorrelatedPurchase> joinedKStream = coffeeStream.join(electronicsStream,
                                                                              purchaseJoiner,
                                                                              twentyMinuteWindow,
                                                                              Joined.with(stringSerde,
                                                                                          purchaseSerde,
                                                                                          purchaseSerde));

        joinedKStream.print(Printed.<String, CorrelatedPurchase>toSysOut().withLabel("joined KStream"));

        // used only to produce data for this application, not typical usage
        MockDataProducer.producePurchaseData();
        
        LOG.info("Starting Join Examples");
        KafkaStreams kafkaStreams = new KafkaStreams(builder.build(), streamsConfig);
        kafkaStreams.start();
        Thread.sleep(65000);
        LOG.info("Shutting down the Join Examples now");
        kafkaStreams.close();
        MockDataProducer.shutdown();


    }