Java Code Examples for org.apache.kafka.streams.TopologyTestDriver#readOutput()

private List<ActingEvent> readOutputTopic(TopologyTestDriver testDriver,
                                          String topic,
                                          Deserializer<String> keyDeserializer,
                                          SpecificAvroDeserializer<ActingEvent> valueDeserializer) {
    List<ActingEvent> results = new ArrayList<>();

    while (true) {
        ProducerRecord<String, ActingEvent> record = testDriver.readOutput(topic, keyDeserializer, valueDeserializer);

        if (record != null) {
            results.add(record.value());
        } else {
            break;
        }
    }

    return results;
}
 

private List<RatedMovie> readOutputTopic(TopologyTestDriver testDriver,
                                         String topic,
                                         Deserializer<String> keyDeserializer,
                                         SpecificAvroDeserializer<RatedMovie> makeRatedMovieDeserializer) {
    List<RatedMovie> results = new ArrayList<>();

    while (true) {
        ProducerRecord<String, RatedMovie> record = testDriver.readOutput(topic, keyDeserializer, makeRatedMovieDeserializer);

        if (record != null) {
            results.add(record.value());
        } else {
            break;
        }
    }

    return results;
}
 

private List<RatingCount> readOutputTopic(TopologyTestDriver testDriver,
                                          String outputTopic,
                                          Deserializer<String> keyDeserializer,
                                          Deserializer<String> valueDeserializer) {
    List<RatingCount> results = new ArrayList<>();

    while(true) {
        ProducerRecord<String, String> record = testDriver.readOutput(outputTopic, keyDeserializer, valueDeserializer);

        if (record != null) {
            results.add(new RatingCount(record.key().toString(), record.value()));
        } else {
            break;
        }
    }

    return results;
}
 

private List<Movie> readOutputTopic(TopologyTestDriver testDriver,
                                          String topic,
                                          Deserializer<String> keyDeserializer,
                                          SpecificAvroDeserializer<Movie> valueDeserializer) {
    List<Movie> results = new ArrayList<>();

    while (true) {
        ProducerRecord<String, Movie> record = testDriver.readOutput(topic, keyDeserializer, valueDeserializer);

        if (record != null) {
            results.add(record.value());
        } else {
            break;
        }
    }

    return results;
}
 

TestTopologyReceiver(BiConsumer<K, V> updateTarget, TopologyTestDriver driver, ReceiverSpec<K, V> spec) {
    getDriverOutput = () -> {
        int count = 0;
        while (true) {
            ProducerRecord<String, V> record = driver.readOutput(spec.topicName,
                    Serdes.String().deserializer(),
                    spec.valueSerde.deserializer());
            if (record == null)
                break;
            count++;
            K key = spec.keyConverter.apply(record.key());
            updateTarget.accept(key, record.value());
        }
        return count;
    };
}
 

@Test void should_aggregateSpans_and_mapDependencies() {
  // Given: configuration
  Duration traceTimeout = Duration.ofSeconds(1);
  SpansSerde spansSerde = new SpansSerde();
  DependencyLinkSerde dependencyLinkSerde = new DependencyLinkSerde();
  // When: topology built
  Topology topology = new SpanAggregationTopology(
      spansTopic,
      traceTopic,
      dependencyTopic,
      traceTimeout,
      true).get();
  TopologyDescription description = topology.describe();
  // Then: single threaded topology
  assertThat(description.subtopologies()).hasSize(1);
  // Given: test driver
  TopologyTestDriver testDriver = new TopologyTestDriver(topology, props);
  // When: two related spans coming on the same Session window
  ConsumerRecordFactory<String, List<Span>> factory =
      new ConsumerRecordFactory<>(spansTopic, new StringSerializer(), spansSerde.serializer());
  Span a = Span.newBuilder().traceId("a").id("a").name("op_a").kind(Span.Kind.CLIENT)
      .localEndpoint(Endpoint.newBuilder().serviceName("svc_a").build())
      .build();
  Span b = Span.newBuilder().traceId("a").id("b").name("op_b").kind(Span.Kind.SERVER)
      .localEndpoint(Endpoint.newBuilder().serviceName("svc_b").build())
      .build();
  testDriver.pipeInput(
      factory.create(spansTopic, a.traceId(), Collections.singletonList(a), 0L));
  testDriver.pipeInput(
      factory.create(spansTopic, b.traceId(), Collections.singletonList(b), 0L));
  // When: and new record arrive, moving the event clock further than inactivity gap
  Span c = Span.newBuilder().traceId("c").id("c").build();
  testDriver.pipeInput(factory.create(spansTopic, c.traceId(), Collections.singletonList(c),
      traceTimeout.toMillis() + 1));
  // Then: a trace is aggregated.1
  ProducerRecord<String, List<Span>> trace =
      testDriver.readOutput(traceTopic, new StringDeserializer(), spansSerde.deserializer());
  assertThat(trace).isNotNull();
  OutputVerifier.compareKeyValue(trace, a.traceId(), Arrays.asList(a, b));
  // Then: a dependency link is created
  ProducerRecord<String, DependencyLink> linkRecord =
      testDriver.readOutput(dependencyTopic, new StringDeserializer(),
          dependencyLinkSerde.deserializer());
  assertThat(linkRecord).isNotNull();
  DependencyLink link = DependencyLink.newBuilder()
      .parent("svc_a").child("svc_b").callCount(1).errorCount(0)
      .build();
  OutputVerifier.compareKeyValue(linkRecord, "svc_a:svc_b", link);
  //Finally close resources
  testDriver.close();
  spansSerde.close();
  dependencyLinkSerde.close();
}
 

@Test
public void shouldFilterDistinctEvents() throws IOException, RestClientException {

  final FindDistinctEvents distinctifier  = new FindDistinctEvents();

  String inputTopic = envProps.getProperty("input.topic.name");
  String outputTopic = envProps.getProperty("output.topic.name");

  final SpecificAvroSerde<Click> clickSerde = makeSerializer(envProps);

  Topology topology = distinctifier.buildTopology(envProps, clickSerde);
  TopologyTestDriver testDriver = new TopologyTestDriver(topology, streamProps);

  Serializer<String> keySerializer = Serdes.String().serializer();

  ConsumerRecordFactory<String, Click> inputFactory = new ConsumerRecordFactory<>(
          keySerializer, clickSerde.serializer());

  final List<Click> clicks = asList(
          new Click("10.0.0.1",
                  "https://docs.confluent.io/current/tutorials/examples/kubernetes/gke-base/docs/index.html",
          "2019-09-16T14:53:43+00:00"),
          new Click("10.0.0.2",
                  "https://www.confluent.io/hub/confluentinc/kafka-connect-datagen",
          "2019-09-16T14:53:43+00:01"),
          new Click("10.0.0.3",
          "https://www.confluent.io/hub/confluentinc/kafka-connect-datagen",
          "2019-09-16T14:53:43+00:03"),
          new Click("10.0.0.1",
          "https://docs.confluent.io/current/tutorials/examples/kubernetes/gke-base/docs/index.html",
          "2019-09-16T14:53:43+00:00"),
          new Click("10.0.0.2",
          "https://www.confluent.io/hub/confluentinc/kafka-connect-datagen",
          "2019-09-16T14:53:43+00:01"),
          new Click("10.0.0.3",
          "https://www.confluent.io/hub/confluentinc/kafka-connect-datagen",
          "2019-09-16T14:53:43+00:03"));

  final List<Click> expectedOutput = asList(clicks.get(0),clicks.get(1),clicks.get(2));

  for (Click clk : clicks) {
    testDriver.pipeInput(inputFactory.create(inputTopic, clk.getIp(), clk));
  }

  Deserializer<String> keyDeserializer = Serdes.String().deserializer();
  List<Click> actualOutput = new ArrayList<>();
  while (true) {
    ProducerRecord<String, Click>
        record =
        testDriver.readOutput(outputTopic, keyDeserializer, clickSerde.deserializer());

    if (record != null) {
      actualOutput.add(record.value());
    } else {
      break;
    }
  }

  Assert.assertEquals(expectedOutput, actualOutput);
}
 

@Test
public void shouldSumTicketSales() throws IOException, RestClientException {
  AggregatingSum aggSum = new AggregatingSum();
  Properties envProps = aggSum.loadEnvProperties(TEST_CONFIG_FILE);
  Properties streamProps = aggSum.buildStreamsProperties(envProps);

  String inputTopic = envProps.getProperty("input.topic.name");
  String outputTopic = envProps.getProperty("output.topic.name");

  final SpecificAvroSerde<TicketSale> ticketSaleSpecificAvroSerde = makeSerializer(envProps);

  Topology topology = aggSum.buildTopology(envProps, ticketSaleSpecificAvroSerde);
  TopologyTestDriver testDriver = new TopologyTestDriver(topology, streamProps);

  Serializer<String> keySerializer = Serdes.String().serializer();
  Deserializer<String> keyDeserializer = Serdes.String().deserializer();

  ConsumerRecordFactory<String, TicketSale>
      inputFactory =
      new ConsumerRecordFactory<>(keySerializer, ticketSaleSpecificAvroSerde.serializer());

  final List<TicketSale>
      input = asList(
                new TicketSale("Die Hard", "2019-07-18T10:00:00Z", 12),
                new TicketSale("Die Hard", "2019-07-18T10:01:00Z", 12),
                new TicketSale("The Godfather", "2019-07-18T10:01:31Z", 12),
                new TicketSale("Die Hard", "2019-07-18T10:01:36Z", 24),
                new TicketSale("The Godfather", "2019-07-18T10:02:00Z", 18),
                new TicketSale("The Big Lebowski", "2019-07-18T11:03:21Z", 12),
                new TicketSale("The Big Lebowski", "2019-07-18T11:03:50Z", 12),
                new TicketSale("The Godfather", "2019-07-18T11:40:00Z", 36),
                new TicketSale("The Godfather", "2019-07-18T11:40:09Z", 18)
              );

  List<Integer> expectedOutput = new ArrayList<Integer>(Arrays.asList(12, 24, 12, 48, 30, 12, 24, 66, 84));

  for (TicketSale ticketSale : input) {
    testDriver.pipeInput(inputFactory.create(inputTopic, "", ticketSale));
  }

  List<Integer> actualOutput = new ArrayList<>();
  while (true) {
    ProducerRecord<String, Integer>
        record =
        testDriver.readOutput(outputTopic, keyDeserializer, Serdes.Integer().deserializer());

    if (record != null) {
      actualOutput.add(record.value());
    } else {
      break;
    }
  }

  System.out.println(actualOutput);
  Assert.assertEquals(expectedOutput, actualOutput);

}
 

@Test
public void shouldCountTicketSales() throws IOException, RestClientException {
  AggregatingCount aggCount = new AggregatingCount();
  Properties envProps = aggCount.loadEnvProperties(TEST_CONFIG_FILE);
  Properties streamProps = aggCount.buildStreamsProperties(envProps);

  String inputTopic = envProps.getProperty("input.topic.name");
  String outputTopic = envProps.getProperty("output.topic.name");

  final SpecificAvroSerde<TicketSale> ticketSaleSpecificAvroSerde = makeSerializer(envProps);

  Topology topology = aggCount.buildTopology(envProps, ticketSaleSpecificAvroSerde);
  TopologyTestDriver testDriver = new TopologyTestDriver(topology, streamProps);

  Serializer<String> keySerializer = Serdes.String().serializer();
  Deserializer<String> keyDeserializer = Serdes.String().deserializer();

  ConsumerRecordFactory<String, TicketSale>
      inputFactory =
      new ConsumerRecordFactory<>(keySerializer, ticketSaleSpecificAvroSerde.serializer());

  final List<TicketSale>
      input = asList(
                new TicketSale("Die Hard", "2019-07-18T10:00:00Z", 12),
                new TicketSale("Die Hard", "2019-07-18T10:01:00Z", 12),
                new TicketSale("The Godfather", "2019-07-18T10:01:31Z", 12),
                new TicketSale("Die Hard", "2019-07-18T10:01:36Z", 24),
                new TicketSale("The Godfather", "2019-07-18T10:02:00Z", 18),
                new TicketSale("The Big Lebowski", "2019-07-18T11:03:21Z", 12),
                new TicketSale("The Big Lebowski", "2019-07-18T11:03:50Z", 12),
                new TicketSale("The Godfather", "2019-07-18T11:40:00Z", 36),
                new TicketSale("The Godfather", "2019-07-18T11:40:09Z", 18)
              );

  List<Long> expectedOutput = new ArrayList<Long>(Arrays.asList(1L, 2L, 1L, 3L, 2L, 1L, 2L, 3L, 4L));

  for (TicketSale ticketSale : input) {
    testDriver.pipeInput(inputFactory.create(inputTopic, "", ticketSale));
  }

  List<Long> actualOutput = new ArrayList<>();
  while (true) {
    ProducerRecord<String, Long>
        record =
        testDriver.readOutput(outputTopic, keyDeserializer, Serdes.Long().deserializer());

    if (record != null) {
      actualOutput.add(record.value());
    } else {
      break;
    }
  }

  System.out.println(actualOutput);
  Assert.assertEquals(expectedOutput, actualOutput);

}
 

@Test
public void shouldCreateKeyedStream() throws IOException, RestClientException {
  StreamsIngest si = new StreamsIngest();
  Properties envProps = si.loadEnvProperties(TEST_CONFIG_FILE);
  Properties streamProps = si.buildStreamsProperties(envProps);

  String inputTopic = envProps.getProperty("input.topic.name");
  String outputTopic = envProps.getProperty("output.topic.name");

  final SpecificAvroSerde<City> citySpecificAvroSerde = makeSerializer(envProps);

  Topology topology = si.buildTopology(envProps, citySpecificAvroSerde);
  TopologyTestDriver testDriver = new TopologyTestDriver(topology, streamProps);

  Serializer<String> keySerializer = Serdes.String().serializer();
  Deserializer<Long> keyDeserializer = Serdes.Long().deserializer();

  ConsumerRecordFactory<String, City>
      inputFactory =
      new ConsumerRecordFactory<>(keySerializer, citySpecificAvroSerde.serializer());

  // Fixture
  City c1 = new City(1L, "Raleigh", "NC");
  City c2 = new City(2L, "Mountain View", "CA");
  City c3 = new City(3L, "Knoxville", "TN");
  City c4 = new City(4L, "Houston", "TX");
  City c5 = new City(5L, "Olympia", "WA");
  City c6 = new City(6L, "Bismarck", "ND");
  // end Fixture

  final List<City>
      input = asList(c1, c2, c3, c4, c5, c6);

  final List<Long> expectedOutput = asList(1L, 2L, 3L, 4L, 5L, 6L);

  for (City city : input) {
    testDriver.pipeInput(inputFactory.create(inputTopic, null, city));
  }

  List<Long> actualOutput = new ArrayList<>();
  while (true) {
    ProducerRecord<Long, City>
        record =
        testDriver.readOutput(outputTopic, keyDeserializer, citySpecificAvroSerde.deserializer());

    if (record != null) {
      actualOutput.add(record.key());
    } else {
      break;
    }
  }

  Assert.assertEquals(expectedOutput, actualOutput);
}
 

@Test
public void shouldFilterGRRMartinsBooks() throws IOException, RestClientException {
  FilterEvents fe = new FilterEvents();
  Properties envProps = fe.loadEnvProperties(TEST_CONFIG_FILE);
  Properties streamProps = fe.buildStreamsProperties(envProps);

  String inputTopic = envProps.getProperty("input.topic.name");
  String outputTopic = envProps.getProperty("output.topic.name");

  final SpecificAvroSerde<Publication> publicationSpecificAvroSerde = makeSerializer(envProps);

  Topology topology = fe.buildTopology(envProps, publicationSpecificAvroSerde);
  TopologyTestDriver testDriver = new TopologyTestDriver(topology, streamProps);

  Serializer<String> keySerializer = Serdes.String().serializer();
  Deserializer<String> keyDeserializer = Serdes.String().deserializer();

  ConsumerRecordFactory<String, Publication>
      inputFactory =
      new ConsumerRecordFactory<>(keySerializer, publicationSpecificAvroSerde.serializer());

  // Fixture
  Publication iceAndFire = new Publication("George R. R. Martin", "A Song of Ice and Fire");
  Publication silverChair = new Publication("C.S. Lewis", "The Silver Chair");
  Publication perelandra = new Publication("C.S. Lewis", "Perelandra");
  Publication fireAndBlood = new Publication("George R. R. Martin", "Fire & Blood");
  Publication theHobbit = new Publication("J. R. R. Tolkien", "The Hobbit");
  Publication lotr = new Publication("J. R. R. Tolkien", "The Lord of the Rings");
  Publication dreamOfSpring = new Publication("George R. R. Martin", "A Dream of Spring");
  Publication fellowship = new Publication("J. R. R. Tolkien", "The Fellowship of the Ring");
  Publication iceDragon = new Publication("George R. R. Martin", "The Ice Dragon");
  // end Fixture

  final List<Publication>
      input = asList(iceAndFire, silverChair, perelandra, fireAndBlood, theHobbit, lotr, dreamOfSpring, fellowship,
                     iceDragon);

  final List<Publication> expectedOutput = asList(iceAndFire, fireAndBlood, dreamOfSpring, iceDragon);

  for (Publication publication : input) {
    testDriver.pipeInput(inputFactory.create(inputTopic, publication.getName(), publication));
  }

  List<Publication> actualOutput = new ArrayList<>();
  while (true) {
    ProducerRecord<String, Publication>
        record =
        testDriver.readOutput(outputTopic, keyDeserializer, publicationSpecificAvroSerde.deserializer());

    if (record != null) {
      actualOutput.add(record.value());
    } else {
      break;
    }
  }

  Assert.assertEquals(expectedOutput, actualOutput);
}
 

private void nullOnDeserException(TopologyTestDriver driver) {
    driver.pipeInput(stringFactory.create(INPUT_TOPIC, KAFKA_KEY, INVALID_INPUT_VALUE));
    val record = driver.readOutput(DEFAULT_OUTPUT_TOPIC, stringDeser, mmdDeser);
    Assert.assertNull(record);
}