Java Code Examples for org.apache.kafka.streams.TopologyTestDriver#pipeInput()

/**
 * <p>Constructor for the test input topic.</p>
 *
 * @param testDriver Kafka's {@link TopologyTestDriver} used in this test.
 * @param topic Name of input topic.
 * @param keySerde Serde for key type in topic.
 * @param valueSerde Serde for value type in topic.
 */
protected TestInput(final TopologyTestDriver testDriver, final String topic, final Serde<K> keySerde,
        final Serde<V> valueSerde) {
    this.testDriver = testDriver;
    this.topic = topic;
    this.keySerde = keySerde;
    this.valueSerde = valueSerde;

    this.consumerFactory = new ConsumerRecordFactory<>(topic,
            keySerde == null ? new UnspecifiedSerializer<K>() : keySerde.serializer(),
            valueSerde == null ? new UnspecifiedSerializer<V>() : valueSerde.serializer()) {
        @Override
        public ConsumerRecord<byte[], byte[]> create(final String topicName, final K key, final V value,
                final Headers headers, final long timestampMs) {
            final ConsumerRecord<byte[], byte[]> record = super.create(topicName, key, value, headers, timestampMs);
            testDriver.pipeInput(record);
            return record;
        }
    };
}
 

@Test
public void testTransformStream() throws IOException {
    TransformStream ts = new TransformStream();
    Properties envProps = ts.loadEnvProperties(TEST_CONFIG_FILE);
    Properties streamProps = ts.buildStreamsProperties(envProps);

    String inputTopic = envProps.getProperty("input.topic.name");
    String outputTopic = envProps.getProperty("output.topic.name");

    Topology topology = ts.buildTopology(envProps);
    TopologyTestDriver testDriver = new TopologyTestDriver(topology, streamProps);

    Serializer<String> keySerializer = Serdes.String().serializer();
    SpecificAvroSerializer<RawMovie> valueSerializer = makeSerializer(envProps);

    Deserializer<String> keyDeserializer = Serdes.String().deserializer();
    SpecificAvroDeserializer<Movie> valueDeserializer = makeDeserializer(envProps);

    ConsumerRecordFactory<String, RawMovie> inputFactory = new ConsumerRecordFactory<>(keySerializer, valueSerializer);

    List<RawMovie> input = new ArrayList<>();
    input.add(RawMovie.newBuilder().setId(294).setTitle("Die Hard::1988").setGenre("action").build());
    input.add(RawMovie.newBuilder().setId(354).setTitle("Tree of Life::2011").setGenre("drama").build());
    input.add(RawMovie.newBuilder().setId(782).setTitle("A Walk in the Clouds::1995").setGenre("romance").build());
    input.add(RawMovie.newBuilder().setId(128).setTitle("The Big Lebowski::1998").setGenre("comedy").build());

    List<Movie> expectedOutput = new ArrayList<>();
    expectedOutput.add(Movie.newBuilder().setTitle("Die Hard").setId(294).setReleaseYear(1988).setGenre("action").build());
    expectedOutput.add(Movie.newBuilder().setTitle("Tree of Life").setId(354).setReleaseYear(2011).setGenre("drama").build());
    expectedOutput.add(Movie.newBuilder().setTitle("A Walk in the Clouds").setId(782).setReleaseYear(1995).setGenre("romance").build());
    expectedOutput.add(Movie.newBuilder().setTitle("The Big Lebowski").setId(128).setReleaseYear(1998).setGenre("comedy").build());

    for (RawMovie rawMovie : input) {
        testDriver.pipeInput(inputFactory.create(inputTopic, rawMovie.getTitle(), rawMovie));
    }

    List<Movie> actualOutput = readOutputTopic(testDriver, outputTopic, keyDeserializer, valueDeserializer);

    assertEquals(expectedOutput, actualOutput);
}
 

@Test void should_persistSpans_and_onlyQueryTraces_whenEnabled() {
  // Given: configs
  Duration traceTtl = Duration.ofMillis(5);
  Duration traceTtlCheckInterval = Duration.ofMinutes(1);
  List<String> autocompleteKeys = Collections.singletonList("environment");
  SpansSerde spansSerde = new SpansSerde();
  // When: topology provided
  Topology topology = new TraceStorageTopology(
      spansTopic,
      autocompleteKeys,
      traceTtl,
      traceTtlCheckInterval,
      0,
      true,
      false).get();
  TopologyDescription description = topology.describe();
  // Then: 1 thread prepared
  assertThat(description.subtopologies()).hasSize(1);
  // Given: streams config
  TopologyTestDriver testDriver = new TopologyTestDriver(topology, props);
  // When: a trace is passed
  ConsumerRecordFactory<String, List<Span>> factory =
      new ConsumerRecordFactory<>(spansTopic, new StringSerializer(), spansSerde.serializer());
  Span a = Span.newBuilder().traceId("a").id("a").name("op_a").kind(Span.Kind.CLIENT)
      .localEndpoint(Endpoint.newBuilder().serviceName("svc_a").build())
      .timestamp(10000L).duration(11L)
      .putTag("environment", "dev")
      .build();
  Span b = Span.newBuilder().traceId("a").id("b").name("op_b").kind(Span.Kind.SERVER)
      .localEndpoint(Endpoint.newBuilder().serviceName("svc_b").build())
      .timestamp(10000L).duration(10L)
      .build();
  Span c = Span.newBuilder().traceId("c").id("c").name("op_a").kind(Span.Kind.CLIENT)
      .localEndpoint(Endpoint.newBuilder().serviceName("svc_a").build())
      .timestamp(10000L).duration(11L)
      .putTag("environment", "dev")
      .build();
  List<Span> spans = Arrays.asList(a, b, c);
  testDriver.pipeInput(factory.create(spansTopic, a.traceId(), spans, 10L));
  // Then: trace stores are filled
  KeyValueStore<String, List<Span>> traces = testDriver.getKeyValueStore(TRACES_STORE_NAME);
  assertThat(traces.get(a.traceId())).containsExactlyElementsOf(spans);
  KeyValueStore<Long, Set<String>> spanIdsByTs =
      testDriver.getKeyValueStore(SPAN_IDS_BY_TS_STORE_NAME);
  KeyValueIterator<Long, Set<String>> ids = spanIdsByTs.all();
  assertThat(ids).hasNext();
  assertThat(ids.next().value).containsExactly(a.traceId());
  // Then: service name stores are filled
  KeyValueStore<String, String> serviceNames =
      testDriver.getKeyValueStore(SERVICE_NAMES_STORE_NAME);
  assertThat(serviceNames).isNull();
  KeyValueStore<String, Set<String>> spanNames =
      testDriver.getKeyValueStore(SPAN_NAMES_STORE_NAME);
  assertThat(spanNames).isNull();
  KeyValueStore<String, Set<String>> autocompleteTags =
      testDriver.getKeyValueStore(AUTOCOMPLETE_TAGS_STORE_NAME);
  assertThat(autocompleteTags).isNull();
  // Finally close resources
  testDriver.close();
  spansSerde.close();
}
 

@Test void should_persistSpans_and_searchQueryTraces_whenAllEnabled() {
  // Given: configs
  Duration traceTtl = Duration.ofMillis(5);
  Duration traceTtlCheckInterval = Duration.ofMinutes(1);
  List<String> autocompleteKeys = Collections.singletonList("environment");
  SpansSerde spansSerde = new SpansSerde();
  // When: topology provided
  Topology topology = new TraceStorageTopology(
      spansTopic,
      autocompleteKeys,
      traceTtl,
      traceTtlCheckInterval,
      0,
      true,
      true).get();
  TopologyDescription description = topology.describe();
  // Then: 1 thread prepared
  assertThat(description.subtopologies()).hasSize(1);
  // Given: streams config
  TopologyTestDriver testDriver = new TopologyTestDriver(topology, props);
  // When: a trace is passed
  ConsumerRecordFactory<String, List<Span>> factory =
      new ConsumerRecordFactory<>(spansTopic, new StringSerializer(), spansSerde.serializer());
  Span a = Span.newBuilder().traceId("a").id("a").name("op_a").kind(Span.Kind.CLIENT)
      .localEndpoint(Endpoint.newBuilder().serviceName("svc_a").build())
      .timestamp(10000L).duration(11L)
      .putTag("environment", "dev")
      .build();
  Span b = Span.newBuilder().traceId("a").id("b").name("op_b").kind(Span.Kind.SERVER)
      .localEndpoint(Endpoint.newBuilder().serviceName("svc_b").build())
      .timestamp(10000L).duration(10L)
      .build();
  Span c = Span.newBuilder().traceId("c").id("c").name("op_a").kind(Span.Kind.CLIENT)
      .localEndpoint(Endpoint.newBuilder().serviceName("svc_a").build())
      .timestamp(10000L).duration(11L)
      .putTag("environment", "dev")
      .build();
  List<Span> spans = Arrays.asList(a, b, c);
  testDriver.pipeInput(factory.create(spansTopic, a.traceId(), spans, 10L));
  // Then: trace stores are filled
  KeyValueStore<String, List<Span>> traces = testDriver.getKeyValueStore(TRACES_STORE_NAME);
  assertThat(traces.get(a.traceId())).containsExactlyElementsOf(spans);
  KeyValueStore<Long, Set<String>> spanIdsByTs =
      testDriver.getKeyValueStore(SPAN_IDS_BY_TS_STORE_NAME);
  KeyValueIterator<Long, Set<String>> ids = spanIdsByTs.all();
  assertThat(ids).hasNext();
  assertThat(ids.next().value).containsExactly(a.traceId());
  // Then: service name stores are filled
  KeyValueStore<String, String> serviceNames =
      testDriver.getKeyValueStore(SERVICE_NAMES_STORE_NAME);
  List<String> serviceNameList = new ArrayList<>();
  serviceNames.all().forEachRemaining(serviceName -> serviceNameList.add(serviceName.value));
  assertThat(serviceNameList).hasSize(2);
  assertThat(serviceNames.get("svc_a")).isEqualTo("svc_a");
  assertThat(serviceNames.get("svc_b")).isEqualTo("svc_b");
  KeyValueStore<String, Set<String>> spanNames =
      testDriver.getKeyValueStore(SPAN_NAMES_STORE_NAME);
  assertThat(spanNames.get("svc_a")).containsExactly("op_a");
  assertThat(spanNames.get("svc_b")).containsExactly("op_b");
  KeyValueStore<String, Set<String>> autocompleteTags =
      testDriver.getKeyValueStore(AUTOCOMPLETE_TAGS_STORE_NAME);
  assertThat(autocompleteTags.get("environment")).containsExactly("dev");
  // When: clock moves forward
  Span d = Span.newBuilder()
      .traceId("d")
      .id("d")
      .timestamp(
          MILLISECONDS.toMicros(traceTtlCheckInterval.toMillis()) + MILLISECONDS.toMicros(20))
      .build();
  testDriver.pipeInput(
      factory.create(spansTopic, d.traceId(), Collections.singletonList(d),
          traceTtlCheckInterval.plusMillis(1).toMillis()));
  // Then: Traces store is empty
  assertThat(traces.get(a.traceId())).isNull();
  // Finally close resources
  testDriver.close();
  spansSerde.close();
}
 

@Test void should_aggregateSpans_and_mapDependencies() {
  // Given: configuration
  Duration traceTimeout = Duration.ofSeconds(1);
  SpansSerde spansSerde = new SpansSerde();
  DependencyLinkSerde dependencyLinkSerde = new DependencyLinkSerde();
  // When: topology built
  Topology topology = new SpanAggregationTopology(
      spansTopic,
      traceTopic,
      dependencyTopic,
      traceTimeout,
      true).get();
  TopologyDescription description = topology.describe();
  // Then: single threaded topology
  assertThat(description.subtopologies()).hasSize(1);
  // Given: test driver
  TopologyTestDriver testDriver = new TopologyTestDriver(topology, props);
  // When: two related spans coming on the same Session window
  ConsumerRecordFactory<String, List<Span>> factory =
      new ConsumerRecordFactory<>(spansTopic, new StringSerializer(), spansSerde.serializer());
  Span a = Span.newBuilder().traceId("a").id("a").name("op_a").kind(Span.Kind.CLIENT)
      .localEndpoint(Endpoint.newBuilder().serviceName("svc_a").build())
      .build();
  Span b = Span.newBuilder().traceId("a").id("b").name("op_b").kind(Span.Kind.SERVER)
      .localEndpoint(Endpoint.newBuilder().serviceName("svc_b").build())
      .build();
  testDriver.pipeInput(
      factory.create(spansTopic, a.traceId(), Collections.singletonList(a), 0L));
  testDriver.pipeInput(
      factory.create(spansTopic, b.traceId(), Collections.singletonList(b), 0L));
  // When: and new record arrive, moving the event clock further than inactivity gap
  Span c = Span.newBuilder().traceId("c").id("c").build();
  testDriver.pipeInput(factory.create(spansTopic, c.traceId(), Collections.singletonList(c),
      traceTimeout.toMillis() + 1));
  // Then: a trace is aggregated.1
  ProducerRecord<String, List<Span>> trace =
      testDriver.readOutput(traceTopic, new StringDeserializer(), spansSerde.deserializer());
  assertThat(trace).isNotNull();
  OutputVerifier.compareKeyValue(trace, a.traceId(), Arrays.asList(a, b));
  // Then: a dependency link is created
  ProducerRecord<String, DependencyLink> linkRecord =
      testDriver.readOutput(dependencyTopic, new StringDeserializer(),
          dependencyLinkSerde.deserializer());
  assertThat(linkRecord).isNotNull();
  DependencyLink link = DependencyLink.newBuilder()
      .parent("svc_a").child("svc_b").callCount(1).errorCount(0)
      .build();
  OutputVerifier.compareKeyValue(linkRecord, "svc_a:svc_b", link);
  //Finally close resources
  testDriver.close();
  spansSerde.close();
  dependencyLinkSerde.close();
}
 

@Test void should_storeDependencies() {
  // Given: configs
  Duration dependenciesRetentionPeriod = Duration.ofMinutes(1);
  Duration dependenciesWindowSize = Duration.ofMillis(100);
  // When: topology created
  Topology topology = new DependencyStorageTopology(
      dependencyTopic,
      dependenciesRetentionPeriod,
      dependenciesWindowSize,
      true).get();
  TopologyDescription description = topology.describe();
  // Then: topology with 1 thread
  assertThat(description.subtopologies()).hasSize(1);
  // Given: streams configuration
  TopologyTestDriver testDriver = new TopologyTestDriver(topology, props);
  // When: a trace is passed
  ConsumerRecordFactory<String, DependencyLink> factory =
      new ConsumerRecordFactory<>(dependencyTopic, new StringSerializer(),
          dependencyLinkSerde.serializer());
  DependencyLink dependencyLink = DependencyLink.newBuilder()
      .parent("svc_a").child("svc_b").callCount(1).errorCount(0)
      .build();
  String dependencyLinkId = "svc_a:svc_b";
  testDriver.pipeInput(factory.create(dependencyTopic, dependencyLinkId, dependencyLink, 10L));
  WindowStore<String, DependencyLink> links = testDriver.getWindowStore(DEPENDENCIES_STORE_NAME);
  // Then: dependency link created
  WindowStoreIterator<DependencyLink> firstLink = links.fetch(dependencyLinkId, 0L, 100L);
  assertThat(firstLink).hasNext();
  assertThat(firstLink.next().value).isEqualTo(dependencyLink);
  // When: new links appear
  testDriver.pipeInput(factory.create(dependencyTopic, dependencyLinkId, dependencyLink, 90L));
  // Then: dependency link increases
  WindowStoreIterator<DependencyLink> secondLink = links.fetch(dependencyLinkId, 0L, 100L);
  assertThat(secondLink).hasNext();
  assertThat(secondLink.next().value.callCount()).isEqualTo(2);
  // When: time moves forward
  testDriver.advanceWallClockTime(dependenciesRetentionPeriod.toMillis() + 91L);
  testDriver.pipeInput(factory.create(dependencyTopic, dependencyLinkId, dependencyLink));
  // Then: dependency link is removed and restarted
  KeyValueIterator<Windowed<String>, DependencyLink> thirdLink = links.all();
  assertThat(thirdLink).hasNext();
  assertThat(thirdLink.next().value.callCount()).isEqualTo(1);
  // Close resources
  testDriver.close();
  dependencyLinkSerde.close();
}
 

@Test
public void testSplitStream() throws IOException {
    SplitStream ss = new SplitStream();
    Properties envProps = ss.loadEnvProperties(TEST_CONFIG_FILE);
    Properties streamProps = ss.buildStreamsProperties(envProps);

    String inputTopic = envProps.getProperty("input.topic.name");
    String outputDramaTopic = envProps.getProperty("output.drama.topic.name");
    String outputFantasyTopic = envProps.getProperty("output.fantasy.topic.name");
    String outputOtherTopic = envProps.getProperty("output.other.topic.name");

    Topology topology = ss.buildTopology(envProps);
    TopologyTestDriver testDriver = new TopologyTestDriver(topology, streamProps);

    Serializer<String> keySerializer = Serdes.String().serializer();
    SpecificAvroSerializer<ActingEvent> valueSerializer = makeSerializer(envProps);

    Deserializer<String> keyDeserializer = Serdes.String().deserializer();
    SpecificAvroDeserializer<ActingEvent> valueDeserializer = makeDeserializer(envProps);

    ConsumerRecordFactory<String, ActingEvent> inputFactory = new ConsumerRecordFactory<>(keySerializer, valueSerializer);

    ActingEvent streep = ActingEvent.newBuilder()
            .setName("Meryl Streep").setTitle("The Iron Lady").setGenre("drama").build();
    ActingEvent smith = ActingEvent.newBuilder()
            .setName("Will Smith").setTitle("Men in Black").setGenre("comedy").build();
    ActingEvent damon = ActingEvent.newBuilder()
            .setName("Matt Damon").setTitle("The Martian").setGenre("drama").build();
    ActingEvent garland = ActingEvent.newBuilder()
            .setName("Judy Garland").setTitle("The Wizard of Oz").setGenre("fantasy").build();
    ActingEvent aniston = ActingEvent.newBuilder()
            .setName("Jennifer Aniston").setTitle("Office Space").setGenre("comedy").build();
    ActingEvent murray = ActingEvent.newBuilder()
            .setName("Bill Murray").setTitle("Ghostbusters").setGenre("fantasy").build();
    ActingEvent bale = ActingEvent.newBuilder()
            .setName("Christian Bale").setTitle("The Dark Knight").setGenre("crime").build();
    ActingEvent dern = ActingEvent.newBuilder()
            .setName("Laura Dern").setTitle("Jurassic Park").setGenre("fantasy").build();
    ActingEvent reeves = ActingEvent.newBuilder()
            .setName("Keanu Reeves").setTitle("The Matrix").setGenre("fantasy").build();
    ActingEvent crowe = ActingEvent.newBuilder()
            .setName("Russell Crowe").setTitle("Gladiator").setGenre("drama").build();
    ActingEvent keaton = ActingEvent.newBuilder()
            .setName("Diane Keaton").setTitle("The Godfather: Part II").setGenre("crime").build();

    List<ActingEvent> input = new ArrayList<>();
    input.add(streep);
    input.add(smith);
    input.add(damon);
    input.add(garland);
    input.add(aniston);
    input.add(murray);
    input.add(bale);
    input.add(dern);
    input.add(reeves);
    input.add(crowe);
    input.add(keaton);

    List<ActingEvent> expectedDrama = new ArrayList<>();
    expectedDrama.add(streep);
    expectedDrama.add(damon);
    expectedDrama.add(crowe);

    List<ActingEvent> expectedFantasy = new ArrayList<>();
    expectedFantasy.add(garland);
    expectedFantasy.add(murray);
    expectedFantasy.add(dern);
    expectedFantasy.add(reeves);

    List<ActingEvent> expectedOther = new ArrayList<>();
    expectedOther.add(smith);
    expectedOther.add(aniston);
    expectedOther.add(bale);
    expectedOther.add(keaton);

    for (ActingEvent event : input) {
        testDriver.pipeInput(inputFactory.create(inputTopic, event.getName(), event));
    }

    List<ActingEvent> actualDrama = readOutputTopic(testDriver, outputDramaTopic, keyDeserializer, valueDeserializer);
    List<ActingEvent> actualFantasy = readOutputTopic(testDriver, outputFantasyTopic, keyDeserializer, valueDeserializer);
    List<ActingEvent> actualOther = readOutputTopic(testDriver, outputOtherTopic, keyDeserializer, valueDeserializer);

    Assert.assertEquals(expectedDrama, actualDrama);
    Assert.assertEquals(expectedFantasy, actualFantasy);
    Assert.assertEquals(expectedOther, actualOther);
}
 

@Test
public void shouldFilterDistinctEvents() throws IOException, RestClientException {

  final FindDistinctEvents distinctifier  = new FindDistinctEvents();

  String inputTopic = envProps.getProperty("input.topic.name");
  String outputTopic = envProps.getProperty("output.topic.name");

  final SpecificAvroSerde<Click> clickSerde = makeSerializer(envProps);

  Topology topology = distinctifier.buildTopology(envProps, clickSerde);
  TopologyTestDriver testDriver = new TopologyTestDriver(topology, streamProps);

  Serializer<String> keySerializer = Serdes.String().serializer();

  ConsumerRecordFactory<String, Click> inputFactory = new ConsumerRecordFactory<>(
          keySerializer, clickSerde.serializer());

  final List<Click> clicks = asList(
          new Click("10.0.0.1",
                  "https://docs.confluent.io/current/tutorials/examples/kubernetes/gke-base/docs/index.html",
          "2019-09-16T14:53:43+00:00"),
          new Click("10.0.0.2",
                  "https://www.confluent.io/hub/confluentinc/kafka-connect-datagen",
          "2019-09-16T14:53:43+00:01"),
          new Click("10.0.0.3",
          "https://www.confluent.io/hub/confluentinc/kafka-connect-datagen",
          "2019-09-16T14:53:43+00:03"),
          new Click("10.0.0.1",
          "https://docs.confluent.io/current/tutorials/examples/kubernetes/gke-base/docs/index.html",
          "2019-09-16T14:53:43+00:00"),
          new Click("10.0.0.2",
          "https://www.confluent.io/hub/confluentinc/kafka-connect-datagen",
          "2019-09-16T14:53:43+00:01"),
          new Click("10.0.0.3",
          "https://www.confluent.io/hub/confluentinc/kafka-connect-datagen",
          "2019-09-16T14:53:43+00:03"));

  final List<Click> expectedOutput = asList(clicks.get(0),clicks.get(1),clicks.get(2));

  for (Click clk : clicks) {
    testDriver.pipeInput(inputFactory.create(inputTopic, clk.getIp(), clk));
  }

  Deserializer<String> keyDeserializer = Serdes.String().deserializer();
  List<Click> actualOutput = new ArrayList<>();
  while (true) {
    ProducerRecord<String, Click>
        record =
        testDriver.readOutput(outputTopic, keyDeserializer, clickSerde.deserializer());

    if (record != null) {
      actualOutput.add(record.value());
    } else {
      break;
    }
  }

  Assert.assertEquals(expectedOutput, actualOutput);
}
 

@Test
public void testJoin() throws IOException {
    JoinStreamToTable jst = new JoinStreamToTable();
    Properties envProps = jst.loadEnvProperties(TEST_CONFIG_FILE);
    Properties streamProps = jst.buildStreamsProperties(envProps);

    String tableTopic = envProps.getProperty("movie.topic.name");
    String streamTopic = envProps.getProperty("rating.topic.name");
    String outputTopic = envProps.getProperty("rated.movies.topic.name");

    Topology topology = jst.buildTopology(envProps);
    TopologyTestDriver testDriver = new TopologyTestDriver(topology, streamProps);

    Serializer<String> keySerializer = Serdes.String().serializer();
    SpecificAvroSerializer<Movie> movieSerializer = makeMovieSerializer(envProps);
    SpecificAvroSerializer<Rating> ratingSerializer = makeRatingSerializer(envProps);

    Deserializer<String> stringDeserializer = Serdes.String().deserializer();
    SpecificAvroDeserializer<RatedMovie> valueDeserializer = makeRatedMovieDeserializer(envProps);

    ConsumerRecordFactory<String, Movie> movieFactory = new ConsumerRecordFactory<>(keySerializer, movieSerializer);
    ConsumerRecordFactory<String, Rating> ratingFactory = new ConsumerRecordFactory<>(keySerializer, ratingSerializer);


    List<Movie> movies = new ArrayList<>();
    movies.add(Movie.newBuilder().setId(294).setTitle("Die Hard").setReleaseYear(1988).build());
    movies.add(Movie.newBuilder().setId(354).setTitle("Tree of Life").setReleaseYear(2011).build());
    movies.add(Movie.newBuilder().setId(782).setTitle("A Walk in the Clouds").setReleaseYear(1998).build());
    movies.add(Movie.newBuilder().setId(128).setTitle("The Big Lebowski").setReleaseYear(1998).build());
    movies.add(Movie.newBuilder().setId(780).setTitle("Super Mario Bros.").setReleaseYear(1993).build());

    List<Rating> ratings = new ArrayList<>();
    ratings.add(Rating.newBuilder().setId(294).setRating(8.2).build());
    ratings.add(Rating.newBuilder().setId(294).setRating(8.5).build());
    ratings.add(Rating.newBuilder().setId(354).setRating(9.9).build());
    ratings.add(Rating.newBuilder().setId(354).setRating(9.7).build());
    ratings.add(Rating.newBuilder().setId(782).setRating(7.8).build());
    ratings.add(Rating.newBuilder().setId(782).setRating(7.7).build());
    ratings.add(Rating.newBuilder().setId(128).setRating(8.7).build());
    ratings.add(Rating.newBuilder().setId(128).setRating(8.4).build());
    ratings.add(Rating.newBuilder().setId(780).setRating(2.1).build());

    List<RatedMovie> ratedMovies = new ArrayList<>();
    ratedMovies.add(RatedMovie.newBuilder().setTitle("Die Hard").setId(294).setReleaseYear(1988).setRating(8.2).build());
    ratedMovies.add(RatedMovie.newBuilder().setTitle("Die Hard").setId(294).setReleaseYear(1988).setRating(8.5).build());
    ratedMovies.add(RatedMovie.newBuilder().setTitle("Tree of Life").setId(354).setReleaseYear(2011).setRating(9.9).build());
    ratedMovies.add(RatedMovie.newBuilder().setTitle("Tree of Life").setId(354).setReleaseYear(2011).setRating(9.7).build());
    ratedMovies.add(RatedMovie.newBuilder().setId(782).setTitle("A Walk in the Clouds").setReleaseYear(1998).setRating(7.8).build());
    ratedMovies.add(RatedMovie.newBuilder().setId(782).setTitle("A Walk in the Clouds").setReleaseYear(1998).setRating(7.7).build());
    ratedMovies.add(RatedMovie.newBuilder().setId(128).setTitle("The Big Lebowski").setReleaseYear(1998).setRating(8.7).build());
    ratedMovies.add(RatedMovie.newBuilder().setId(128).setTitle("The Big Lebowski").setReleaseYear(1998).setRating(8.4).build());
    ratedMovies.add(RatedMovie.newBuilder().setId(780).setTitle("Super Mario Bros.").setReleaseYear(1993).setRating(2.1).build());

    for(Movie movie : movies) {
        testDriver.pipeInput(movieFactory.create(tableTopic, movie.getId().toString(), movie));
    }

    for(Rating rating: ratings) {
        testDriver.pipeInput(ratingFactory.create(streamTopic, rating.getId().toString(), rating));
    }

    List<RatedMovie> actualOutput = readOutputTopic(testDriver, outputTopic, stringDeserializer, valueDeserializer);

    assertEquals(ratedMovies, actualOutput);
}
 

@Test
public void testWindows() throws IOException {
    TumblingWindow tw = new TumblingWindow();
    Properties envProps = tw.loadEnvProperties(TEST_CONFIG_FILE);
    Properties streamProps = tw.buildStreamsProperties(envProps);

    String inputTopic = envProps.getProperty("rating.topic.name");
    String outputTopic = envProps.getProperty("rating.count.topic.name");

    Topology topology = tw.buildTopology(envProps);
    TopologyTestDriver testDriver = new TopologyTestDriver(topology, streamProps);

    Serializer<String> stringSerializer = Serdes.String().serializer();
    SpecificAvroSerializer<Rating> ratingSerializer = makeRatingSerializer(envProps);
    Deserializer<String> stringDeserializer = Serdes.String().deserializer();

    ConsumerRecordFactory<String, Rating> ratingFactory = new ConsumerRecordFactory<>(stringSerializer, ratingSerializer);

    List<Rating> ratings = new ArrayList<>();
    ratings.add(Rating.newBuilder().setTitle("Die Hard").setReleaseYear(1988).setRating(8.2).setTimestamp("2019-04-25T18:00:00-0000").build());
    ratings.add(Rating.newBuilder().setTitle("Die Hard").setReleaseYear(1988).setRating(7.6).setTimestamp("2019-04-25T18:05:00-0000").build());
    ratings.add(Rating.newBuilder().setTitle("Tree of Life").setReleaseYear(2011).setRating(4.9).setTimestamp("2019-04-25T21:00:00-0000").build());
    ratings.add(Rating.newBuilder().setTitle("Tree of Life").setReleaseYear(2011).setRating(9.9).setTimestamp("2019-04-25T21:11:00-0000").build());
    ratings.add(Rating.newBuilder().setTitle("A Walk in the Clouds").setReleaseYear(1998).setRating(3.6).setTimestamp("2019-04-25T13:00:00-0000").build());
    ratings.add(Rating.newBuilder().setTitle("A Walk in the Clouds").setReleaseYear(1998).setRating(7.1).setTimestamp("2019-04-25T13:01:00-0000").build());
    ratings.add(Rating.newBuilder().setTitle("The Big Lebowski").setReleaseYear(1998).setRating(8.6).setTimestamp("2019-04-25T19:30:00-0000").build());
    ratings.add(Rating.newBuilder().setTitle("The Big Lebowski").setReleaseYear(1998).setRating(7.0).setTimestamp("2019-04-25T19:35:00-0000").build());
    ratings.add(Rating.newBuilder().setTitle("Super Mario Bros.").setReleaseYear(1993).setRating(3.5).setTimestamp("2019-04-25T11:15:00-0000").build());
    ratings.add(Rating.newBuilder().setTitle("Super Mario Bros.").setReleaseYear(1993).setRating(2.0).setTimestamp("2019-04-25T11:40:00-0000").build());

    List<RatingCount> ratingCounts = new ArrayList<>();
    ratingCounts.add(new RatingCount("Die Hard", "1"));
    ratingCounts.add(new RatingCount("Die Hard", "2"));
    ratingCounts.add(new RatingCount("Tree of Life", "1"));
    ratingCounts.add(new RatingCount("Tree of Life", "1"));
    ratingCounts.add(new RatingCount("A Walk in the Clouds", "1"));
    ratingCounts.add(new RatingCount("A Walk in the Clouds", "2"));
    ratingCounts.add(new RatingCount("The Big Lebowski", "1"));
    ratingCounts.add(new RatingCount("The Big Lebowski", "2"));
    ratingCounts.add(new RatingCount("Super Mario Bros.", "1"));
    ratingCounts.add(new RatingCount("Super Mario Bros.", "1"));

    for(Rating rating : ratings) {
        testDriver.pipeInput(ratingFactory.create(inputTopic, rating.getTitle(), rating));
    }

    List<RatingCount> actualOutput = readOutputTopic(testDriver,
                                                     outputTopic,
                                                     stringDeserializer,
                                                     stringDeserializer);

    assertEquals(ratingCounts.size(), actualOutput.size());
    for(int n = 0; n < ratingCounts.size(); n++) {
        assertEquals(ratingCounts.get(n).toString(), actualOutput.get(n).toString());
    }
}
 

@Test
public void shouldSumTicketSales() throws IOException, RestClientException {
  AggregatingSum aggSum = new AggregatingSum();
  Properties envProps = aggSum.loadEnvProperties(TEST_CONFIG_FILE);
  Properties streamProps = aggSum.buildStreamsProperties(envProps);

  String inputTopic = envProps.getProperty("input.topic.name");
  String outputTopic = envProps.getProperty("output.topic.name");

  final SpecificAvroSerde<TicketSale> ticketSaleSpecificAvroSerde = makeSerializer(envProps);

  Topology topology = aggSum.buildTopology(envProps, ticketSaleSpecificAvroSerde);
  TopologyTestDriver testDriver = new TopologyTestDriver(topology, streamProps);

  Serializer<String> keySerializer = Serdes.String().serializer();
  Deserializer<String> keyDeserializer = Serdes.String().deserializer();

  ConsumerRecordFactory<String, TicketSale>
      inputFactory =
      new ConsumerRecordFactory<>(keySerializer, ticketSaleSpecificAvroSerde.serializer());

  final List<TicketSale>
      input = asList(
                new TicketSale("Die Hard", "2019-07-18T10:00:00Z", 12),
                new TicketSale("Die Hard", "2019-07-18T10:01:00Z", 12),
                new TicketSale("The Godfather", "2019-07-18T10:01:31Z", 12),
                new TicketSale("Die Hard", "2019-07-18T10:01:36Z", 24),
                new TicketSale("The Godfather", "2019-07-18T10:02:00Z", 18),
                new TicketSale("The Big Lebowski", "2019-07-18T11:03:21Z", 12),
                new TicketSale("The Big Lebowski", "2019-07-18T11:03:50Z", 12),
                new TicketSale("The Godfather", "2019-07-18T11:40:00Z", 36),
                new TicketSale("The Godfather", "2019-07-18T11:40:09Z", 18)
              );

  List<Integer> expectedOutput = new ArrayList<Integer>(Arrays.asList(12, 24, 12, 48, 30, 12, 24, 66, 84));

  for (TicketSale ticketSale : input) {
    testDriver.pipeInput(inputFactory.create(inputTopic, "", ticketSale));
  }

  List<Integer> actualOutput = new ArrayList<>();
  while (true) {
    ProducerRecord<String, Integer>
        record =
        testDriver.readOutput(outputTopic, keyDeserializer, Serdes.Integer().deserializer());

    if (record != null) {
      actualOutput.add(record.value());
    } else {
      break;
    }
  }

  System.out.println(actualOutput);
  Assert.assertEquals(expectedOutput, actualOutput);

}
 

@Test
public void shouldCountTicketSales() throws IOException, RestClientException {
  AggregatingCount aggCount = new AggregatingCount();
  Properties envProps = aggCount.loadEnvProperties(TEST_CONFIG_FILE);
  Properties streamProps = aggCount.buildStreamsProperties(envProps);

  String inputTopic = envProps.getProperty("input.topic.name");
  String outputTopic = envProps.getProperty("output.topic.name");

  final SpecificAvroSerde<TicketSale> ticketSaleSpecificAvroSerde = makeSerializer(envProps);

  Topology topology = aggCount.buildTopology(envProps, ticketSaleSpecificAvroSerde);
  TopologyTestDriver testDriver = new TopologyTestDriver(topology, streamProps);

  Serializer<String> keySerializer = Serdes.String().serializer();
  Deserializer<String> keyDeserializer = Serdes.String().deserializer();

  ConsumerRecordFactory<String, TicketSale>
      inputFactory =
      new ConsumerRecordFactory<>(keySerializer, ticketSaleSpecificAvroSerde.serializer());

  final List<TicketSale>
      input = asList(
                new TicketSale("Die Hard", "2019-07-18T10:00:00Z", 12),
                new TicketSale("Die Hard", "2019-07-18T10:01:00Z", 12),
                new TicketSale("The Godfather", "2019-07-18T10:01:31Z", 12),
                new TicketSale("Die Hard", "2019-07-18T10:01:36Z", 24),
                new TicketSale("The Godfather", "2019-07-18T10:02:00Z", 18),
                new TicketSale("The Big Lebowski", "2019-07-18T11:03:21Z", 12),
                new TicketSale("The Big Lebowski", "2019-07-18T11:03:50Z", 12),
                new TicketSale("The Godfather", "2019-07-18T11:40:00Z", 36),
                new TicketSale("The Godfather", "2019-07-18T11:40:09Z", 18)
              );

  List<Long> expectedOutput = new ArrayList<Long>(Arrays.asList(1L, 2L, 1L, 3L, 2L, 1L, 2L, 3L, 4L));

  for (TicketSale ticketSale : input) {
    testDriver.pipeInput(inputFactory.create(inputTopic, "", ticketSale));
  }

  List<Long> actualOutput = new ArrayList<>();
  while (true) {
    ProducerRecord<String, Long>
        record =
        testDriver.readOutput(outputTopic, keyDeserializer, Serdes.Long().deserializer());

    if (record != null) {
      actualOutput.add(record.value());
    } else {
      break;
    }
  }

  System.out.println(actualOutput);
  Assert.assertEquals(expectedOutput, actualOutput);

}
 

@Test
public void shouldCreateKeyedStream() throws IOException, RestClientException {
  StreamsIngest si = new StreamsIngest();
  Properties envProps = si.loadEnvProperties(TEST_CONFIG_FILE);
  Properties streamProps = si.buildStreamsProperties(envProps);

  String inputTopic = envProps.getProperty("input.topic.name");
  String outputTopic = envProps.getProperty("output.topic.name");

  final SpecificAvroSerde<City> citySpecificAvroSerde = makeSerializer(envProps);

  Topology topology = si.buildTopology(envProps, citySpecificAvroSerde);
  TopologyTestDriver testDriver = new TopologyTestDriver(topology, streamProps);

  Serializer<String> keySerializer = Serdes.String().serializer();
  Deserializer<Long> keyDeserializer = Serdes.Long().deserializer();

  ConsumerRecordFactory<String, City>
      inputFactory =
      new ConsumerRecordFactory<>(keySerializer, citySpecificAvroSerde.serializer());

  // Fixture
  City c1 = new City(1L, "Raleigh", "NC");
  City c2 = new City(2L, "Mountain View", "CA");
  City c3 = new City(3L, "Knoxville", "TN");
  City c4 = new City(4L, "Houston", "TX");
  City c5 = new City(5L, "Olympia", "WA");
  City c6 = new City(6L, "Bismarck", "ND");
  // end Fixture

  final List<City>
      input = asList(c1, c2, c3, c4, c5, c6);

  final List<Long> expectedOutput = asList(1L, 2L, 3L, 4L, 5L, 6L);

  for (City city : input) {
    testDriver.pipeInput(inputFactory.create(inputTopic, null, city));
  }

  List<Long> actualOutput = new ArrayList<>();
  while (true) {
    ProducerRecord<Long, City>
        record =
        testDriver.readOutput(outputTopic, keyDeserializer, citySpecificAvroSerde.deserializer());

    if (record != null) {
      actualOutput.add(record.key());
    } else {
      break;
    }
  }

  Assert.assertEquals(expectedOutput, actualOutput);
}
 

@Test
public void shouldFilterGRRMartinsBooks() throws IOException, RestClientException {
  FilterEvents fe = new FilterEvents();
  Properties envProps = fe.loadEnvProperties(TEST_CONFIG_FILE);
  Properties streamProps = fe.buildStreamsProperties(envProps);

  String inputTopic = envProps.getProperty("input.topic.name");
  String outputTopic = envProps.getProperty("output.topic.name");

  final SpecificAvroSerde<Publication> publicationSpecificAvroSerde = makeSerializer(envProps);

  Topology topology = fe.buildTopology(envProps, publicationSpecificAvroSerde);
  TopologyTestDriver testDriver = new TopologyTestDriver(topology, streamProps);

  Serializer<String> keySerializer = Serdes.String().serializer();
  Deserializer<String> keyDeserializer = Serdes.String().deserializer();

  ConsumerRecordFactory<String, Publication>
      inputFactory =
      new ConsumerRecordFactory<>(keySerializer, publicationSpecificAvroSerde.serializer());

  // Fixture
  Publication iceAndFire = new Publication("George R. R. Martin", "A Song of Ice and Fire");
  Publication silverChair = new Publication("C.S. Lewis", "The Silver Chair");
  Publication perelandra = new Publication("C.S. Lewis", "Perelandra");
  Publication fireAndBlood = new Publication("George R. R. Martin", "Fire & Blood");
  Publication theHobbit = new Publication("J. R. R. Tolkien", "The Hobbit");
  Publication lotr = new Publication("J. R. R. Tolkien", "The Lord of the Rings");
  Publication dreamOfSpring = new Publication("George R. R. Martin", "A Dream of Spring");
  Publication fellowship = new Publication("J. R. R. Tolkien", "The Fellowship of the Ring");
  Publication iceDragon = new Publication("George R. R. Martin", "The Ice Dragon");
  // end Fixture

  final List<Publication>
      input = asList(iceAndFire, silverChair, perelandra, fireAndBlood, theHobbit, lotr, dreamOfSpring, fellowship,
                     iceDragon);

  final List<Publication> expectedOutput = asList(iceAndFire, fireAndBlood, dreamOfSpring, iceDragon);

  for (Publication publication : input) {
    testDriver.pipeInput(inputFactory.create(inputTopic, publication.getName(), publication));
  }

  List<Publication> actualOutput = new ArrayList<>();
  while (true) {
    ProducerRecord<String, Publication>
        record =
        testDriver.readOutput(outputTopic, keyDeserializer, publicationSpecificAvroSerde.deserializer());

    if (record != null) {
      actualOutput.add(record.value());
    } else {
      break;
    }
  }

  Assert.assertEquals(expectedOutput, actualOutput);
}
 

private void nullOnDeserException(TopologyTestDriver driver) {
    driver.pipeInput(stringFactory.create(INPUT_TOPIC, KAFKA_KEY, INVALID_INPUT_VALUE));
    val record = driver.readOutput(DEFAULT_OUTPUT_TOPIC, stringDeser, mmdDeser);
    Assert.assertNull(record);
}