io.vertx.kafka.client.consumer.KafkaConsumer Java Examples

@Override
public void start(Promise<Void> startFuture) throws Exception {

  Map<String, String> config = new HashMap<>();
  config.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
  config.put(ConsumerConfig.GROUP_ID_CONFIG, "my-group");
  config.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");
  config.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
  config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
  config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());

  KafkaConsumer consumer = KafkaConsumer.create(vertx, config);
  consumer.handler(r -> {

    System.out.println(r);
  });
  consumer.subscribe("my-topic");

  startFuture.complete();
}
 

public void exampleUsingVertxDeserializers2(Vertx vertx) {
  Map<String, String> config = new HashMap<>();
  config.put("bootstrap.servers", "localhost:9092");
  config.put("group.id", "my_group");
  config.put("auto.offset.reset", "earliest");
  config.put("enable.auto.commit", "false");

  // Creating a consumer able to deserialize buffers
  KafkaConsumer<Buffer, Buffer> bufferConsumer = KafkaConsumer.create(vertx, config, Buffer.class, Buffer.class);

  // Creating a consumer able to deserialize json objects
  KafkaConsumer<JsonObject, JsonObject> jsonObjectConsumer = KafkaConsumer.create(vertx, config, JsonObject.class, JsonObject.class);

  // Creating a consumer able to deserialize json arrays
  KafkaConsumer<JsonArray, JsonArray> jsonArrayConsumer = KafkaConsumer.create(vertx, config, JsonArray.class, JsonArray.class);
}
 

/**
 * Example about how Kafka consumer can pause reading from a topic partition
 * and then resume read operation for continuing to get messages
 * @param vertx Vert.x instance
 * @param consumer KafkaConsumer instance
 */
public void exampleConsumerFlowControl(Vertx vertx, KafkaConsumer<String, String> consumer) {
  TopicPartition topicPartition = new TopicPartition()
    .setTopic("test")
    .setPartition(0);

  // registering the handler for incoming messages
  consumer.handler(record -> {
    System.out.println("key=" + record.key() + ",value=" + record.value() +
      ",partition=" + record.partition() + ",offset=" + record.offset());

    // i.e. pause/resume on partition 0, after reading message up to offset 5
    if ((record.partition() == 0) && (record.offset() == 5)) {

      // pause the read operations
      consumer.pause(topicPartition)
        .onSuccess(v -> System.out.println("Paused"))
        .onSuccess(v -> vertx.setTimer(5000, timeId ->
          // resume read operations
          consumer.resume(topicPartition)
        ));
    }
  });
}
 

/**
 * Example to demonstrate how one can use the new endOffsets API (introduced with Kafka
 * 0.10.1.1) to look up the last offset for a given partition.
 * @param consumer Consumer to be used
 */
public void exampleConsumerEndOffsets(KafkaConsumer<String, String> consumer) {
  Set<TopicPartition> topicPartitions = new HashSet<>();
  TopicPartition topicPartition = new TopicPartition().setTopic("test").setPartition(0);
  topicPartitions.add(topicPartition);

  consumer.endOffsets(topicPartitions)
    .onSuccess(results ->
      results.forEach((topic, beginningOffset) ->
        System.out.println(
          "End offset for topic=" + topic.getTopic() + ", partition=" +
            topic.getPartition() + ", beginningOffset=" + beginningOffset
        )
      )
    );

  // Convenience method for single-partition lookup
  consumer
    .endOffsets(topicPartition)
    .onSuccess(endOffset ->
      System.out.println(
        "End offset for topic=" + topicPartition.getTopic() + ", partition=" +
          topicPartition.getPartition() + ", endOffset=" + endOffset
      )
  );
}
 

/**
 * Kafka consumer initialization. It should be the first call for preparing the Kafka consumer.
 */
protected void initConsumer(boolean shouldAttachBatchHandler, Properties config) {

    // create a consumer
    KafkaConfig kafkaConfig = this.bridgeConfig.getKafkaConfig();
    Properties props = new Properties();
    props.putAll(kafkaConfig.getConfig());
    props.putAll(kafkaConfig.getConsumerConfig().getConfig());
    props.put(ConsumerConfig.GROUP_ID_CONFIG, this.groupId);
    if (this.bridgeConfig.getTracing() != null) {
        props.put(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, TracingConsumerInterceptor.class.getName());
    }

    if (config != null)
        props.putAll(config);

    this.consumer = KafkaConsumer.create(this.vertx, props, keyDeserializer, valueDeserializer);

    if (shouldAttachBatchHandler)
        this.consumer.batchHandler(this::handleKafkaBatch);
}
 

@Test
// Regression test for ISS-73: undeployment of a verticle with unassigned consumer fails
public void testUndeployUnassignedConsumer(TestContext ctx) {
  Properties config = kafkaCluster.useTo().getConsumerProperties("testUndeployUnassignedConsumer_consumer",
    "testUndeployUnassignedConsumer_consumer", OffsetResetStrategy.EARLIEST);
  config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
  config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);

  Async async = ctx.async(1);
  vertx.deployVerticle(new AbstractVerticle() {
    @Override
    public void start() {
      KafkaConsumer<String, String> consumer = KafkaConsumer.create(vertx, config);
    }
  }, ctx.asyncAssertSuccess(id -> {
    vertx.undeploy(id, ctx.asyncAssertSuccess(v2 -> async.complete()));
  }));

  async.awaitSuccess(10000);
  waitUntil("Expected " + countThreads("vert.x-kafka-consumer-thread") + " == " + numVertxKafkaConsumerThread, () -> countThreads("vert.x-kafka-consumer-thread") == numKafkaConsumerNetworkThread);
}
 

@Test
public void testPollTimeout(TestContext ctx) throws Exception {
  Async async = ctx.async();
  String topicName = "testPollTimeout";
  Properties config = kafkaCluster.useTo().getConsumerProperties(topicName, topicName, OffsetResetStrategy.EARLIEST);
  config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
  config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);

  io.vertx.kafka.client.common.TopicPartition topicPartition = new io.vertx.kafka.client.common.TopicPartition(topicName, 0);
  KafkaConsumer<Object, Object> consumerWithCustomTimeout = KafkaConsumer.create(vertx, config);

  int pollingTimeout = 1500;
  // Set the polling timeout to 1500 ms (default is 1000)
  consumerWithCustomTimeout.pollTimeout(Duration.ofMillis(pollingTimeout));
  // Subscribe to the empty topic (we want the poll() call to timeout!)
  consumerWithCustomTimeout.subscribe(topicName, subscribeRes -> {
    consumerWithCustomTimeout.handler(rec -> {}); // Consumer will now immediately poll once
    long beforeSeek = System.currentTimeMillis();
    consumerWithCustomTimeout.seekToBeginning(topicPartition, seekRes -> {
      long durationWShortTimeout = System.currentTimeMillis() - beforeSeek;
      ctx.assertTrue(durationWShortTimeout >= pollingTimeout, "Operation must take at least as long as the polling timeout");
      consumerWithCustomTimeout.close();
      async.countDown();
    });
  });
}
 

public void exampleSubscribe(KafkaConsumer<String, String> consumer) {
  // register the handler for incoming messages
  consumer.handler(record -> {
    System.out.println("Processing key=" + record.key() + ",value=" + record.value() +
      ",partition=" + record.partition() + ",offset=" + record.offset());
  });

  // subscribe to several topics with list
  Set<String> topics = new HashSet<>();
  topics.add("topic1");
  topics.add("topic2");
  topics.add("topic3");
  consumer.subscribe(topics);

  // or using a Java regex
  Pattern pattern = Pattern.compile("topic\\d");
  consumer.subscribe(pattern);

  // or just subscribe to a single topic
  consumer.subscribe("a-single-topic");
}
 

/**
 * Example about how Kafka consumer receives messages
 * from a topic requesting a specific partition for that
 * @param consumer KafkaConsumer instance
 */
public void exampleConsumerAssignPartition(KafkaConsumer<String, String> consumer) {
  // register the handler for incoming messages
  consumer.handler(record -> {
    System.out.println("key=" + record.key() + ",value=" + record.value() +
      ",partition=" + record.partition() + ",offset=" + record.offset());
  });

  //
  Set<TopicPartition> topicPartitions = new HashSet<>();
  topicPartitions.add(new TopicPartition()
    .setTopic("test")
    .setPartition(0));

  // requesting to be assigned the specific partition
  consumer
    .assign(topicPartitions)
    .onSuccess(v -> System.out.println("Partition assigned"))
    // After the assignment is completed, get the assigned partitions to this consumer
    .compose(v -> consumer.assignment())
    .onSuccess(partitions -> {
      for (TopicPartition topicPartition : partitions) {
        System.out.println(topicPartition.getTopic() + " " + topicPartition.getPartition());
      }
    });
}
 

protected <V, K> KafkaConsumer<K, V> installConsumerSpy(AmqpSinkBridgeEndpoint<K, V> endpoint)
        throws NoSuchFieldException, IllegalAccessException {
    Field consumerField = AmqpSinkBridgeEndpoint.class.getSuperclass().getDeclaredField("consumer");
    consumerField.setAccessible(true);
    KafkaConsumer<K, V> consumer = (KafkaConsumer<K, V>) consumerField.get(endpoint);
    KafkaConsumer<K, V> consumerSpy = spy(consumer);
    consumerField.set(endpoint, consumerSpy);
    return consumerSpy;
}
 

/**
 * Example about how it's possible to get information
 * on partitions for a specified topic (valid for both
 * Kafka consumer and producer instances)
 * @param consumer KafkaConsumer instance
 */
public void exampleConsumerPartitionsFor(KafkaConsumer<String, String> consumer) {
  // asking partitions information about specific topic
  consumer
    .partitionsFor("test")
    .onSuccess(partitions -> {
      for (PartitionInfo partitionInfo : partitions) {
        System.out.println(partitionInfo);
      }
    });
}
 

/**
 * Example about how it's possible to get messages from Kafka
 * doing poll at application level instead of using the internal
 * one in the client
 *
 * @param vertx Vert.x instance
 * @param consumer KafkaConsumer instance
 */
public void exampleConsumerWithPoll(Vertx vertx, KafkaConsumer<String, String> consumer) {
  // subscribes to the topic
  consumer
    .subscribe("test")
    .onSuccess(v -> {
      System.out.println("Consumer subscribed");

      // Let's poll every second
      vertx.setPeriodic(1000, timerId ->
        consumer
          .poll(Duration.ofMillis(100))
          .onSuccess(records -> {
            for (int i = 0; i < records.size(); i++) {
              KafkaConsumerRecord<String, String> record = records.recordAt(i);
              System.out.println("key=" + record.key() + ",value=" + record.value() +
                ",partition=" + record.partition() + ",offset=" + record.offset());
            }
          })
          .onFailure(cause -> {
            System.out.println("Something went wrong when polling " + cause.toString());
            cause.printStackTrace();

            // Stop polling if something went wrong
            vertx.cancelTimer(timerId);
          })
      );
  });
}
 

/**
 * Example about how Kafka consumer can handle manual commit
 * of the current offset for a topic partition
 * @param consumer KafkaConsumer instance
 */
public void exampleConsumerManualOffsetCommit(KafkaConsumer<String, String> consumer) {
  // consumer is processing read messages

  // committing offset of the last read message
  consumer.commit().onSuccess(v ->
    System.out.println("Last read message offset committed")
  );
}
 

public void exampleSeek(KafkaConsumer<String, String> consumer) {
  TopicPartition topicPartition = new TopicPartition()
    .setTopic("test")
    .setPartition(0);

  // seek to a specific offset
  consumer
    .seek(topicPartition, 10)
    .onSuccess(v -> System.out.println("Seeking done"));

}
 

public void exampleSeekToBeginning(KafkaConsumer<String, String> consumer) {
  TopicPartition topicPartition = new TopicPartition()
    .setTopic("test")
    .setPartition(0);

  // seek to the beginning of the partition
  consumer
    .seekToBeginning(Collections.singleton(topicPartition))
    .onSuccess(v -> System.out.println("Seeking done"));
}
 

/**
 * Example to demonstrate how one can use the new offsetsForTimes API (introduced with Kafka
 * 0.10.1.1) to look up an offset by timestamp, i.e. search parameter is an epoch timestamp and
 * the call returns the lowest offset with ingestion timestamp >= given timestamp.
 * @param consumer Consumer to be used
 */
public void exampleConsumerOffsetsForTimes(KafkaConsumer<String, String> consumer) {
  Map<TopicPartition, Long> topicPartitionsWithTimestamps = new HashMap<>();
  TopicPartition topicPartition = new TopicPartition().setTopic("test").setPartition(0);

  // We are interested in the offset for data ingested 60 seconds ago
  long timestamp = (System.currentTimeMillis() - 60000);

  topicPartitionsWithTimestamps.put(topicPartition, timestamp);
  consumer
    .offsetsForTimes(topicPartitionsWithTimestamps)
    .onSuccess(results ->
      results.forEach((topic, offset) ->
        System.out.println(
          "Offset for topic=" + topic.getTopic() +
          ", partition=" + topic.getPartition() + "\n" +
          ", timestamp=" + timestamp + ", offset=" + offset.getOffset() +
          ", offsetTimestamp=" + offset.getTimestamp()
        )
      )
  );

  // Convenience method for single-partition lookup
  consumer.offsetsForTimes(topicPartition, timestamp).onSuccess(offsetAndTimestamp ->
    System.out.println(
      "Offset for topic=" + topicPartition.getTopic() +
      ", partition=" + topicPartition.getPartition() + "\n" +
      ", timestamp=" + timestamp + ", offset=" + offsetAndTimestamp.getOffset() +
      ", offsetTimestamp=" + offsetAndTimestamp.getTimestamp()
    )
  );
}
 

/**
 * Example to demonstrate how one can use the new beginningOffsets API (introduced with Kafka
 * 0.10.1.1) to look up the first offset for a given partition.
 * @param consumer Consumer to be used
 */
public void exampleConsumerBeginningOffsets(KafkaConsumer<String, String> consumer) {
  Set<TopicPartition> topicPartitions = new HashSet<>();
  TopicPartition topicPartition = new TopicPartition().setTopic("test").setPartition(0);
  topicPartitions.add(topicPartition);

  consumer
    .beginningOffsets(topicPartitions)
    .onSuccess(results ->
      results.forEach((topic, beginningOffset) ->
        System.out.println(
          "Beginning offset for topic=" + topic.getTopic() + ", partition=" +
            topic.getPartition() + ", beginningOffset=" + beginningOffset
        )
      )
    );

  // Convenience method for single-partition lookup
  consumer
    .beginningOffsets(topicPartition)
    .onSuccess(beginningOffset ->
      System.out.println(
        "Beginning offset for topic=" + topicPartition.getTopic() + ", partition=" +
          topicPartition.getPartition() + ", beginningOffset=" + beginningOffset
      )
    );
}
 

/**
 * Example about how Kafka consumer can seek in the partition
 * changing the offset from which starting to read messages
 * @param consumer KafkaConsumer instance
 */
public void exampleSeekToEnd(KafkaConsumer<String, String> consumer) {
  TopicPartition topicPartition = new TopicPartition()
    .setTopic("test")
    .setPartition(0);

  // seek to the end of the partition
  consumer
    .seekToEnd(Collections.singleton(topicPartition))
    .onSuccess(v -> System.out.println("Seeking done"));
}
 

/**
 * Example about how Kafka client (producer or consumer)
 * can handle errors and exception during communication
 * with the Kafka cluster
 * @param consumer KafkaProducer instance
 */
public void exampleErrorHandling(KafkaConsumer<String, String> consumer) {
  // setting handler for errors
  consumer.exceptionHandler(e -> {
    System.out.println("Error = " + e.getMessage());
  });
}
 

/**
 * Example about Kafka consumer and producer creation
 * @param vertx Vert.x instance
 */
public void exampleCreateConsumer(Vertx vertx) {
  // creating the consumer using map config
  Map<String, String> config = new HashMap<>();
  config.put("bootstrap.servers", "localhost:9092");
  config.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
  config.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
  config.put("group.id", "my_group");
  config.put("auto.offset.reset", "earliest");
  config.put("enable.auto.commit", "false");

  // use consumer for interacting with Apache Kafka
  KafkaConsumer<String, String> consumer = KafkaConsumer.create(vertx, config);
}
 

Consumer(KafkaClientProperties properties, CompletableFuture<Integer> resultPromise, IntPredicate msgCntPredicate, String topic, String clientName) {
    super(resultPromise, msgCntPredicate);
    this.properties = properties;
    this.topic = topic;
    this.clientName = clientName;
    this.vertx = Vertx.vertx();
    this.consumer = KafkaConsumer.create(vertx, properties.getProperties());
}
 

@Override
public KafkaConsumer<K, V> assignment(Handler<AsyncResult<Set<TopicPartition>>> handler) {
  this.stream.assignment(done -> {

    if (done.succeeded()) {
      handler.handle(Future.succeededFuture(Helper.from(done.result())));
    } else {
      handler.handle(Future.failedFuture(done.cause()));
    }

  });
  return this;
}
 

@Test
public void testConsumeWithPollNoMessages(TestContext ctx) {
  final String topicName = "testConsumeWithPollNoMessages";
  final String consumerId = topicName;
  Properties config = kafkaCluster.useTo().getConsumerProperties(consumerId, consumerId, OffsetResetStrategy.EARLIEST);
  config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
  config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
  KafkaConsumer<String, String> consumer = KafkaConsumer.create(vertx, config);
  Async done = ctx.async();
  AtomicInteger count = new AtomicInteger(5);

  consumer.subscribe(Collections.singleton(topicName), subscribeResult -> {

    if (subscribeResult.succeeded()) {

      vertx.setPeriodic(1000, t -> {
        consumer.poll(Duration.ofMillis(100), pollResult -> {
          if (pollResult.succeeded()) {
            if (pollResult.result().size() > 0) {
              ctx.fail();
            } else {
              if (count.decrementAndGet() == 0) {
                vertx.cancelTimer(t);
                done.complete();
              }
            }
          } else {
            ctx.fail();
          }
        });
      });

    } else {
      ctx.fail();
    }
  });
}
 

@Override
public KafkaConsumer<K, V> listTopics(Handler<AsyncResult<Map<String,List<PartitionInfo>>>> handler) {
  this.stream.listTopics(done -> {

    if (done.succeeded()) {
      // TODO: use Helper class and stream approach
      Map<String,List<PartitionInfo>> topics = new HashMap<>();

      for (Map.Entry<String,List<org.apache.kafka.common.PartitionInfo>> topicEntry: done.result().entrySet()) {

        List<PartitionInfo> partitions = new ArrayList<>();

        for (org.apache.kafka.common.PartitionInfo kafkaPartitionInfo: topicEntry.getValue()) {

          PartitionInfo partitionInfo = new PartitionInfo();

          partitionInfo
            .setInSyncReplicas(
              Stream.of(kafkaPartitionInfo.inSyncReplicas()).map(Helper::from).collect(Collectors.toList()))
            .setLeader(Helper.from(kafkaPartitionInfo.leader()))
            .setPartition(kafkaPartitionInfo.partition())
            .setReplicas(
              Stream.of(kafkaPartitionInfo.replicas()).map(Helper::from).collect(Collectors.toList()))
            .setTopic(kafkaPartitionInfo.topic());

          partitions.add(partitionInfo);

        }

        topics.put(topicEntry.getKey(), partitions);
      }
      handler.handle(Future.succeededFuture(topics));
    } else {
      handler.handle(Future.failedFuture(done.cause()));
    }

  });
  return this;
}
 

@Test
public void testConsumeWithPoll(TestContext ctx) {
  final String topicName = "testConsumeWithPoll";
  final String consumerId = topicName;
  Async batch = ctx.async();
  int numMessages = 1000;
  kafkaCluster.useTo().produceStrings(numMessages, batch::complete, () ->
    new ProducerRecord<>(topicName, "value")
  );
  batch.awaitSuccess(20000);
  Properties config = kafkaCluster.useTo().getConsumerProperties(consumerId, consumerId, OffsetResetStrategy.EARLIEST);
  config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
  config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
  KafkaConsumer<String, String> consumer = KafkaConsumer.create(vertx, config);
  Async done = ctx.async();
  AtomicInteger count = new AtomicInteger(numMessages);

  consumer.subscribe(Collections.singleton(topicName), subscribeResult -> {

    if (subscribeResult.succeeded()) {

      vertx.setPeriodic(1000, t -> {
        consumer.poll(Duration.ofMillis(100), pollResult -> {
          if (pollResult.succeeded()) {
            if (count.updateAndGet(o -> count.get() - pollResult.result().size()) == 0) {
              vertx.cancelTimer(t);
              done.complete();
            }
          } else {
            ctx.fail();
          }
        });
      });

    } else {
      ctx.fail();
    }
  });
}
 

@Test
public void testNotCommitted(TestContext ctx) throws Exception {

  String topicName = "testNotCommitted";
  String consumerId = topicName;
  kafkaCluster.createTopic(topicName, 1, 1);
  Properties config = kafkaCluster.useTo().getConsumerProperties(consumerId, consumerId, OffsetResetStrategy.EARLIEST);
  config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
  config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);

  Async done = ctx.async();

  KafkaConsumer<Object, Object> consumer = KafkaConsumer.create(vertx, config);
  consumer.handler(rec -> {});
  consumer.partitionsAssignedHandler(partitions -> {
    for (io.vertx.kafka.client.common.TopicPartition partition : partitions) {
      consumer.committed(partition, ar -> {
        if (ar.succeeded()) {
          ctx.assertNull(ar.result());
        } else {
          ctx.fail(ar.cause());
        }
      });
    }
    done.complete();
  });

  consumer.subscribe(Collections.singleton(topicName));
}
 

@Override
public KafkaConsumer<K, V> partitionsFor(String topic, Handler<AsyncResult<List<PartitionInfo>>> handler) {

  this.stream.partitionsFor(topic, done -> {

    if (done.succeeded()) {
      // TODO: use Helper class and stream approach
      List<PartitionInfo> partitions = new ArrayList<>();
      for (org.apache.kafka.common.PartitionInfo kafkaPartitionInfo: done.result()) {

        PartitionInfo partitionInfo = new PartitionInfo();

        partitionInfo
          .setInSyncReplicas(
            Stream.of(kafkaPartitionInfo.inSyncReplicas()).map(Helper::from).collect(Collectors.toList()))
          .setLeader(Helper.from(kafkaPartitionInfo.leader()))
          .setPartition(kafkaPartitionInfo.partition())
          .setReplicas(
            Stream.of(kafkaPartitionInfo.replicas()).map(Helper::from).collect(Collectors.toList()))
          .setTopic(kafkaPartitionInfo.topic());

        partitions.add(partitionInfo);
      }
      handler.handle(Future.succeededFuture(partitions));
    } else {
      handler.handle(Future.failedFuture(done.cause()));
    }
  });
  return this;
}
 

@Override
public KafkaConsumer<K, V> batchHandler(Handler<KafkaConsumerRecords<K, V>> handler) {
  stream.batchHandler(records -> {
    handler.handle(new KafkaConsumerRecordsImpl<>(records));
  });
  return this;
}
 

@Test
public void testCleanupInConsumer(TestContext ctx) {
  String topicName = "testCleanupInConsumer";
  Properties config = kafkaCluster.useTo().getConsumerProperties("testCleanupInConsumer_consumer",
    "testCleanupInConsumer_consumer", OffsetResetStrategy.EARLIEST);
  config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
  config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);

  Async async = ctx.async(2);
  Async produceLatch = ctx.async();
  vertx.deployVerticle(new AbstractVerticle() {
    boolean deployed = false;
    @Override
    public void start(Promise<Void> fut) {
      KafkaConsumer<String, String> consumer = KafkaConsumer.create(vertx, config);
      deployed = true;
      consumer.handler(record -> {
        if (deployed) {
          deployed = false;
          vertx.undeploy(context.deploymentID(), ctx.asyncAssertSuccess(v2 -> async.countDown()));
        }
      });
      consumer.assign(new TopicPartition(topicName, 0), fut);
    }
  }, ctx.asyncAssertSuccess(v ->  produceLatch.complete()));
  produceLatch.awaitSuccess(10000);
  kafkaCluster.useTo().produce("testCleanupInConsumer_producer", 100,
    new StringSerializer(), new StringSerializer(), async::countDown,
    () -> new ProducerRecord<>(topicName, "the_value"));

  async.awaitSuccess(10000);
  waitUntil("Expected " + countThreads("vert.x-kafka-consumer-thread") + " == " + numVertxKafkaConsumerThread, () -> countThreads("vert.x-kafka-consumer-thread") == numKafkaConsumerNetworkThread);
}
 

@Test
public void testConsumerBatchHandler(TestContext ctx) throws Exception {
  String topicName = "testConsumerBatchHandler";
  String consumerId = topicName;
  Async batch1 = ctx.async();
  AtomicInteger index = new AtomicInteger();
  int numMessages = 500;
  kafkaCluster.useTo().produceStrings(numMessages, batch1::complete, () ->
    new ProducerRecord<>(topicName, 0, "key-" + index.get(), "value-" + index.getAndIncrement()));
  batch1.awaitSuccess(10000);
  Properties config = kafkaCluster.useTo().getConsumerProperties(consumerId, consumerId, OffsetResetStrategy.EARLIEST);
  config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
  config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);

  KafkaConsumer<Object, Object> wrappedConsumer = KafkaConsumer.create(vertx, config);
  wrappedConsumer.exceptionHandler(ctx::fail);
  AtomicInteger count = new AtomicInteger(numMessages);
  Async batchHandler = ctx.async();
  batchHandler.handler(ar -> wrappedConsumer.close());
  wrappedConsumer.batchHandler(records -> {
    ctx.assertEquals(numMessages, records.size());
    for (int i = 0; i < records.size(); i++) {
      KafkaConsumerRecord<Object, Object> record = records.recordAt(i);
      int dec = count.decrementAndGet();
      if (dec >= 0) {
        ctx.assertEquals("key-" + (numMessages - dec - 1), record.key());
      } else {
        ctx.assertEquals("key-" + (-1 - dec), record.key());
      }
    }
    batchHandler.complete();
  });
  wrappedConsumer.handler(rec -> {});
  wrappedConsumer.subscribe(Collections.singleton(topicName));
}