io.vertx.kafka.client.common.TopicPartition Java Examples

public Future<Map<TopicPartition, OffsetAndMetadata>> listConsumerGroupOffsets(String groupId, ListConsumerGroupOffsetsOptions options) {
  ContextInternal ctx = (ContextInternal) vertx.getOrCreateContext();
  Promise<Map<TopicPartition, OffsetAndMetadata>> promise = ctx.promise();

  ListConsumerGroupOffsetsResult listConsumerGroupOffsetsResult = this.adminClient.listConsumerGroupOffsets(groupId, Helper.to(options));
  listConsumerGroupOffsetsResult.partitionsToOffsetAndMetadata().whenComplete((cgo, ex) -> {

    if (ex == null) {
      Map<TopicPartition, OffsetAndMetadata> consumerGroupOffsets = new HashMap<>();

      for (Map.Entry<org.apache.kafka.common.TopicPartition, org.apache.kafka.clients.consumer.OffsetAndMetadata> cgoOffset : cgo.entrySet()) {
        consumerGroupOffsets.put(Helper.from(cgoOffset.getKey()), Helper.from(cgoOffset.getValue()));
      }
      promise.complete(consumerGroupOffsets);
    } else {
      promise.fail(ex);
    }
  });
  return promise.future();
}
 

@Test
public void shouldAssignMultiplePartitions() {
    Set<TopicPartition> vertxPartitions = toSet(
        new TopicPartition("test-topic-1", 0),
        new TopicPartition("test-topic-2", 1)
    );
    given(mockAxleConsumer.assign(vertxPartitions))
        .willReturn(completedFuture(null));

    Collection<Partition> partitions = asList(
        Partition.create("test-topic-1", 0),
        Partition.create("test-topic-2", 1)
    );
    StepVerifier.create(consumer.assign(partitions))
        .verifyComplete();
}
 

@Test
public void testEquality(final TestContext context) {
  final TopicPartition t1 = new TopicPartition("topic1", 0);
  final TopicPartition t2 = new TopicPartition("topic1", 0);
  final TopicPartition t3 = new TopicPartition(null, 0);
  final TopicPartition t4 = new TopicPartition(null, 0);

  context.assertEquals(t1, t1);
  context.assertEquals(t1.hashCode(), t1.hashCode());

  context.assertEquals(t1, t2);
  context.assertEquals(t1.hashCode(), t2.hashCode());

  context.assertEquals(t3, t4);
  context.assertEquals(t3.hashCode(), t4.hashCode());
}
 

@Override
public Uni<Void> onPartitionsAssigned(KafkaConsumer<?, ?> consumer, Set<TopicPartition> set) {
    // will perform the underlying operation but simulate an error on the first attempt
    return super.onPartitionsAssigned(consumer, set)
            .onItem()
            .produceUni(a -> {
                if (!set.isEmpty() && failOnFirstAttempt.getAndSet(false)) {
                    return Uni
                            .createFrom()
                            .failure(new Exception("testing failure"));
                } else {
                    return Uni
                            .createFrom()
                            .item(a);
                }
            });
}
 

@Test
public void testCleanupInConsumer(TestContext ctx) {
  String topicName = "testCleanupInConsumer";
  Properties config = kafkaCluster.useTo().getConsumerProperties("testCleanupInConsumer_consumer",
    "testCleanupInConsumer_consumer", OffsetResetStrategy.EARLIEST);
  config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
  config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);

  Async async = ctx.async(2);
  Async produceLatch = ctx.async();
  vertx.deployVerticle(new AbstractVerticle() {
    boolean deployed = false;
    @Override
    public void start(Promise<Void> fut) {
      KafkaConsumer<String, String> consumer = KafkaConsumer.create(vertx, config);
      deployed = true;
      consumer.handler(record -> {
        if (deployed) {
          deployed = false;
          vertx.undeploy(context.deploymentID(), ctx.asyncAssertSuccess(v2 -> async.countDown()));
        }
      });
      consumer.assign(new TopicPartition(topicName, 0), fut);
    }
  }, ctx.asyncAssertSuccess(v ->  produceLatch.complete()));
  produceLatch.awaitSuccess(10000);
  kafkaCluster.useTo().produce("testCleanupInConsumer_producer", 100,
    new StringSerializer(), new StringSerializer(), async::countDown,
    () -> new ProducerRecord<>(topicName, "the_value"));

  async.awaitSuccess(10000);
  waitUntil("Expected " + countThreads("vert.x-kafka-consumer-thread") + " == " + numVertxKafkaConsumerThread, () -> countThreads("vert.x-kafka-consumer-thread") == numKafkaConsumerNetworkThread);
}
 

@Test
public void shouldSeekToEndOfSinglePartition() {
    given(mockAxleConsumer.seekToEnd(new TopicPartition("test-topic", 1)))
        .willReturn(completedFuture(null));

    StepVerifier.create(consumer.seekToEnd(Partition.create("test-topic", 1)))
        .verifyComplete();
}
 

@Test
public void shouldGetCommitted() {
    given(mockAxleConsumer.committed(new TopicPartition("test-topic", 1)))
        .willReturn(completedFuture(new OffsetAndMetadata(2, "test-metadata")));

    StepVerifier.create(consumer.committed(Partition.create("test-topic", 1)))
        .expectNext(2L)
        .verifyComplete();
}
 

protected void seekToBeginning(Set<TopicPartition> topicPartitionSet, Handler<AsyncResult<Void>> seekHandler) {
    this.consumer.seekToBeginning(topicPartitionSet, result -> {
        if (seekHandler != null) {
            seekHandler.handle(result);
        }
    });
}
 

@Override
public void committed(TopicPartition topicPartition, Handler<AsyncResult<OffsetAndMetadata>> handler) {
  this.stream.committed(Helper.to(topicPartition), done -> {

    if (done.succeeded()) {
      handler.handle(Future.succeededFuture(Helper.from(done.result())));
    } else {
      handler.handle(Future.failedFuture(done.cause()));
    }
  });
}
 

@Override
public void endOffsets(TopicPartition topicPartition, Handler<AsyncResult<Long>> handler) {
  Set<TopicPartition> topicPartitions = new HashSet<>();
  topicPartitions.add(topicPartition);
  this.stream.endOffsets(Helper.to(topicPartitions), done -> {
    if(done.succeeded()) {
      for(long endOffset : done.result().values()) {
        handler.handle(Future.succeededFuture(endOffset));
        break;
      }
    } else {
      handler.handle(Future.failedFuture(done.cause()));
    }
  });
}
 

@Test
public void shouldGetPosition() {
    given(mockAxleConsumer.position(new TopicPartition("test-topic", 1)))
        .willReturn(completedFuture(1L));

    StepVerifier.create(consumer.position(Partition.create("test-topic", 1)))
        .expectNext(1L)
        .verifyComplete();
}
 

@Test
public void shouldGetBeginningOffset() {
    given(mockAxleConsumer.beginningOffsets(new TopicPartition("test-topic", 1)))
        .willReturn(completedFuture(2L));

    StepVerifier.create(consumer.beginningOffset(Partition.create("test-topic", 1)))
        .expectNext(2L)
        .verifyComplete();
}
 

@Test
public void shouldGetEndOffset() {
    given(mockAxleConsumer.endOffsets(new TopicPartition("test-topic", 1)))
        .willReturn(completedFuture(2L));

    StepVerifier.create(consumer.endOffset(Partition.create("test-topic", 1)))
        .expectNext(2L)
        .verifyComplete();
}
 

@Test
public void shouldGetTimeOffset() {
    given(mockAxleConsumer.offsetsForTimes(new TopicPartition("test-topic", 1), 2L))
        .willReturn(completedFuture(new OffsetAndTimestamp(2L, 3L)));

    StepVerifier.create(consumer.timeOffset(Partition.create("test-topic", 1), 2L))
        .expectNext(2L)
        .verifyComplete();
}
 

protected void seek(TopicPartition topicPartition, long offset, Handler<AsyncResult<Void>> seekHandler) {
    this.consumer.seek(topicPartition, offset, result -> {
        if (seekHandler != null) {
            seekHandler.handle(result);
        }
    });
}
 

@Override
public void paused(Handler<AsyncResult<Set<TopicPartition>>> handler) {

  this.stream.paused(done -> {

    if (done.succeeded()) {
      handler.handle(Future.succeededFuture(Helper.from(done.result())));
    } else {
      handler.handle(Future.failedFuture(done.cause()));
    }
  });
}
 

@Override
public Uni<Void> onPartitionsAssigned(KafkaConsumer<?, ?> consumer, Set<TopicPartition> set) {
    rebalanceCount++;
    return Uni
            .combine()
            .all()
            .unis(set
                    .stream()
                    .map(topicPartition -> consumer.endOffsets(topicPartition)
                            .onItem()
                            .produceUni(o -> consumer.seek(topicPartition, Math.max(0L, o - 5L))))
                    .collect(Collectors.toList()))
            .combinedWith(a -> null);
}
 

@Test
public void testABeanConsumingTheKafkaMessagesWithPartitions() {
    kafka.createTopic("data-2", 2, 1);
    ConsumptionBean bean = deploy(myKafkaSourceConfig(2, ConsumptionConsumerRebalanceListener.class.getSimpleName(), null));
    KafkaUsage usage = new KafkaUsage();
    List<Integer> list = bean.getResults();
    assertThat(list).isEmpty();
    AtomicInteger counter = new AtomicInteger();
    new Thread(() -> usage.produceIntegers(10, null,
            () -> new ProducerRecord<>("data-2", counter.getAndIncrement()))).start();

    await().atMost(2, TimeUnit.MINUTES).until(() -> list.size() >= 10);
    assertThat(list).containsExactlyInAnyOrder(1, 2, 3, 4, 5, 6, 7, 8, 9, 10);

    List<KafkaRecord<String, Integer>> messages = bean.getKafkaMessages();
    messages.forEach(m -> {
        assertThat(m.getTopic()).isEqualTo("data-2");
        assertThat(m.getTimestamp()).isAfter(Instant.EPOCH);
        assertThat(m.getPartition()).isGreaterThan(-1);
    });

    ConsumptionConsumerRebalanceListener consumptionConsumerRebalanceListener = getConsumptionConsumerRebalanceListener();
    assertThat(consumptionConsumerRebalanceListener.getAssigned().size()).isEqualTo(2);
    for (int i = 0; i < 2; i++) {
        TopicPartition topicPartition = consumptionConsumerRebalanceListener.getAssigned().get(i);
        assertThat(topicPartition).isNotNull();
        assertThat(topicPartition.getTopic()).isEqualTo("data-2");
    }
}
 

public static Handler<Set<org.apache.kafka.common.TopicPartition>> adaptHandler(Handler<Set<TopicPartition>> handler) {
  if (handler != null) {
    return topicPartitions -> handler.handle(Helper.from(topicPartitions));
  } else {
    return null;
  }
}
 

/**
 * Example to demonstrate how one can use the new beginningOffsets API (introduced with Kafka
 * 0.10.1.1) to look up the first offset for a given partition.
 * @param consumer Consumer to be used
 */
public void exampleConsumerBeginningOffsets(KafkaConsumer<String, String> consumer) {
  Set<TopicPartition> topicPartitions = new HashSet<>();
  TopicPartition topicPartition = new TopicPartition().setTopic("test").setPartition(0);
  topicPartitions.add(topicPartition);

  consumer
    .beginningOffsets(topicPartitions)
    .onSuccess(results ->
      results.forEach((topic, beginningOffset) ->
        System.out.println(
          "Beginning offset for topic=" + topic.getTopic() + ", partition=" +
            topic.getPartition() + ", beginningOffset=" + beginningOffset
        )
      )
    );

  // Convenience method for single-partition lookup
  consumer
    .beginningOffsets(topicPartition)
    .onSuccess(beginningOffset ->
      System.out.println(
        "Beginning offset for topic=" + topicPartition.getTopic() + ", partition=" +
          topicPartition.getPartition() + ", beginningOffset=" + beginningOffset
      )
    );
}
 

private void partitionsAssignedHandler(Set<TopicPartition> partitions) {

        if (partitions.isEmpty()) {

            sendAmqpError(AmqpBridge.newError(AmqpBridge.AMQP_ERROR_NO_PARTITIONS,
                    "All partitions already have a receiver"));
        } else {

            if (!this.sender.isOpen()) {
                this.sender
                        .setSource(this.sender.getRemoteSource())
                        .open();
            }
        }
    }
 

public void doListSubscriptions(RoutingContext routingContext) {
    this.listSubscriptions(listSubscriptionsResult -> {

        if (listSubscriptionsResult.succeeded()) {
            JsonObject root = new JsonObject();
            JsonArray topicsArray = new JsonArray();
            JsonArray partitionsArray = new JsonArray();

            HashMap<String, JsonArray> partitions = new HashMap<>();
            for (TopicPartition topicPartition: listSubscriptionsResult.result()) {
                if (!topicsArray.contains(topicPartition.getTopic())) {
                    topicsArray.add(topicPartition.getTopic());
                }
                if (!partitions.containsKey(topicPartition.getTopic())) {
                    partitions.put(topicPartition.getTopic(), new JsonArray());
                }
                partitions.put(topicPartition.getTopic(), partitions.get(topicPartition.getTopic()).add(topicPartition.getPartition()));
            }
            for (Map.Entry<String, JsonArray> part: partitions.entrySet()) {
                JsonObject topic = new JsonObject();
                topic.put(part.getKey(), part.getValue());
                partitionsArray.add(topic);
            }
            root.put("topics", topicsArray);
            root.put("partitions", partitionsArray);

            HttpUtils.sendResponse(routingContext, HttpResponseStatus.OK.code(), BridgeContentType.KAFKA_JSON, root.toBuffer());
        } else {
            HttpBridgeError error = new HttpBridgeError(
                    HttpResponseStatus.INTERNAL_SERVER_ERROR.code(),
                    listSubscriptionsResult.cause().getMessage()
            );
            HttpUtils.sendResponse(routingContext, HttpResponseStatus.INTERNAL_SERVER_ERROR.code(),
                    BridgeContentType.KAFKA_JSON, error.toJson().toBuffer());
        }
    });
}
 

public static Map<TopicPartition, OffsetAndTimestamp> fromTopicPartitionOffsetAndTimestamp(Map<org.apache.kafka.common.TopicPartition, org.apache.kafka.clients.consumer.OffsetAndTimestamp> topicPartitionOffsetAndTimestamps) {
  return topicPartitionOffsetAndTimestamps.entrySet().stream()
    .filter(e-> e.getValue() != null)
    .collect(Collectors.toMap(
      e -> new TopicPartition(e.getKey().topic(), e.getKey().partition()),
      e ->new OffsetAndTimestamp(e.getValue().offset(), e.getValue().timestamp()))
    );
}
 

@Override
public KafkaConsumer<K, V> seek(TopicPartition topicPartition, long offset, Handler<AsyncResult<Void>> completionHandler) {
  this.stream.seek(Helper.to(topicPartition), offset, completionHandler);
  return this;
}
 

@Override
public Future<Void> pause(TopicPartition topicPartition) {
  return this.pause(Collections.singleton(topicPartition));
}
 

@Override
public Future<Map<TopicPartition, Long>> endOffsets(Set<TopicPartition> topicPartitions) {
  Promise<Map<TopicPartition, Long>> promise = Promise.promise();
  endOffsets(topicPartitions, promise);
  return promise.future();
}
 

@Override
public Future<Void> seek(TopicPartition topicPartition, long offset) {
  Promise<Void> promise = Promise.promise();
  this.seek(topicPartition, offset, promise);
  return promise.future();
}
 

@Override
public KafkaConsumer<K, V> seekToBeginning(TopicPartition topicPartition, Handler<AsyncResult<Void>> completionHandler) {
  return this.seekToBeginning(Collections.singleton(topicPartition), completionHandler);
}
 

@Override
public KafkaConsumer<K, V> seekToBeginning(Set<TopicPartition> topicPartitions, Handler<AsyncResult<Void>> completionHandler) {
  this.stream.seekToBeginning(Helper.to(topicPartitions), completionHandler);
  return this;
}
 

@Override
public Future<OffsetAndMetadata> committed(TopicPartition topicPartition) {
  Promise<OffsetAndMetadata> promise = Promise.promise();
  committed(topicPartition, promise);
  return promise.future();
}