io.vertx.kafka.client.producer.KafkaWriteStream Java Examples

@Override
public void start() throws Exception {
  systemMBean = ManagementFactory.getPlatformMXBean(OperatingSystemMXBean.class);

  // A random identifier
  String pid = UUID.randomUUID().toString();

  // Get the kafka producer config
  JsonObject config = config();

  // Create the producer
  producer = KafkaWriteStream.create(vertx.getDelegate(), config.getMap(), String.class, JsonObject.class);

  // Publish the metircs in Kafka
  vertx.setPeriodic(1000, id -> {
    JsonObject metrics = new JsonObject();
    metrics.put("CPU", systemMBean.getProcessCpuLoad());
    metrics.put("Mem", systemMBean.getTotalPhysicalMemorySize() - systemMBean.getFreePhysicalMemorySize());
    producer.write(new ProducerRecord<>("the_topic", new JsonObject().put(pid, metrics)));
  });
}
 

public KafkaSink(Vertx vertx, KafkaConnectorOutgoingConfiguration config) {
    JsonObject kafkaConfiguration = extractProducerConfiguration(config);

    stream = KafkaWriteStream.create(vertx.getDelegate(), kafkaConfiguration.getMap());
    stream.exceptionHandler(log::unableToWrite);

    partition = config.getPartition();
    retries = config.getRetries();
    key = config.getKey().orElse(null);
    topic = config.getTopic().orElseGet(config::getChannel);
    boolean waitForWriteCompletion = config.getWaitForWriteCompletion();
    int maxInflight = config.getMaxInflightMessages();
    if (maxInflight == 5) { // 5 is the Kafka default.
        maxInflight = config.config().getOptionalValue(ProducerConfig.MAX_BLOCK_MS_CONFIG, Integer.class)
                .orElse(5);
    }
    int inflight = maxInflight;
    subscriber = ReactiveStreams.<Message<?>> builder()
            .via(new KafkaSenderProcessor(inflight, waitForWriteCompletion, writeMessageToKafka()))
            .onError(log::unableToDispatch)
            .ignore();
}
 

@Test
public void testBufferCodecString(TestContext ctx) throws Exception {
  testCodec(ctx,
    "testBufferCodecString",
    cfg -> {
      cfg.put("key.serializer", BufferSerializer.class);
      cfg.put("value.serializer", BufferSerializer.class);
      return KafkaWriteStream.create(vertx, cfg);
    },
    cfg -> {
      cfg.put("key.deserializer", BufferDeserializer.class);
      cfg.put("value.deserializer", BufferDeserializer.class);
      return KafkaReadStream.create(vertx, cfg);
    },
    i -> Buffer.buffer("key-" + i),
    i -> Buffer.buffer("value-" + i));
}
 

@Test
public void initTransactionsFailsOnWrongConfig(TestContext ctx) {
  final Properties noTransactionalIdConfigured = kafkaCluster.useTo().getProducerProperties("nonTransactionalProducer");
  noTransactionalIdConfigured.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
  noTransactionalIdConfigured.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);

  final KafkaWriteStream<Object, Object> nonTransactionalProducer = producer(Vertx.vertx(), noTransactionalIdConfigured);
  nonTransactionalProducer.exceptionHandler(ctx::fail);
  nonTransactionalProducer.initTransactions(ctx.asyncAssertFailure(cause -> {
    ctx.assertTrue(cause instanceof IllegalStateException);
  }));
}
 

private <K, V> void testCodec(TestContext ctx,
                              String prefix,
                              Function<Properties,KafkaWriteStream<K, V>> producerFactory,
                              Function<Properties, KafkaReadStream<K, V>> consumerFactory,
                              Function<Integer, K> keyConv,
                              Function<Integer, V> valueConv) throws Exception {
  Properties producerConfig = kafkaCluster.useTo().getProducerProperties(prefix+"the_producer");
  KafkaWriteStream<K, V> writeStream = producerFactory.apply(producerConfig);
  producer = writeStream;
  writeStream.exceptionHandler(ctx::fail);
  int numMessages = 100000;
  ConcurrentLinkedDeque<K> keys = new ConcurrentLinkedDeque<K>();
  ConcurrentLinkedDeque<V> values = new ConcurrentLinkedDeque<V>();
  for (int i = 0;i < numMessages;i++) {
    K key = keyConv.apply(i);
    V value = valueConv.apply(i);
    keys.add(key);
    values.add(value);
    writeStream.write(new ProducerRecord<>(prefix + topic, 0, key, value));
  }
  Async done = ctx.async();
  Properties consumerConfig = kafkaCluster.useTo().getConsumerProperties(prefix+"the_consumer", prefix+"the_consumer", OffsetResetStrategy.EARLIEST);
  KafkaReadStream<K, V> readStream = consumerFactory.apply(consumerConfig);
  consumer = readStream;
  AtomicInteger count = new AtomicInteger(numMessages);
  readStream.exceptionHandler(ctx::fail);
  readStream.handler(rec -> {
    ctx.assertEquals(keys.pop(), rec.key());
    ctx.assertEquals(values.pop(), rec.value());
    if (count.decrementAndGet() == 0) {
      done.complete();
    }
  });
  readStream.subscribe(Collections.singleton(prefix + topic));
}
 

public void testProducerConsumer(TestContext ctx) throws Exception {

    int numMsg = 100;
    String topic = "abc-def";

    Properties producerProps = new Properties();
    producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
//    props.put(ProducerConfig.ACKS_CONFIG, "all");
    KafkaWriteStream<String, String> producer = ProducerTest.producer(vertx, producerProps);
    for (int i = 0;i < numMsg;i++) {
      producer.write(new ProducerRecord<>(topic, 0, 0L, "the_key_" + i, "the_value_" + i));
    }
    producer.close();
//    List<String> msg = ku.readMessages("testtopic", 100);
//    assertEquals(100, msg.size());


    Map<String, Object> consumerProps = new HashMap<>();
    consumerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
    consumerProps.put("zookeeper.connect", "localhost:2181");
    consumerProps.put("group.id", "test_group_2");
    consumerProps.put("enable.auto.commit", "false");
    consumerProps.put("auto.offset.reset", "earliest");
    KafkaReadStream<String, String> consumer = KafkaReadStream.create(vertx, consumerProps);
    consumer.subscribe(Collections.singleton(topic));
    AtomicInteger received = new AtomicInteger();

    Async async = ctx.async();
    consumer.handler(rec -> {
      if (received.incrementAndGet() == numMsg) {
        async.complete();
      }
    });
  }
 

@Test
public void testProducerError(TestContext ctx) throws Exception {
  TestProducer mock = new TestProducer();
  KafkaWriteStream<String, String> producer = ProducerTest.producer(Vertx.vertx(), mock);
  producer.write(new ProducerRecord<>("the_topic", 0, 0L, "abc", "def"));
  RuntimeException cause = new RuntimeException();
  Async async = ctx.async();
  producer.exceptionHandler(err -> {
    ctx.assertEquals(cause, err);
    async.complete();
  });
  mock.assertErrorNext(cause);
}
 

private void testProducerDrain(TestContext ctx, RuntimeException failure) throws Exception {
  TestProducer mock = new TestProducer();
  KafkaWriteStream<String, String> producer = ProducerTest.producer(Vertx.vertx(), mock);
  int sent = 0;
  while (!producer.writeQueueFull()) {
    producer.write(new ProducerRecord<>("the_topic", 0, 0L, "abc", "def"));
    sent++;
  }
  Async async = ctx.async();
  producer.drainHandler(v -> {
    ctx.assertTrue(Context.isOnVertxThread());
    ctx.assertTrue(Context.isOnEventLoopThread());
    async.complete();
  });
  for (int i = 0;i < sent / 2;i++) {
    if (failure != null) {
      mock.assertErrorNext(failure);
    } else {
      mock.assertCompleteNext();
    }
  }
  if (failure != null) {
    mock.assertErrorNext(failure);
  } else {
    mock.assertCompleteNext();
  }
  assertFalse(producer.writeQueueFull());
}
 

static void close(TestContext ctx, KafkaWriteStream<?, ?> producer) {
  if (producer != null) {
    close(ctx, handler -> producer.close(2000L, handler));
  }
}
 

@Override
public KafkaWriteStream<K, V> initTransactions(Handler<AsyncResult<Void>> handler) {
  return executeBlocking(handler, this.producer::initTransactions);
}
 

@Override
public KafkaWriteStream<K, V> beginTransaction(Handler<AsyncResult<Void>> handler) {
  return executeBlocking(handler, this.producer::beginTransaction);
}
 

@Override
public KafkaWriteStream<K, V> commitTransaction(Handler<AsyncResult<Void>> handler) {
  return executeBlocking(handler, this.producer::commitTransaction);
}
 

@Override
public KafkaWriteStream<K, V> abortTransaction(Handler<AsyncResult<Void>> handler) {
  return executeBlocking(handler, this.producer::abortTransaction);
}
 

public static <K, V> KafkaProducer<K, V> createShared(Vertx vertx, String name, Properties config) {
  return createShared(vertx, name, () -> KafkaWriteStream.create(vertx, config));
}
 

public static <K, V> KafkaProducer<K, V> createShared(Vertx vertx, String name, Map<String, String> config) {
  return createShared(vertx, name, () -> KafkaWriteStream.create(vertx, new HashMap<>(config)));
}
 

static <K, V> KafkaWriteStream<K, V> producer(Vertx vertx, Properties config, Class<K> keyType, Class<V> valueType) {
  return KafkaWriteStream.create(vertx, config, keyType, valueType);
}
 

static <K, V> KafkaWriteStream<K, V> producer(Vertx vertx, Properties config) {
  return KafkaWriteStream.create(vertx, config);
}
 

public static <K, V> KafkaProducer<K, V> createShared(Vertx vertx, String name, Properties config, Class<K> keyType, Class<V> valueType) {
  return createShared(vertx, name, () -> KafkaWriteStream.create(vertx, config, keyType, valueType));
}
 

@Override
public KafkaWriteStream<K, V> asStream() {
  return this.stream;
}
 

public KafkaProducerImpl(Vertx vertx, KafkaWriteStream<K, V> stream) {
  this(vertx, stream, new CloseHandler(stream::close));
}
 

public KafkaProducerImpl(Vertx vertx, KafkaWriteStream<K, V> stream, CloseHandler closeHandler) {
  this.vertx = vertx;
  this.stream = stream;
  this.closeHandler = closeHandler;
}
 

public SharedProducer(KafkaWriteStream stream) {
  this.producer = stream.unwrap();
  this.closeHandler = new CloseHandler(stream::close);
}
 

public static <K, V> KafkaProducer<K, V> createShared(Vertx vertx, String name, Map<String, String> config, Serializer<K> keySerializer, Serializer<V> valueSerializer) {
  return createShared(vertx, name, () -> KafkaWriteStream.create(vertx, new HashMap<>(config), keySerializer, valueSerializer));
}
 

public static <K, V> KafkaProducer<K, V> createShared(Vertx vertx, String name, Map<String, String> config, Class<K> keyType, Class<V> valueType) {
  return createShared(vertx, name, () -> KafkaWriteStream.create(vertx, new HashMap<>(config), keyType, valueType));
}
 

public static <K, V> KafkaProducer<K, V> createShared(Vertx vertx, String name, Properties config, Serializer<K> keySerializer, Serializer<V> valueSerializer) {
  return createShared(vertx, name, () -> KafkaWriteStream.create(vertx, config, keySerializer, valueSerializer));
}