org.apache.pulsar.client.api.Message Java Examples

private void testReadMessages(String topic, boolean enableBatch) throws Exception {
    int numKeys = 10;

    Set<String> keys = publishMessages(topic, numKeys, enableBatch);
    Reader<byte[]> reader = pulsarClient.newReader()
            .topic(topic)
            .startMessageId(MessageId.earliest)
            .readerName(subscription)
            .create();

    while (reader.hasMessageAvailable()) {
        Message<byte[]> message = reader.readNext();
        Assert.assertTrue(keys.remove(message.getKey()));
    }
    Assert.assertTrue(keys.isEmpty());

    Reader<byte[]> readLatest = pulsarClient.newReader().topic(topic).startMessageId(MessageId.latest)
                                            .readerName(subscription + "latest").create();
    Assert.assertFalse(readLatest.hasMessageAvailable());
}
 

@Test(timeOut = 20000, dataProvider = "lastDeletedBatching")
public void testEmptyCompactionLedger(boolean batching) throws Exception {
    String topic = "persistent://my-property/use/my-ns/my-topic1";

    Producer<byte[]> producer = pulsarClient.newProducer().topic(topic).enableBatching(batching)
            .messageRoutingMode(MessageRoutingMode.SinglePartition).create();

    pulsarClient.newConsumer().topic(topic).subscriptionName("sub1").readCompacted(true).subscribe().close();

    producer.newMessage().key("1").value("1".getBytes()).send();
    producer.newMessage().key("2").value("2".getBytes()).send();
    producer.newMessage().key("1").value("".getBytes()).send();
    producer.newMessage().key("2").value("".getBytes()).send();

    Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
    compactor.compact(topic).get();

    // consumer with readCompacted enabled only get compacted entries
    try (Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("sub1")
            .readCompacted(true).subscribe()) {
        Message<byte[]> m = consumer.receive(2, TimeUnit.SECONDS);
        assertNull(m);
    }
}
 

@Override
protected Message<T> internalReceive(int timeout, TimeUnit unit) throws PulsarClientException {
    Message<T> message;
    try {
        message = incomingMessages.poll(timeout, unit);
        if (message == null) {
            return null;
        }
        messageProcessed(message);
        return beforeConsume(message);
    } catch (InterruptedException e) {
        State state = getState();
        if (state != State.Closing && state != State.Closed) {
            stats.incrementNumReceiveFailed();
            throw PulsarClientException.unwrap(e);
        } else {
            return null;
        }
    }
}
 

@Override
public int convert(BatchMaker batchMaker, Source.Context context, String messageId, Message message)
    throws StageException {
  byte[] payload = message.getData();
  int count = 0;
  if (payload.length > 0) {
    try {
      for (Record record : ServicesUtil.parseAll(context, context, messageConfig.produceSingleRecordPerMessage,
          messageId, payload)) {
        Map<String, String> messageProperties = message.getProperties();
        messageProperties.forEach((key, value) -> record.getHeader().setAttribute(key, value == null ? "" : value));
        batchMaker.addRecord(record);
        ++count;
      }
    } catch (StageException e) {
      handleException(context, messageId, e);
    }
  }
  return count;
}
 

protected void testPulsarFunctionality(String pulsarBrokerUrl) throws Exception {

        try (
            PulsarClient client = PulsarClient.builder()
                .serviceUrl(pulsarBrokerUrl)
                .build();
            Consumer consumer = client.newConsumer()
                .topic(TEST_TOPIC)
                .subscriptionName("test-subs")
                .subscribe();
            Producer<byte[]> producer = client.newProducer()
                .topic(TEST_TOPIC)
                .create()
        ) {

            producer.send("test containers".getBytes());
            CompletableFuture<Message> future = consumer.receiveAsync();
            Message message = future.get(5, TimeUnit.SECONDS);

            assertThat(new String(message.getData()))
                .isEqualTo("test containers");
        }
    }
 

private void failPendingReceive() {
    lock.readLock().lock();
    try {
        if (listenerExecutor != null && !listenerExecutor.isShutdown()) {
            while (!pendingReceives.isEmpty()) {
                CompletableFuture<Message<T>> receiveFuture = pendingReceives.poll();
                if (receiveFuture != null) {
                    receiveFuture.completeExceptionally(
                            new PulsarClientException.AlreadyClosedException("Consumer is already closed"));
                } else {
                    break;
                }
            }
        }
    } finally {
        lock.readLock().unlock();
    }
}
 

@Override
CompletableFuture<MessageId> internalSendAsync(Message<?> message) {

    switch (getState()) {
    case Ready:
    case Connecting:
        break; // Ok
    case Closing:
    case Closed:
        return FutureUtil.failedFuture(new PulsarClientException.AlreadyClosedException("Producer already closed"));
    case Terminated:
        return FutureUtil.failedFuture(new PulsarClientException.TopicTerminatedException("Topic was terminated"));
    case Failed:
    case Uninitialized:
        return FutureUtil.failedFuture(new PulsarClientException.NotConnectedException());
    }

    int partition = routerPolicy.choosePartition(message, topicMetadata);
    checkArgument(partition >= 0 && partition < topicMetadata.numPartitions(),
            "Illegal partition index chosen by the message routing policy: " + partition);
    return producers.get(partition).internalSendAsync(message);
}
 

@Override
public void reconsumeLater(Message<?> message, long delayTime, TimeUnit unit) throws PulsarClientException {
    if (!conf.isRetryEnable()) {
        throw new PulsarClientException("reconsumeLater method not support!");
    }
    try {
        reconsumeLaterAsync(message, delayTime, unit).get();
    } catch (Exception e) {
        Throwable t = e.getCause();
        if (t instanceof PulsarClientException) {
            throw (PulsarClientException) t;
        } else {
            throw new PulsarClientException(t);
        }
    }
}
 

private ConsumerContext(
        @Nonnull ILogger logger,
        @Nonnull PulsarClient client,
        @Nonnull List<String> topics,
        @Nonnull Map<String, Object> consumerConfig,
        @Nonnull SupplierEx<Schema<M>> schemaSupplier,
        @Nonnull SupplierEx<BatchReceivePolicy> batchReceivePolicySupplier,
        @Nonnull FunctionEx<Message<M>, T> projectionFn
) throws PulsarClientException {

    this.logger = logger;
    this.projectionFn = projectionFn;
    this.client = client;
    this.consumer = client.newConsumer(schemaSupplier.get())
                          .topics(topics)
                          .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
                          .loadConf(consumerConfig)
                          .batchReceivePolicy(batchReceivePolicySupplier.get())
                          .subscriptionType(SubscriptionType.Shared)
                          .subscribe();
}
 

private void triggerZeroQueueSizeListener(final Message<T> message) {
    checkNotNull(listener, "listener can't be null");
    checkNotNull(message, "unqueued message can't be null");

    listenerExecutor.execute(() -> {
        stats.updateNumMsgsReceived(message);
        try {
            if (log.isDebugEnabled()) {
                log.debug("[{}][{}] Calling message listener for unqueued message {}", topic, subscription,
                        message.getMessageId());
            }
            waitingOnListenerForZeroQueueSize = true;
            trackMessage(message);
            listener.received(ZeroQueueConsumerImpl.this, beforeConsume(message));
        } catch (Throwable t) {
            log.error("[{}][{}] Message listener error in processing unqueued message: {}", topic, subscription,
                    message.getMessageId(), t);
        }
        increaseAvailablePermits(cnx());
        waitingOnListenerForZeroQueueSize = false;
    });
}
 

@Test(invocationTimeOut = 1000)
public void testNotifyPendingReceivedCallback_WorkNormally() {
    CompletableFuture<Message<ConsumerImpl>> receiveFuture = new CompletableFuture<>();
    MessageImpl message = mock(MessageImpl.class);
    ConsumerImpl<ConsumerImpl> spy = spy(consumer);

    consumer.pendingReceives.add(receiveFuture);
    doReturn(message).when(spy).beforeConsume(any());
    doNothing().when(spy).messageProcessed(message);
    spy.notifyPendingReceivedCallback(message, null);
    Message<ConsumerImpl> receivedMessage = receiveFuture.join();

    verify(spy, times(1)).beforeConsume(message);
    verify(spy, times(1)).messageProcessed(message);
    Assert.assertTrue(receiveFuture.isDone());
    Assert.assertFalse(receiveFuture.isCompletedExceptionally());
    Assert.assertEquals(receivedMessage, message);
}
 

@Test(timeOut = 20000, dataProvider = "lastDeletedBatching")
public void testCompactionWithLastDeletedKey(boolean batching) throws Exception {
    String topic = "persistent://my-property/use/my-ns/my-topic1";

    Producer<byte[]> producer = pulsarClient.newProducer().topic(topic).enableBatching(batching)
            .messageRoutingMode(MessageRoutingMode.SinglePartition).create();

    pulsarClient.newConsumer().topic(topic).subscriptionName("sub1").readCompacted(true).subscribe().close();

    producer.newMessage().key("1").value("1".getBytes()).send();
    producer.newMessage().key("2").value("2".getBytes()).send();
    producer.newMessage().key("3").value("3".getBytes()).send();
    producer.newMessage().key("1").value("".getBytes()).send();
    producer.newMessage().key("2").value("".getBytes()).send();

    Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
    compactor.compact(topic).get();

    Set<String> expected = Sets.newHashSet("3");
    // consumer with readCompacted enabled only get compacted entries
    try (Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("sub1")
            .readCompacted(true).subscribe()) {
        Message<byte[]> m = consumer.receive(2, TimeUnit.SECONDS);
        assertTrue(expected.remove(m.getKey()));
    }
}
 

/**
 * Receive the messages as a batch.
 * In this method, emitted items are created by applying the projection function
 * to the messages received from Pulsar client. If there is an event time
 * associated with the message, it sets the event time as the timestamp of the
 * emitted item. Otherwise, it sets the publish time(which always exists)
 * of the message as the timestamp.
 */
private void fillBuffer(SourceBuilder.TimestampedSourceBuffer<T> sourceBuffer) throws PulsarClientException {
    if (reader == null) {
        createReader();
    }
    int count = 0;
    while (!queue.isEmpty() && count++ < MAX_FILL_MESSAGES) {
        Message<M> message = queue.poll();
        long timestamp;
        if (message.getEventTime() != 0) {
            timestamp = message.getEventTime();
        } else {
            timestamp = message.getPublishTime();
        }
        T item = projectionFn.apply(message);
        offset = message.getMessageId();
        if (item != null) {
            sourceBuffer.add(item, timestamp);
        }
    }
}
 

@Test
public void testBatchingAwareness() throws Exception {
    Message<?> msg = mock(Message.class);
    when(msg.getKey()).thenReturn(null);

    Clock clock = mock(Clock.class);

    RoundRobinPartitionMessageRouterImpl router = new RoundRobinPartitionMessageRouterImpl(
            HashingScheme.JavaStringHash, 0, true, 10, clock);
    TopicMetadataImpl metadata = new TopicMetadataImpl(100);

    // time at `12345*` milliseconds
    for (int i = 0; i < 10; i++) {
        when(clock.millis()).thenReturn(123450L + i);

        assertEquals(45, router.choosePartition(msg, metadata));
    }

    // time at `12346*` milliseconds
    for (int i = 0; i < 10; i++) {
        when(clock.millis()).thenReturn(123460L + i);

        assertEquals(46, router.choosePartition(msg, metadata));
    }
}
 

public static void main(String[] args) throws PulsarClientException, JsonProcessingException {

        PulsarClient pulsarClient = PulsarClient.builder().serviceUrl("http://localhost:8080").build();

        Consumer<JsonPojo> consumer = pulsarClient.newConsumer(JSONSchema.of
                (SchemaDefinition.<JsonPojo>builder().withPojo(JsonPojo.class).build())) //
                .topic("persistent://my-property/use/my-ns/my-topic") //
                .subscriptionName("my-subscription-name").subscribe();

        Message<JsonPojo> msg = null;

        for (int i = 0; i < 100; i++) {
            msg = consumer.receive();
            // do something
            System.out.println("Received: " + msg.getValue().content);
        }

        // Acknowledge the consumption of all messages at once
        consumer.acknowledgeCumulative(msg);
        pulsarClient.close();
    }
 

private org.apache.pulsar.client.api.Producer<byte[]> createNewProducer(String topic) {
    try {
        pulsarProducerBuilder.messageRoutingMode(MessageRoutingMode.CustomPartition);
        pulsarProducerBuilder.messageRouter(new MessageRouter() {
            private static final long serialVersionUID = 1L;

            @Override
            public int choosePartition(Message<?> msg, TopicMetadata metadata) {
                // https://kafka.apache.org/08/documentation.html#producerapi
                // The default partitioner is based on the hash of the key.
                return partitioner.partition(msg.getKey(), metadata.numPartitions());
            }
        });
        log.info("Creating producer for topic {} with config {}", topic, pulsarProducerBuilder.toString());
        return pulsarProducerBuilder.clone().topic(topic).create();
    } catch (PulsarClientException e) {
        throw new RuntimeException(e);
    }
}
 

private static Map<Integer, String> consumeCompactedTopic(PulsarClient client,
                                                          String topic,
                                                          String subscription,
                                                          int numKeys) throws PulsarClientException {
    Map<Integer, String> keys = Maps.newHashMap();
    try (Consumer<byte[]> consumer = client.newConsumer()
         .readCompacted(true)
         .topic(topic)
         .subscriptionName(subscription)
         .subscribe()
    ) {
        for (int i = 0; i < numKeys; i++) {
            Message<byte[]> m = consumer.receive();
            keys.put(Integer.parseInt(m.getKey()), new String(m.getValue(), UTF_8));
        }
    }
    return keys;
}
 

/**
 * Receive the messages as a batch. The {@link BatchReceivePolicy} is
 * configured while creating the Pulsar {@link Consumer}.
 * In this method, emitted items are created by applying the projection function
 * to the messages received from Pulsar client. If there is an event time
 * associated with the message, it sets the event time as the timestamp of the
 * emitted item. Otherwise, it sets the publish time(which always exists)
 * of the message as the timestamp.
 */
private void fillBuffer(SourceBuilder.TimestampedSourceBuffer<T> sourceBuffer) throws PulsarClientException {
    Messages<M> messages = consumer.batchReceive();
    for (Message<M> message : messages) {
        if (message.getEventTime() != 0) {
            sourceBuffer.add(projectionFn.apply(message), message.getEventTime());
        } else {
            sourceBuffer.add(projectionFn.apply(message), message.getPublishTime());
        }
    }
    consumer.acknowledgeAsync(messages)
            .exceptionally(t -> {
                logger.warning(buildLogMessage(messages));
                return null;
            });
}
 

@Test
public void testPartitions() throws Exception {
    @Cleanup
    PulsarClient client = internalSetUpForClient(true, proxyService.getServiceUrlTls());
    String topicName = "persistent://sample/test/local/partitioned-topic" + System.currentTimeMillis();
    TenantInfo tenantInfo = createDefaultTenantInfo();
    admin.tenants().createTenant("sample", tenantInfo);
    admin.topics().createPartitionedTopic(topicName, 2);

    @Cleanup
    Producer<byte[]> producer = client.newProducer(Schema.BYTES).topic(topicName)
            .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();

    // Create a consumer directly attached to broker
    @Cleanup
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName)
            .subscriptionName("my-sub").subscribe();

    for (int i = 0; i < 10; i++) {
        producer.send("test".getBytes());
    }

    for (int i = 0; i < 10; i++) {
        Message<byte[]> msg = consumer.receive(1, TimeUnit.SECONDS);
        checkNotNull(msg);
    }
}
 

@Test(dataProvider = "batch")
public void testSyncProducerAndConsumer(int batchMessageDelayMs) throws Exception {
    log.info("-- Starting {} test --", methodName);

    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING)
                .topic("persistent://my-property/use/my-ns/my-topic1")
                .subscriptionName("my-subscriber-name")
                .subscriptionType(SubscriptionType.Exclusive)
                .subscribe();

    ProducerBuilder<String> producerBuilder = pulsarClient.newProducer(Schema.STRING)
            .topic("persistent://my-property/use/my-ns/my-topic1");

    if (batchMessageDelayMs != 0) {
        producerBuilder.enableBatching(true)
            .batchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS)
            .batchingMaxMessages(5);
    } else {
        producerBuilder.enableBatching(false);
    }

    Producer<String> producer = producerBuilder.create();
    for (int i = 0; i < 10; i++) {
        producer.send("my-message-" + i);
    }

    Message<String> msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = msg.getValue();
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 

@Test
public void testAppendWithKey() {
    final Appender appender = ctx.getConfiguration().getAppender("PulsarAppenderWithKey");
    final LogEvent logEvent = createLogEvent();
    appender.append(logEvent);
    Message<byte[]> item;
    synchronized (history) {
        assertEquals(1, history.size());
        item = history.get(0);
    }
    assertNotNull(item);
    String msgKey = item.getKey();
    assertEquals(msgKey, "key");
    assertEquals(LOG_MESSAGE, new String(item.getData(), StandardCharsets.UTF_8));
}
 

protected Message<T> beforeConsume(Message<T> message) {
    if (interceptors != null) {
        return interceptors.beforeConsume(this, message);
    } else {
        return message;
    }
}
 

@Test
public void testProducerAndConsumerPassed() throws Exception {
    log.info("-- {} -- start", methodName);

    Consumer<byte[]> consumer = pulsarClient.newConsumer()
        .topic("persistent://my-property/my-ns/my-topic")
        .subscriptionName("my-subscriber-name")
        .subscribe();

    ProducerBuilder<byte[]> producerBuilder = pulsarClient.newProducer()
        .topic("persistent://my-property/my-ns/my-topic")
        .enableBatching(false);

    Producer<byte[]> producer = producerBuilder.create();
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        log.info("Produced message: [{}]", message);
    }

    Message<byte[]> msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.info("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();

    log.info("-- {} -- end", methodName);
}
 

public void testPublishAndConsume(String serviceUrl, boolean isPersistent) throws Exception {
    String topicName = generateTopicName("testpubconsume", isPersistent);

    int numMessages = 10;

    try (PulsarClient client = PulsarClient.builder()
        .serviceUrl(serviceUrl)
        .build()) {

        try (Consumer<String> consumer = client.newConsumer(Schema.STRING)
            .topic(topicName)
            .subscriptionName("my-sub")
            .subscribe()) {

            try (Producer<String> producer = client.newProducer(Schema.STRING)
                .topic(topicName)
                .create()) {

                for (int i = 0; i < numMessages; i++) {
                    producer.send("smoke-message-" + i);
                }
            }

            for (int i = 0; i < numMessages; i++) {
                Message<String> m = consumer.receive();
                assertEquals("smoke-message-" + i, m.getValue());
            }
        }
    }
}
 

@Override
public int choosePartition(Message<?> msg, TopicMetadata metadata) {
    if (msg.hasProperty(PARTITION_ID)) {
        return Integer.parseInt(msg.getProperty(PARTITION_ID));
    } else {
        return super.choosePartition(msg, metadata);
    }
}
 

protected <T extends Comparable<T>> void receiveMessagesCheckOrderAndDuplicate
        (List<Consumer<T>> consumerList, int messagesToReceive) throws PulsarClientException {
    Set<T> messagesReceived = Sets.newHashSet();
    for (Consumer<T> consumer : consumerList) {
        Message<T> currentReceived;
        Map<String, Message<T>> lastReceivedMap = new HashMap<>();
        while (true) {
            try {
                currentReceived = consumer.receive(3, TimeUnit.SECONDS);
            } catch (PulsarClientException e) {
                log.info("no more messages to receive for consumer {}", consumer.getConsumerName());
                break;
            }
            // Make sure that messages are received in order
            if (currentReceived != null) {
                consumer.acknowledge(currentReceived);
                if (lastReceivedMap.containsKey(currentReceived.getTopicName())) {
                    assertTrue(currentReceived.getMessageId().compareTo(
                            lastReceivedMap.get(currentReceived.getTopicName()).getMessageId()) > 0,
                            "Received messages are not in order.");
                }
            } else {
                break;
            }
            lastReceivedMap.put(currentReceived.getTopicName(), currentReceived);
            // Make sure that there are no duplicates
            assertTrue(messagesReceived.add(currentReceived.getValue()),
                    "Received duplicate message " + currentReceived.getValue());
        }
    }
    assertEquals(messagesReceived.size(), messagesToReceive);
}
 

/**
 * Test that sequence id from a producer is correct when there are send errors
 */
@Test
public void testCheckSequenceId() throws Exception {
    admin.namespaces().createNamespace("prop/my-test", Collections.singleton("usc"));

    String topicName = "prop/my-test/my-topic";
    int N = 10;

    PulsarClient client = PulsarClient.builder().serviceUrl(pulsar.getBrokerServiceUrl()).build();

    // Create consumer
    Consumer<String> consumer = client.newConsumer(Schema.STRING).topic(topicName).subscriptionName("sub")
            .subscribe();

    // Fence the topic by opening the ManagedLedger for the topic outside the Pulsar broker. This will cause the
    // broker to fail subsequent send operation and it will trigger a recover
    ManagedLedgerClientFactory clientFactory = new ManagedLedgerClientFactory(pulsar.getConfiguration(),
            pulsar.getZkClient(), pulsar.getBookKeeperClientFactory());
    ManagedLedgerFactory mlFactory = clientFactory.getManagedLedgerFactory();
    ManagedLedger ml = mlFactory.open(TopicName.get(topicName).getPersistenceNamingEncoding());
    ml.close();
    clientFactory.close();

    // Create a producer
    Producer<String> producer = client.newProducer(Schema.STRING).topic(topicName).create();

    for (int i = 0; i < N; i++) {
        producer.send("Hello-" + i);
    }

    for (int i = 0; i < N; i++) {
        Message<String> msg = consumer.receive();
        assertEquals(msg.getValue(), "Hello-" + i);
        assertEquals(msg.getSequenceId(), i);
        consumer.acknowledge(msg);
    }

    client.close();
}
 

@Test
public void testMessageImplReplicatedInfo() {
    String from = "ClusterNameOfReplicatedFrom";
    MessageMetadata.Builder builder = MessageMetadata.newBuilder().setReplicatedFrom(from);
    ByteBuffer payload = ByteBuffer.wrap(new byte[0]);
    Message<byte[]> msg = MessageImpl.create(builder, payload, Schema.BYTES);

    assertTrue(msg.isReplicated());
    assertEquals(msg.getReplicatedFrom(), from);
}
 

@Override
protected boolean canEnqueueMessage(Message<T> message) {
    if (listener != null) {
        triggerZeroQueueSizeListener(message);
        return false;
    } else {
        return true;
    }
}
 

@Override
protected CompletableFuture<Message<T>> internalReceiveAsync() {
    CompletableFuture<Message<T>> future = super.internalReceiveAsync();
    if (!future.isDone()) {
        // We expect the message to be not in the queue yet
        sendFlowPermitsToBroker(cnx(), 1);
    }

    return future;
}