org.apache.pulsar.client.api.SubscriptionType Java Examples

protected static Consumer<Double> getConsumer(String topicName) throws PulsarClientException {
    if (!integerConsumerMap.containsKey(topicName)) {
        Consumer<Double> newConsumer = getClient()
                .newConsumer(Schema.DOUBLE)
                .topic(topicName)
                .consumerName("hazelcast-jet-consumer-" + topicName)
                .subscriptionName("hazelcast-jet-subscription")
                .subscriptionType(SubscriptionType.Exclusive)
                .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
                .receiverQueueSize(QUEUE_CAPACITY)
                .subscribe();
        integerConsumerMap.put(topicName, newConsumer);
        return newConsumer;
    } else {
        return integerConsumerMap.get(topicName);
    }
}
 

@Override
public void redeliverUnacknowledgedMessages(Set<MessageId> messageIds) {
    if (messageIds.isEmpty()) {
        return;
    }

    checkArgument(messageIds.stream().findFirst().get() instanceof TopicMessageIdImpl);

    if (conf.getSubscriptionType() != SubscriptionType.Shared) {
        // We cannot redeliver single messages if subscription type is not Shared
        redeliverUnacknowledgedMessages();
        return;
    }
    removeExpiredMessagesFromQueue(messageIds);
    messageIds.stream().map(messageId -> (TopicMessageIdImpl)messageId)
        .collect(Collectors.groupingBy(TopicMessageIdImpl::getTopicPartitionName, Collectors.toSet()))
        .forEach((topicName, messageIds1) ->
            consumers.get(topicName)
                .redeliverUnacknowledgedMessages(messageIds1.stream()
                    .map(mid -> mid.getInnerMessageId()).collect(Collectors.toSet())));
    resumeReceivingFromPausedConsumersIfNeeded();
}
 

protected SubType getSubType() {
    SubscriptionType type = conf.getSubscriptionType();
    switch (type) {
    case Exclusive:
        return SubType.Exclusive;

    case Shared:
        return SubType.Shared;

    case Failover:
        return SubType.Failover;

    case Key_Shared:
        return SubType.Key_Shared;
    }

    // Should not happen since we cover all cases above
    return null;
}
 

private static <T> void ensureSubscriptionCreated(String inputTopicName,
                                                  String subscriptionName,
                                                  Schema<T> inputTopicSchema)
        throws Exception {
    // ensure the function subscription exists before we start producing messages
    try (PulsarClient client = PulsarClient.builder()
        .serviceUrl(pulsarCluster.getPlainTextServiceUrl())
        .build()) {
        try (Consumer<T> ignored = client.newConsumer(inputTopicSchema)
            .topic(inputTopicName)
            .subscriptionType(SubscriptionType.Shared)
            .subscriptionName(subscriptionName)
            .subscribe()) {
        }
    }
}
 

private static <T> void ensureSubscriptionCreated(String inputTopicName,
                                                  String subscriptionName,
                                                  Schema<T> inputTopicSchema)
        throws Exception {
    // ensure the function subscription exists before we start producing messages
    try (PulsarClient client = PulsarClient.builder()
        .serviceUrl(container.getPlainTextServiceUrl())
        .build()) {
        try (Consumer<T> ignored = client.newConsumer(inputTopicSchema)
            .topic(inputTopicName)
            .subscriptionType(SubscriptionType.Shared)
            .subscriptionName(subscriptionName)
            .subscribe()) {
        }
    }
}
 

@Override
protected void setup() throws Exception {
    super.internalSetup();
    super.producerBaseSetup();

    serviceUrl = pulsar.getWebServiceAddress();

    pulsarSpoutConf = new PulsarSpoutConfiguration();
    pulsarSpoutConf.setServiceUrl(serviceUrl);
    pulsarSpoutConf.setTopic(topic);
    pulsarSpoutConf.setSubscriptionName(subscriptionName);
    pulsarSpoutConf.setMessageToValuesMapper(messageToValuesMapper);
    pulsarSpoutConf.setFailedRetriesTimeout(1, TimeUnit.SECONDS);
    pulsarSpoutConf.setMaxFailedRetries(2);
    pulsarSpoutConf.setSharedConsumerEnabled(true);
    pulsarSpoutConf.setMetricsTimeIntervalInSecs(60);
    pulsarSpoutConf.setSubscriptionType(SubscriptionType.Shared);
    spout = new PulsarSpout(pulsarSpoutConf, PulsarClient.builder());
    mockCollector = new MockSpoutOutputCollector();
    SpoutOutputCollector collector = new SpoutOutputCollector(mockCollector);
    TopologyContext context = mock(TopologyContext.class);
    when(context.getThisComponentId()).thenReturn("test-spout-" + methodName);
    when(context.getThisTaskId()).thenReturn(0);
    spout.open(Maps.newHashMap(), context, collector);
    producer = pulsarClient.newProducer().topic(topic).create();
}
 

@Test
public void testFailedConsumer() {
    PulsarSpoutConfiguration pulsarSpoutConf = new PulsarSpoutConfiguration();
    pulsarSpoutConf.setServiceUrl(serviceUrl);
    pulsarSpoutConf.setTopic("persistent://invalidTopic");
    pulsarSpoutConf.setSubscriptionName(subscriptionName);
    pulsarSpoutConf.setMessageToValuesMapper(messageToValuesMapper);
    pulsarSpoutConf.setFailedRetriesTimeout(1, TimeUnit.SECONDS);
    pulsarSpoutConf.setMaxFailedRetries(2);
    pulsarSpoutConf.setSharedConsumerEnabled(false);
    pulsarSpoutConf.setMetricsTimeIntervalInSecs(60);
    pulsarSpoutConf.setSubscriptionType(SubscriptionType.Shared);
    PulsarSpout spout = new PulsarSpout(pulsarSpoutConf, PulsarClient.builder());
    MockSpoutOutputCollector mockCollector = new MockSpoutOutputCollector();
    SpoutOutputCollector collector = new SpoutOutputCollector(mockCollector);
    TopologyContext context = mock(TopologyContext.class);
    when(context.getThisComponentId()).thenReturn("new-test" + methodName);
    when(context.getThisTaskId()).thenReturn(0);
    try {
        spout.open(Maps.newHashMap(), context, collector);
        fail("should have failed as consumer creation failed");
    } catch (IllegalStateException e) {
        // Ok
    }
}
 

@Test(dataProvider = "containerBuilder")
public void testRetrieveSequenceIdGenerated(BatcherBuilder builder) throws Exception {

    int numMsgs = 10;
    final String topicName = "persistent://prop/ns-abc/testRetrieveSequenceIdGenerated-" + UUID.randomUUID();
    final String subscriptionName = "sub-1";

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
            .subscriptionType(SubscriptionType.Shared).subscribe();

    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
            .batchingMaxPublishDelay(5, TimeUnit.SECONDS).batchingMaxMessages(numMsgs).enableBatching(true)
            .batcherBuilder(builder)
            .create();

    List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
    for (int i = 0; i < numMsgs; i++) {
        byte[] message = ("my-message-" + i).getBytes();
        sendFutureList.add(producer.sendAsync(message));
    }
    FutureUtil.waitForAll(sendFutureList).get();

    for (int i = 0; i < numMsgs; i++) {
        Message<byte[]> received = consumer.receive();
        Assert.assertEquals(received.getSequenceId(), i);
        consumer.acknowledge(received);
    }

    producer.close();
    consumer.close();
}
 

@Override
public CompletableFuture<BenchmarkConsumer> createConsumer(String topic, String subscriptionName,
                ConsumerCallback consumerCallback) {
    return client.newConsumer().subscriptionType(SubscriptionType.Failover).messageListener((consumer, msg) -> {
        consumerCallback.messageReceived(msg.getData(), msg.getPublishTime());
        consumer.acknowledgeAsync(msg);
    }).topic(topic).subscriptionName(subscriptionName).subscribeAsync()
                    .thenApply(consumer -> new PulsarBenchmarkConsumer(consumer));


}
 

@Test(dataProvider = "batch")
public void testSyncProducerAndConsumer(int batchMessageDelayMs) throws Exception {
    log.info("-- Starting {} test --", methodName);

    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING)
                .topic("persistent://my-property/use/my-ns/my-topic1")
                .subscriptionName("my-subscriber-name")
                .subscriptionType(SubscriptionType.Exclusive)
                .subscribe();

    ProducerBuilder<String> producerBuilder = pulsarClient.newProducer(Schema.STRING)
            .topic("persistent://my-property/use/my-ns/my-topic1");

    if (batchMessageDelayMs != 0) {
        producerBuilder.enableBatching(true)
            .batchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS)
            .batchingMaxMessages(5);
    } else {
        producerBuilder.enableBatching(false);
    }

    Producer<String> producer = producerBuilder.create();
    for (int i = 0; i < 10; i++) {
        producer.send("my-message-" + i);
    }

    Message<String> msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = msg.getValue();
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 

public RawReaderImpl(PulsarClientImpl client, String topic, String subscription,
                     CompletableFuture<Consumer<byte[]>> consumerFuture) {
    consumerConfiguration = new ConsumerConfigurationData<>();
    consumerConfiguration.getTopicNames().add(topic);
    consumerConfiguration.setSubscriptionName(subscription);
    consumerConfiguration.setSubscriptionType(SubscriptionType.Exclusive);
    consumerConfiguration.setReceiverQueueSize(DEFAULT_RECEIVER_QUEUE_SIZE);
    consumerConfiguration.setReadCompacted(true);

    consumer = new RawConsumerImpl(client, consumerConfiguration,
                                   consumerFuture);
}
 

@Test()
public void testFailedZeroQueueSizeBatchMessage() throws PulsarClientException {

    int batchMessageDelayMs = 100;
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://prop-xyz/use/ns-abc/topic1")
            .subscriptionName("my-subscriber-name").subscriptionType(SubscriptionType.Shared).receiverQueueSize(0)
            .subscribe();

    ProducerBuilder<byte[]> producerBuilder = pulsarClient.newProducer()
        .topic("persistent://prop-xyz/use/ns-abc/topic1")
        .messageRoutingMode(MessageRoutingMode.SinglePartition);

    if (batchMessageDelayMs != 0) {
        producerBuilder.enableBatching(true).batchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS)
                .batchingMaxMessages(5);
    } else {
        producerBuilder.enableBatching(false);
    }

    Producer<byte[]> producer = producerBuilder.create();
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }

    try {
        consumer.receiveAsync().handle((ok, e) -> {
            if (e == null) {
                // as zero receiverQueueSize doesn't support batch message, must receive exception at callback.
                Assert.fail();
            }
            return null;
        });
    } finally {
        consumer.close();
    }
}
 

@Test
public void testZeroQueueSizeMessageRedelivery() throws PulsarClientException {
    final String topic = "persistent://prop/ns-abc/testZeroQueueSizeMessageRedelivery";
    Consumer<Integer> consumer = pulsarClient.newConsumer(Schema.INT32)
        .topic(topic)
        .receiverQueueSize(0)
        .subscriptionName("sub")
        .subscriptionType(SubscriptionType.Shared)
        .ackTimeout(1, TimeUnit.SECONDS)
        .subscribe();

    final int messages = 10;
    Producer<Integer> producer = pulsarClient.newProducer(Schema.INT32)
        .topic(topic)
        .enableBatching(false)
        .create();

    for (int i = 0; i < messages; i++) {
        producer.send(i);
    }

    Set<Integer> receivedMessages = new HashSet<>();
    for (int i = 0; i < messages * 2; i++) {
        receivedMessages.add(consumer.receive().getValue());
    }

    Assert.assertEquals(receivedMessages.size(), messages);

    consumer.close();
    producer.close();
}
 

@Test
public void testZeroQueueSizeMessageRedeliveryForListener() throws Exception {
    final String topic = "persistent://prop/ns-abc/testZeroQueueSizeMessageRedeliveryForListener";
    final int messages = 10;
    final CountDownLatch latch = new CountDownLatch(messages * 2);
    Set<Integer> receivedMessages = new HashSet<>();
    Consumer<Integer> consumer = pulsarClient.newConsumer(Schema.INT32)
        .topic(topic)
        .receiverQueueSize(0)
        .subscriptionName("sub")
        .subscriptionType(SubscriptionType.Shared)
        .ackTimeout(1, TimeUnit.SECONDS)
        .messageListener((MessageListener<Integer>) (c, msg) -> {
            try {
                receivedMessages.add(msg.getValue());
            } finally {
                latch.countDown();
            }
        })
        .subscribe();

    Producer<Integer> producer = pulsarClient.newProducer(Schema.INT32)
        .topic(topic)
        .enableBatching(false)
        .create();

    for (int i = 0; i < messages; i++) {
        producer.send(i);
    }

    latch.await();
    Assert.assertEquals(receivedMessages.size(), messages);

    consumer.close();
    producer.close();
}
 

@Test(dataProvider = "batch")
public void testSendTimeout(int batchMessageDelayMs) throws Exception {
    log.info("-- Starting {} test --", methodName);

    Consumer<byte[]> consumer = pulsarClient.newConsumer()
            .topic("persistent://my-property/use/my-ns/my-topic5")
            .subscriptionName("my-subscriber-name")
            .subscriptionType(SubscriptionType.Exclusive)
            .subscribe();

    Producer<byte[]> producer = pulsarClient.newProducer()
            .topic("persistent://my-property/use/my-ns/my-topic5")
            .batchingMaxMessages(5)
            .batchingMaxPublishDelay(2 * BATCHING_MAX_PUBLISH_DELAY_THRESHOLD, TimeUnit.MILLISECONDS)
            .enableBatching(batchMessageDelayMs != 0)
            .sendTimeout(1, TimeUnit.SECONDS)
            .create();

    final String message = "my-message";

    // Trigger the send timeout
    stopBroker();

    Future<MessageId> future = producer.sendAsync(message.getBytes());

    try {
        future.get();
        Assert.fail("Send operation should have failed");
    } catch (ExecutionException e) {
        // Expected
    }

    startBroker();

    // We should not have received any message
    Message<byte[]> msg = consumer.receive(3, TimeUnit.SECONDS);
    Assert.assertNull(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 

@Test(timeOut = testTimeout)
public void testDifferentTopicsNameSubscribe() throws Exception {
    String key = "TopicsFromDifferentNamespace";
    final String subscriptionName = "my-ex-subscription-" + key;

    final String topicName1 = "persistent://prop/use/ns-abc1/topic-1-" + key;
    final String topicName2 = "persistent://prop/use/ns-abc2/topic-2-" + key;
    final String topicName3 = "persistent://prop/use/ns-abc3/topic-3-" + key;
    List<String> topicNames = Lists.newArrayList(topicName1, topicName2, topicName3);

    TenantInfo tenantInfo = createDefaultTenantInfo();
    admin.tenants().createTenant("prop", tenantInfo);
    admin.topics().createPartitionedTopic(topicName2, 2);
    admin.topics().createPartitionedTopic(topicName3, 3);

    // 2. Create consumer
    try {
        Consumer consumer = pulsarClient.newConsumer()
            .topics(topicNames)
            .subscriptionName(subscriptionName)
            .subscriptionType(SubscriptionType.Shared)
            .ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS)
            .subscribe();
        assertTrue(consumer instanceof MultiTopicsConsumerImpl);
    } catch (IllegalArgumentException e) {
        // expected for have different namespace
    }
}
 

@Test
public void testBatchMessageIndexAckForSharedSubscription() throws PulsarClientException, ExecutionException, InterruptedException {
    final String topic = "testBatchMessageIndexAckForSharedSubscription";

    @Cleanup
    Consumer<Integer> consumer = pulsarClient.newConsumer(Schema.INT32)
        .topic(topic)
        .subscriptionName("sub")
        .receiverQueueSize(100)
        .subscriptionType(SubscriptionType.Shared)
        .ackTimeout(1, TimeUnit.SECONDS)
        .subscribe();

    @Cleanup
    Producer<Integer> producer = pulsarClient.newProducer(Schema.INT32)
        .topic(topic)
        .batchingMaxPublishDelay(50, TimeUnit.MILLISECONDS)
        .create();

    final int messages = 100;
    List<CompletableFuture<MessageId>> futures = new ArrayList<>(messages);
    for (int i = 0; i < messages; i++) {
        futures.add(producer.sendAsync(i));
    }
    FutureUtil.waitForAll(futures).get();

    for (int i = 0; i < messages; i++) {
        if (i % 2 == 0) {
            consumer.acknowledge(consumer.receive());
        }
    }

    List<Message<Integer>> received = new ArrayList<>(messages);
    for (int i = 0; i < messages; i++) {
        received.add(consumer.receive());
    }

    Assert.assertEquals(received.size(), 100);
}
 

@Override
public int take(BatchMaker batchMaker, Source.Context context, int batchSize) throws StageException {
  long start = System.currentTimeMillis();
  int numMessagesConsumed = 0;

  while (System.currentTimeMillis() - start < basicConfig.maxWaitTime && numMessagesConsumed < batchSize) {
    if (LOG.isTraceEnabled()) {
      LOG.trace("Attempting to take up to '{}' messages", (batchSize - numMessagesConsumed));
    }
    try {
      Message message;
      long pollInterval = basicConfig.maxWaitTime - (System.currentTimeMillis() - start);
      if (pollInterval > Integer.MAX_VALUE) {
        message = messageConsumer.receive(POLL_INTERVAL, TimeUnit.MILLISECONDS);
      } else {
        message = messageConsumer.receive(Long.valueOf(pollInterval).intValue(), TimeUnit.MILLISECONDS);
      }

      if (message != null) {
        if (LOG.isTraceEnabled()) {
          LOG.trace("Got message: '{}'", new String(message.getData()));
        }
        String messageId = Base64.getEncoder().encodeToString(message.getMessageId().toByteArray());
        numMessagesConsumed += pulsarMessageConverter.convert(batchMaker, context, messageId, message);
        if (!pulsarConfig.pulsarTopicsSelector.equals(PulsarTopicsSelector.SINGLE_TOPIC) ||
            !SubscriptionType.Exclusive.equals(pulsarConfig.subscriptionType.getSubscriptionType())) {
          sentButNotACKMessages.add(message);
        } else {
          lastSentButNotACKMessage = message;
        }
      }
    } catch (PulsarClientException e) {
      throw new StageException(PulsarErrors.PULSAR_08, e.toString(), e);
    }
  }

  return numMessagesConsumed;
}
 

@Override
public CompletableFuture<Consumer<T>> subscribeAsync() {
    if (conf.getTopicNames().isEmpty() && conf.getTopicsPattern() == null) {
        return FutureUtil
                .failedFuture(new InvalidConfigurationException("Topic name must be set on the consumer builder"));
    }

    if (StringUtils.isBlank(conf.getSubscriptionName())) {
        return FutureUtil.failedFuture(
                new InvalidConfigurationException("Subscription name must be set on the consumer builder"));
    }

    if (conf.getKeySharedPolicy() != null && conf.getSubscriptionType() != SubscriptionType.Key_Shared) {
        return FutureUtil.failedFuture(
                new InvalidConfigurationException("KeySharedPolicy must set with KeyShared subscription"));
    }
    if(conf.isRetryEnable() && conf.getTopicNames().size() > 0 ) {
        TopicName topicFirst = TopicName.get(conf.getTopicNames().iterator().next());
        String retryLetterTopic = topicFirst.getNamespace() + "/" + conf.getSubscriptionName() + RetryMessageUtil.RETRY_GROUP_TOPIC_SUFFIX;
        String deadLetterTopic = topicFirst.getNamespace() + "/" + conf.getSubscriptionName() + RetryMessageUtil.DLQ_GROUP_TOPIC_SUFFIX;
        if(conf.getDeadLetterPolicy() == null) {
            conf.setDeadLetterPolicy(DeadLetterPolicy.builder()
                                    .maxRedeliverCount(RetryMessageUtil.MAX_RECONSUMETIMES)
                                    .retryLetterTopic(retryLetterTopic)
                                    .deadLetterTopic(deadLetterTopic)
                                    .build());
        } else {
            if (StringUtils.isBlank(conf.getDeadLetterPolicy().getRetryLetterTopic())) {
                conf.getDeadLetterPolicy().setRetryLetterTopic(retryLetterTopic);
            }
            if (StringUtils.isBlank(conf.getDeadLetterPolicy().getDeadLetterTopic())) {
                conf.getDeadLetterPolicy().setDeadLetterTopic(deadLetterTopic);
            }
        }
        conf.getTopicNames().add(conf.getDeadLetterPolicy().getRetryLetterTopic());
    }
    return interceptorList == null || interceptorList.size() == 0 ?
            client.subscribeAsync(conf, schema, null) :
            client.subscribeAsync(conf, schema, new ConsumerInterceptors<>(interceptorList));
}
 

public static void main(String[] args) throws IOException {
    // Create a Pulsar client instance. A single instance can be shared across many
    // producers and consumer within the same application
    PulsarClient client = PulsarClient.builder()
            .serviceUrl(SERVICE_URL)
            .build();

    //Configure consumer specific settings.
    Consumer<byte[]> consumer = client.newConsumer()
            .topic(TOPIC_NAME)
            // Allow multiple consumers to attach to the same subscription
            // and get messages dispatched as a queue
            .subscriptionType(SubscriptionType.Shared)
            .subscriptionName(SUBSCRIPTION_NAME)
            .subscribe();


    // Once the consumer is created, it can be used for the entire application lifecycle
    System.out.println("Created consumer for the topic "+ TOPIC_NAME);

    do {
        // Wait until a message is available
        Message<byte[]> msg = consumer.receive();

        // Extract the message as a printable string and then log
        String content = new String(msg.getData());
        System.out.println("Received message '"+content+"' with ID "+msg.getMessageId());

        // Acknowledge processing of the message so that it can be deleted
        consumer.acknowledge(msg);
    } while (true);
}
 

@Test
public void checkMessages() throws PulsarClientException {

    @Cleanup
    PulsarClient client = PulsarClient.builder()
            .serviceUrl(pulsarContainer.getPlainTextPulsarBrokerUrl())
            .build();

    final String inputTopic = "input";

    Producer<String> producer = client.newProducer(Schema.STRING)
            .topic(inputTopic)
            .enableBatching(false)
            .create();

    Consumer<String> consumer = client.newConsumer(Schema.STRING)
            .topic(inputTopic)
            .subscriptionName("test-subs")
            .ackTimeout(10, TimeUnit.SECONDS)
            .subscriptionType(SubscriptionType.Exclusive)
            .subscribe();

    producer.send("Hello!");
    Message<String> message = consumer.receive(10, TimeUnit.SECONDS);

    Assert.assertEquals(message.getValue(), "Hello!");

}
 

@Test
public void testAckStatsOnPartitionedTopicForExclusiveSubscription() throws PulsarAdminException, PulsarClientException, InterruptedException {
    final String topic = "persistent://my-property/my-ns/testAckStatsOnPartitionedTopicForExclusiveSubscription";
    admin.topics().createPartitionedTopic(topic, 3);
    Consumer<byte[]> consumer = pulsarClient.newConsumer()
            .topic(topic)
            .subscriptionType(SubscriptionType.Exclusive)
            .subscriptionName("sub")
            .subscribe();

    Producer<byte[]> producer = pulsarClient.newProducer()
            .topic(topic)
            .create();

    final int messages = 10;
    for (int i = 0; i < messages; i++) {
        producer.send(("message-" + i).getBytes());
    }

    int received = 0;
    for (int i = 0; i < messages; i++) {
        consumer.acknowledge(consumer.receive());
        received++;
    }
    Assert.assertEquals(messages, received);

    // wait acknowledge send
    Thread.sleep(2000);

    for (int i = 0; i < 3; i++) {
        TopicStats stats = admin.topics().getStats(topic + "-partition-" + i);
        Assert.assertEquals(stats.subscriptions.size(), 1);
        Assert.assertEquals(stats.subscriptions.entrySet().iterator().next().getValue().consumers.size(), 1);
        Assert.assertEquals(stats.subscriptions.entrySet().iterator().next().getValue().consumers.get(0).unackedMessages, 0);
    }
}
 

private static PulsarSourceConfig getPulsarConfigs(boolean multiple) {
    PulsarSourceConfig pulsarConfig = new PulsarSourceConfig();
    pulsarConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE);
    if (multiple) {
        pulsarConfig.setTopicSchema(multipleConsumerConfigs);
    } else {
        pulsarConfig.setTopicSchema(consumerConfigs);
    }
    pulsarConfig.setTypeClassName(String.class.getName());
    pulsarConfig.setSubscriptionPosition(SubscriptionInitialPosition.Latest);
    pulsarConfig.setSubscriptionType(SubscriptionType.Shared);
    return pulsarConfig;
}
 

@Test
public void checkMessages() throws PulsarClientException {

    @Cleanup
    PulsarClient client = PulsarClient.builder()
            .serviceUrl(pulsarContainer.getPlainTextPulsarBrokerUrl())
            .build();

    final String inputTopic = "input";

    Producer<String> producer = client.newProducer(Schema.STRING)
            .topic(inputTopic)
            .enableBatching(false)
            .create();

    Consumer<String> consumer = client.newConsumer(Schema.STRING)
            .topic(inputTopic)
            .subscriptionName("test-subs")
            .ackTimeout(10, TimeUnit.SECONDS)
            .subscriptionType(SubscriptionType.Exclusive)
            .subscribe();

    producer.send("Hello!");
    Message<String> message = consumer.receive(10, TimeUnit.SECONDS);

    Assert.assertEquals(message.getValue(), "Hello!");

}
 

@Test
public void testProtocolVersionAdvertisement() throws Exception {
    final String topic = "persistent://sample/test/local/protocol-version-advertisement";
    final String sub = "my-sub";

    ClientConfigurationData conf = new ClientConfigurationData();
    conf.setServiceUrl(proxyService.getServiceUrl());

    @Cleanup
    PulsarClient client = getClientActiveConsumerChangeNotSupported(conf);

    @Cleanup
    Producer<byte[]> producer = client.newProducer().topic(topic).create();

    @Cleanup
    Consumer<byte[]> consumer = client.newConsumer().topic(topic).subscriptionName(sub)
            .subscriptionType(SubscriptionType.Failover).subscribe();

    for (int i = 0; i < 10; i++) {
        producer.send("test-msg".getBytes());
    }

    for (int i = 0; i < 10; i++) {
        Message<byte[]> msg = consumer.receive(10, TimeUnit.SECONDS);
        checkNotNull(msg);
        consumer.acknowledge(msg);
    }
}
 

@Test
private void testProtocolVersionAdvertisement() throws Exception {
    final String topic = "persistent://sample/test/local/protocol-version-advertisement";
    final String sub = "my-sub";

    ClientConfigurationData conf = new ClientConfigurationData();
    conf.setServiceUrl(proxyService.getServiceUrl());
    PulsarClient client = getClientActiveConsumerChangeNotSupported(conf);

    Producer<byte[]> producer = client.newProducer().topic(topic).create();
    Consumer<byte[]> consumer = client.newConsumer().topic(topic).subscriptionName(sub)
            .subscriptionType(SubscriptionType.Failover).subscribe();

    for (int i = 0; i < 10; i++) {
        producer.send("test-msg".getBytes());
    }

    for (int i = 0; i < 10; i++) {
        Message<byte[]> msg = consumer.receive(10, TimeUnit.SECONDS);
        checkNotNull(msg);
        consumer.acknowledge(msg);
    }

    producer.close();
    consumer.close();
    client.close();
}
 

protected void nonPartitionedTopicSendAndReceiveWithExclusive(String serviceUrl, boolean isPersistent) throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String topicName = getNonPartitionedTopic("test-non-partitioned-consume-exclusive", isPersistent);
    @Cleanup
    final PulsarClient client = PulsarClient.builder()
            .serviceUrl(serviceUrl)
            .build();
    @Cleanup
    final Consumer<String> consumer = client.newConsumer(Schema.STRING)
            .topic(topicName)
            .subscriptionName("test-sub")
            .subscriptionType(SubscriptionType.Exclusive)
            .subscribe();
    try {
        client.newConsumer(Schema.STRING)
                .topic(topicName)
                .subscriptionName("test-sub")
                .subscriptionType(SubscriptionType.Exclusive)
                .subscribe();
        fail("should be failed");
    } catch (PulsarClientException ignore) {
    }
    final int messagesToSend = 10;
    final String producerName = "producerForExclusive";
    @Cleanup
    final Producer<String> producer = client.newProducer(Schema.STRING)
            .topic(topicName)
            .enableBatching(false)
            .producerName(producerName)
            .create();
    for (int i = 0; i < messagesToSend; i++) {
        MessageId messageId = producer.newMessage().value(producer.getProducerName() + "-" + i).send();
        assertNotNull(messageId);
    }
    log.info("public messages complete.");
    receiveMessagesCheckOrderAndDuplicate(Collections.singletonList(consumer), messagesToSend);
    log.info("-- Exiting {} test --", methodName);
}
 

@Test(dataProvider = "ServiceUrls")
public void testEffectivelyOnceEnabled(String serviceUrl) throws Exception {
    String nsName = generateNamespaceName();
    pulsarCluster.createNamespace(nsName);
    pulsarCluster.enableDeduplication(nsName, true);

    String topicName = generateTopicName(nsName, "testeffectivelyonce", true);

    @Cleanup
    PulsarClient client = PulsarClient.builder()
        .serviceUrl(serviceUrl)
        .build();

    @Cleanup
    Consumer<String> consumer = client.newConsumer(Schema.STRING)
        .topic(topicName)
        .subscriptionName("test-sub")
        .ackTimeout(10, TimeUnit.SECONDS)
        .subscriptionType(SubscriptionType.Exclusive)
        .subscribe();

    @Cleanup
    Producer<String> producer = client.newProducer(Schema.STRING)
        .topic(topicName)
        .enableBatching(false)
        .producerName("effectively-once-producer")
        .initialSequenceId(1L)
        .create();

    // send messages
    sendMessagesIdempotency(producer);

    // checkout the result
    checkMessagesIdempotencyEnabled(consumer);
}
 

private static void publishAndConsumeAvroMessages(String inputTopic,
                                                  String outputTopic,
                                                  int numMessages) throws Exception {

    @Cleanup PulsarClient client = PulsarClient.builder()
        .serviceUrl(pulsarCluster.getPlainTextServiceUrl())
        .build();

    @Cleanup Consumer<String> consumer = client.newConsumer(Schema.STRING)
        .topic(outputTopic)
        .subscriptionType(SubscriptionType.Exclusive)
        .subscriptionName("test-sub")
        .subscribe();

    @Cleanup Producer<CustomObject> producer = client.newProducer(Schema.AVRO(CustomObject.class))
        .topic(inputTopic)
        .create();

    for (int i = 0; i < numMessages; i++) {
        CustomObject co = new CustomObject(i);
        producer.send(co);
    }

    for (int i = 0; i < numMessages; i++) {
        Message<String> msg = consumer.receive();
        assertEquals("value-" + i, msg.getValue());
    }
}
 

@Test(dataProvider = "ServiceUrls")
public void testEffectivelyOnceDisabled(String serviceUrl) throws Exception {
    String nsName = generateNamespaceName();
    pulsarCluster.createNamespace(nsName);

    String topicName = generateTopicName(nsName, "testeffectivelyonce", true);

    @Cleanup
    PulsarClient client = PulsarClient.builder()
        .serviceUrl(serviceUrl)
        .build();

    @Cleanup
    Consumer<String> consumer = client.newConsumer(Schema.STRING)
        .topic(topicName)
        .subscriptionName("test-sub")
        .ackTimeout(10, TimeUnit.SECONDS)
        .subscriptionType(SubscriptionType.Exclusive)
        .subscribe();

    @Cleanup
    Producer<String> producer = client.newProducer(Schema.STRING)
        .topic(topicName)
        .enableBatching(false)
        .producerName("effectively-once-producer")
        .initialSequenceId(1L)
        .create();

    // send messages
    sendMessagesIdempotency(producer);

    // checkout the result
    checkMessagesIdempotencyDisabled(consumer);
}