Java Code Examples for org.apache.pulsar.client.api.Consumer#close()

The following examples show how to use org.apache.pulsar.client.api.Consumer#close() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Test
public void upgradeToReplicatedSubscriptionAfterRestart() throws Exception {
    String topic = "upgradeToReplicatedSubscriptionAfterRestart-" + System.nanoTime();

    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING)
            .topic(topic)
            .subscriptionName("sub")
            .replicateSubscriptionState(false)
            .subscribe();

    TopicStats stats = admin.topics().getStats(topic);
    assertFalse(stats.subscriptions.get("sub").isReplicated);
    consumer.close();

    admin.topics().unload(topic);

    consumer = pulsarClient.newConsumer(Schema.STRING)
            .topic(topic)
            .subscriptionName("sub")
            .replicateSubscriptionState(true)
            .subscribe();

    stats = admin.topics().getStats(topic);
    assertTrue(stats.subscriptions.get("sub").isReplicated);
    consumer.close();
}
 
Example 2
Source Project: pulsar   File: PersistentTopicE2ETest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGC() throws Exception {
    // 1. Simple successful GC
    String topicName = "persistent://prop/ns-abc/topic-10";
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    producer.close();

    assertTrue(pulsar.getBrokerService().getTopicReference(topicName).isPresent());
    runGC();
    assertFalse(pulsar.getBrokerService().getTopicReference(topicName).isPresent());

    // 2. Topic is not GCed with live connection
    String subName = "sub1";
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();

    runGC();
    assertTrue(pulsar.getBrokerService().getTopicReference(topicName).isPresent());

    // 3. Topic with subscription is not GCed even with no connections
    consumer.close();

    runGC();
    assertTrue(pulsar.getBrokerService().getTopicReference(topicName).isPresent());

    // 4. Topic can be GCed after unsubscribe
    admin.topics().deleteSubscription(topicName, subName);

    runGC();
    assertFalse(pulsar.getBrokerService().getTopicReference(topicName).isPresent());
}
 
Example 3
Source Project: pulsar   File: PersistentTopicE2ETest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSimpleCloseTopic() throws Exception {
    final String topicName = "persistent://prop/ns-abc/topic5";
    final String subName = "sub5";

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();
    Producer<byte[]> producer = pulsarClient.newProducer()
        .topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();

    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();
    assertNotNull(topicRef);
    PersistentSubscription subRef = topicRef.getSubscription(subName);
    assertNotNull(subRef);

    Message<byte[]> msg;
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        msg = consumer.receive();
        consumer.acknowledge(msg);
    }

    producer.close();
    consumer.close();

    topicRef.close().get();
    assertFalse(pulsar.getBrokerService().getTopicReference(topicName).isPresent());
}
 
Example 4
Source Project: pulsar   File: V1_ProducerConsumerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testEnabledChecksumClient() throws Exception {
    log.info("-- Starting {} test --", methodName);

    final int totalMsg = 10;

    Consumer<byte[]> consumer = pulsarClient.newConsumer()
            .topic("persistent://my-property/use/my-ns/my-topic1")
            .subscriptionName("my-subscriber-name")
            .subscribe();

    final int batchMessageDelayMs = 300;
    Producer<byte[]> producer = pulsarClient.newProducer()
            .topic("persistent://my-property/use/my-ns/my-topic1")
            .enableBatching(true)
            .batchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS)
            .batchingMaxMessages(5)
            .create();

    for (int i = 0; i < totalMsg; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }

    Message<byte[]>msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < totalMsg; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 
Example 5
Source Project: pulsar   File: PersistentTopicE2ETest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * A topic that has retention policy set to -1, should not be GCed until it has been inactive for at least the
 * retention time and the data should never be deleted
 */
@Test
public void testInfiniteRetentionPolicy() throws Exception {
    // Retain data forever
    admin.namespaces().setRetention("prop/ns-abc", new RetentionPolicies(-1, -1));

    // 1. Simple successful GC
    String topicName = "persistent://prop/ns-abc/topic-10";
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    producer.close();

    assertNotNull(pulsar.getBrokerService().getTopicReference(topicName));
    runGC();
    // Should not have been deleted, since we have retention
    assertNotNull(pulsar.getBrokerService().getTopicReference(topicName));

    // Remove retention
    admin.namespaces().setRetention("prop/ns-abc", new RetentionPolicies(0, 10));
    Thread.sleep(300);

    // 2. Topic is not GCed with live connection
    String subName = "sub1";
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();

    runGC();
    assertNotNull(pulsar.getBrokerService().getTopicReference(topicName));

    // 3. Topic with subscription is not GCed even with no connections
    consumer.close();

    runGC();
    assertNotNull(pulsar.getBrokerService().getTopicReference(topicName));

    // 4. Topic can be GCed after unsubscribe
    admin.topics().deleteSubscription(topicName, subName);

    runGC();
    assertFalse(pulsar.getBrokerService().getTopicReference(topicName).isPresent());
}
 
Example 6
Source Project: pulsar   File: BrokerBkEnsemblesTests.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeOut=20000)
public void testTopicWithWildCardChar() throws Exception {
    PulsarClient client = PulsarClient.builder()
            .serviceUrl(pulsar.getWebServiceAddress())
            .statsInterval(0, TimeUnit.SECONDS)
            .build();

    final String ns1 = "prop/usc/topicWithSpecialChar";
    try {
        admin.namespaces().createNamespace(ns1);
    } catch (Exception e) {

    }

    final String topic1 = "persistent://"+ns1+"/`[email protected]#$%^&*()-_+=[]://{}|\\;:'\"<>,./?-30e04524";
    final String subName1 = "c1";
    final byte[] content = "test".getBytes();

    Consumer<byte[]> consumer = client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe();
    org.apache.pulsar.client.api.Producer<byte[]> producer = client.newProducer().topic(topic1).create();

    producer.send(content);
    Message<byte[]> msg = consumer.receive();
    Assert.assertEquals(msg.getData(), content);
    consumer.close();
    producer.close();
    client.close();
}
 
Example 7
Source Project: pulsar   File: PersistentTopicE2ETest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testConsumerFlowControl() throws Exception {
    final String topicName = "persistent://prop/ns-abc/topic2";
    final String subName = "sub2";

    Message<byte[]> msg;
    int recvQueueSize = 4;

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
            .receiverQueueSize(recvQueueSize).subscribe();
    Producer<byte[]> producer = pulsarClient.newProducer()
        .topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();

    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();
    assertNotNull(topicRef);
    PersistentSubscription subRef = topicRef.getSubscription(subName);
    assertNotNull(subRef);

    // 1. initial receive queue size recorded
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(getAvailablePermits(subRef), recvQueueSize);

    for (int i = 0; i < recvQueueSize / 2; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        msg = consumer.receive();
        consumer.acknowledge(msg);
    }

    // 2. queue size re-adjusted after successful receive of half of window size
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(getAvailablePermits(subRef), recvQueueSize);

    consumer.close();
    assertFalse(subRef.getDispatcher().isConsumerConnected());
}
 
Example 8
Source Project: pulsar   File: ZeroQueueSizeTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testZeroQueueSizeMessageRedeliveryForAsyncReceive() throws PulsarClientException, ExecutionException, InterruptedException {
    final String topic = "persistent://prop/ns-abc/testZeroQueueSizeMessageRedeliveryForAsyncReceive";
    Consumer<Integer> consumer = pulsarClient.newConsumer(Schema.INT32)
        .topic(topic)
        .receiverQueueSize(0)
        .subscriptionName("sub")
        .subscriptionType(SubscriptionType.Shared)
        .ackTimeout(1, TimeUnit.SECONDS)
        .subscribe();

    final int messages = 10;
    Producer<Integer> producer = pulsarClient.newProducer(Schema.INT32)
        .topic(topic)
        .enableBatching(false)
        .create();

    for (int i = 0; i < messages; i++) {
        producer.send(i);
    }

    Set<Integer> receivedMessages = new HashSet<>();
    List<CompletableFuture<Message<Integer>>> futures = new ArrayList<>(20);
    for (int i = 0; i < messages * 2; i++) {
        futures.add(consumer.receiveAsync());
    }
    for (CompletableFuture<Message<Integer>> future : futures) {
        receivedMessages.add(future.get().getValue());
    }

    Assert.assertEquals(receivedMessages.size(), messages);

    consumer.close();
    producer.close();
}
 
Example 9
Source Project: pulsar   File: SaslAuthenticateTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testProducerAndConsumerPassed() throws Exception {
    log.info("-- {} -- start", methodName);

    Consumer<byte[]> consumer = pulsarClient.newConsumer()
        .topic("persistent://my-property/my-ns/my-topic")
        .subscriptionName("my-subscriber-name")
        .subscribe();

    ProducerBuilder<byte[]> producerBuilder = pulsarClient.newProducer()
        .topic("persistent://my-property/my-ns/my-topic")
        .enableBatching(false);

    Producer<byte[]> producer = producerBuilder.create();
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        log.info("Produced message: [{}]", message);
    }

    Message<byte[]> msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.info("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();

    log.info("-- {} -- end", methodName);
}
 
Example 10
Source Project: pulsar   File: PrometheusMetricsTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testPerNamespaceStats() throws Exception {
    Producer<byte[]> p1 = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1").create();
    Producer<byte[]> p2 = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic2").create();

    Consumer<byte[]> c1 = pulsarClient.newConsumer()
            .topic("persistent://my-property/use/my-ns/my-topic1")
            .subscriptionName("test")
            .subscribe();

    Consumer<byte[]> c2 = pulsarClient.newConsumer()
            .topic("persistent://my-property/use/my-ns/my-topic2")
            .subscriptionName("test")
            .subscribe();

    final int messages = 10;

    for (int i = 0; i < messages; i++) {
        String message = "my-message-" + i;
        p1.send(message.getBytes());
        p2.send(message.getBytes());
    }

    for (int i = 0; i < messages; i++) {
        c1.acknowledge(c1.receive());
        c2.acknowledge(c2.receive());
    }

    ByteArrayOutputStream statsOut = new ByteArrayOutputStream();
    PrometheusMetricsGenerator.generate(pulsar, false, false, statsOut);
    String metricsStr = new String(statsOut.toByteArray());

    Multimap<String, Metric> metrics = parseMetrics(metricsStr);

    metrics.entries().forEach(e -> {
        System.out.println(e.getKey() + ": " + e.getValue());
    });

    // There should be 1 metric aggregated per namespace
    List<Metric> cm = (List<Metric>) metrics.get("pulsar_storage_write_latency_le_1");
    assertEquals(cm.size(), 1);
    assertNull(cm.get(0).tags.get("topic"));
    assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns");

    cm = (List<Metric>) metrics.get("pulsar_producers_count");
    assertEquals(cm.size(), 2);
    assertNull(cm.get(1).tags.get("topic"));
    assertEquals(cm.get(1).tags.get("namespace"), "my-property/use/my-ns");

    cm = (List<Metric>) metrics.get("pulsar_in_bytes_total");
    assertEquals(cm.size(), 1);
    assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns");

    cm = (List<Metric>) metrics.get("pulsar_in_messages_total");
    assertEquals(cm.size(), 1);
    assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns");

    cm = (List<Metric>) metrics.get("pulsar_out_bytes_total");
    assertEquals(cm.size(), 1);
    assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns");

    cm = (List<Metric>) metrics.get("pulsar_out_messages_total");
    assertEquals(cm.size(), 1);
    assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns");

    p1.close();
    p2.close();
    c1.close();
    c2.close();
}
 
Example 11
Source Project: pulsar   File: PrometheusMetricsTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testPerConsumerStats() throws Exception {
    Producer<byte[]> p1 = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1").create();
    Producer<byte[]> p2 = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic2").create();

    Consumer<byte[]> c1 = pulsarClient.newConsumer()
            .topic("persistent://my-property/use/my-ns/my-topic1")
            .subscriptionName("test")
            .subscribe();

    Consumer<byte[]> c2 = pulsarClient.newConsumer()
            .topic("persistent://my-property/use/my-ns/my-topic2")
            .subscriptionName("test")
            .subscribe();

    final int messages = 10;

    for (int i = 0; i < messages; i++) {
        String message = "my-message-" + i;
        p1.send(message.getBytes());
        p2.send(message.getBytes());
    }

    for (int i = 0; i < messages; i++) {
        c1.acknowledge(c1.receive());
        c2.acknowledge(c2.receive());
    }

    ByteArrayOutputStream statsOut = new ByteArrayOutputStream();
    PrometheusMetricsGenerator.generate(pulsar, true, true, statsOut);
    String metricsStr = new String(statsOut.toByteArray());

    Multimap<String, Metric> metrics = parseMetrics(metricsStr);

    metrics.entries().forEach(e -> {
        System.out.println(e.getKey() + ": " + e.getValue());
    });

    // There should be 1 metric aggregated per namespace
    List<Metric> cm = (List<Metric>) metrics.get("pulsar_out_bytes_total");
    assertEquals(cm.size(), 4);
    assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns");
    assertEquals(cm.get(0).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2");
    assertEquals(cm.get(0).tags.get("subscription"), "test");

    assertEquals(cm.get(1).tags.get("namespace"), "my-property/use/my-ns");
    assertEquals(cm.get(1).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2");
    assertEquals(cm.get(1).tags.get("subscription"), "test");
    assertEquals(cm.get(1).tags.get("consumer_id"), "1");

    assertEquals(cm.get(2).tags.get("namespace"), "my-property/use/my-ns");
    assertEquals(cm.get(2).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1");
    assertEquals(cm.get(2).tags.get("subscription"), "test");

    assertEquals(cm.get(3).tags.get("namespace"), "my-property/use/my-ns");
    assertEquals(cm.get(3).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1");
    assertEquals(cm.get(3).tags.get("subscription"), "test");
    assertEquals(cm.get(3).tags.get("consumer_id"), "0");

    cm = (List<Metric>) metrics.get("pulsar_out_messages_total");
    assertEquals(cm.size(), 4);
    assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns");
    assertEquals(cm.get(0).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2");
    assertEquals(cm.get(0).tags.get("subscription"), "test");

    assertEquals(cm.get(1).tags.get("namespace"), "my-property/use/my-ns");
    assertEquals(cm.get(1).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2");
    assertEquals(cm.get(1).tags.get("subscription"), "test");
    assertEquals(cm.get(1).tags.get("consumer_id"), "1");

    assertEquals(cm.get(2).tags.get("namespace"), "my-property/use/my-ns");
    assertEquals(cm.get(2).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1");
    assertEquals(cm.get(2).tags.get("subscription"), "test");

    assertEquals(cm.get(3).tags.get("namespace"), "my-property/use/my-ns");
    assertEquals(cm.get(3).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1");
    assertEquals(cm.get(3).tags.get("subscription"), "test");
    assertEquals(cm.get(3).tags.get("consumer_id"), "0");

    p1.close();
    p2.close();
    c1.close();
    c2.close();
}
 
Example 12
Source Project: pulsar   File: V1_ProducerConsumerTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeOut = 2000)
public void testAsyncProducerAndConsumer() throws Exception {
    log.info("-- Starting {} test --", methodName);

    final int totalMsg = 100;
    final Set<String> produceMsgs = Sets.newHashSet();
    final Set<String> consumeMsgs = Sets.newHashSet();

    Consumer<byte[]> consumer = pulsarClient.newConsumer()
            .topic("persistent://my-property/use/my-ns/my-topic1")
            .subscriptionName("my-subscriber-name")
            .subscriptionType(SubscriptionType.Exclusive)
            .subscribe();

    // produce message
    Producer<byte[]> producer = pulsarClient.newProducer()
            .topic("persistent://my-property/use/my-ns/my-topic1")
            .create();
    for (int i = 0; i < totalMsg; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        produceMsgs.add(message);
    }

    log.info(" start receiving messages :");
    CountDownLatch latch = new CountDownLatch(totalMsg);
    // receive messages
    ExecutorService executor = Executors.newFixedThreadPool(1);
    receiveAsync(consumer, totalMsg, 0, latch, consumeMsgs, executor);

    latch.await();

    // verify message produced correctly
    assertEquals(produceMsgs.size(), totalMsg);
    // verify produced and consumed messages must be exactly same
    produceMsgs.removeAll(consumeMsgs);
    assertTrue(produceMsgs.isEmpty());

    producer.close();
    consumer.close();
    executor.shutdownNow();
    log.info("-- Exiting {} test --", methodName);
}
 
Example 13
Source Project: pulsar   File: PersistentQueueE2ETest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testReplayOnConsumerDisconnect() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/shared-topic3";
    final String subName = "sub3";
    final int numMsgs = 100;

    final List<String> messagesProduced = Lists.newArrayListWithCapacity(numMsgs);
    final List<String> messagesConsumed = new BlockingArrayQueue<>(numMsgs);

    Consumer<byte[]> consumer1 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
            .subscriptionType(SubscriptionType.Shared).messageListener((consumer, msg) -> {
                try {
                    consumer.acknowledge(msg);
                    messagesConsumed.add(new String(msg.getData()));
                } catch (Exception e) {
                    fail("Should not fail");
                }
            }).subscribe();

    // consumer2 does not ack messages
    PulsarClient newPulsarClient = newPulsarClient(lookupUrl.toString(), 0);// Creates new client connection
    Consumer<byte[]> consumer2 = newPulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
            .subscriptionType(SubscriptionType.Shared).messageListener((consumer, msg) -> {
                // do notthing
            }).subscribe();

    List<CompletableFuture<MessageId>> futures = Lists.newArrayListWithCapacity(numMsgs * 2);
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    for (int i = 0; i < numMsgs; i++) {
        String message = "msg-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
        messagesProduced.add(message);
    }
    FutureUtil.waitForAll(futures).get();
    producer.close();

    consumer2.close();

    for (int n = 0; n < 10 && messagesConsumed.size() < numMsgs; n++) {
        Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    }

    // 1. consumer1 gets all messages
    assertTrue(CollectionUtils.subtract(messagesProduced, messagesConsumed).isEmpty());

    consumer1.close();
    newPulsarClient.close();

    deleteTopic(topicName);
}
 
Example 14
Source Project: pulsar   File: TopicsConsumerImplTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeOut = testTimeout)
public void testSyncProducerAndConsumer() throws Exception {
    String key = "TopicsConsumerSyncTest";
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 30;

    final String topicName1 = "persistent://prop/use/ns-abc/topic-1-" + key;
    final String topicName2 = "persistent://prop/use/ns-abc/topic-2-" + key;
    final String topicName3 = "persistent://prop/use/ns-abc/topic-3-" + key;
    List<String> topicNames = Lists.newArrayList(topicName1, topicName2, topicName3);

    TenantInfo tenantInfo = createDefaultTenantInfo();
    admin.tenants().createTenant("prop", tenantInfo);
    admin.topics().createPartitionedTopic(topicName2, 2);
    admin.topics().createPartitionedTopic(topicName3, 3);

    // 1. producer connect
    Producer<byte[]> producer1 = pulsarClient.newProducer().topic(topicName1)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();
    Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topicName2)
        .enableBatching(false)
        .messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition)
        .create();
    Producer<byte[]> producer3 = pulsarClient.newProducer().topic(topicName3)
        .enableBatching(false)
        .messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition)
        .create();

    // 2. Create consumer
    Consumer<byte[]> consumer = pulsarClient.newConsumer()
        .topics(topicNames)
        .subscriptionName(subscriptionName)
        .subscriptionType(SubscriptionType.Shared)
        .ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS)
        .receiverQueueSize(4)
        .subscribe();
    assertTrue(consumer instanceof MultiTopicsConsumerImpl);

    // 3. producer publish messages
    for (int i = 0; i < totalMessages / 3; i++) {
        producer1.send((messagePredicate + "producer1-" + i).getBytes());
        producer2.send((messagePredicate + "producer2-" + i).getBytes());
        producer3.send((messagePredicate + "producer3-" + i).getBytes());
    }

    int messageSet = 0;
    Message<byte[]> message = consumer.receive();
    do {
        assertTrue(message instanceof TopicMessageImpl);
        messageSet ++;
        consumer.acknowledge(message);
        log.debug("Consumer acknowledged : " + new String(message.getData()));
        message = consumer.receive(500, TimeUnit.MILLISECONDS);
    } while (message != null);
    assertEquals(messageSet, totalMessages);

    consumer.unsubscribe();
    consumer.close();
    producer1.close();
    producer2.close();
    producer3.close();
}
 
Example 15
Source Project: pulsar   File: PersistentTopicE2ETest.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Verify: 1. Broker should not replay already acknowledged messages 2. Dispatcher should not stuck while
 * dispatching new messages due to previous-replay of invalid/already-acked messages
 *
 * @throws Exception
 */
@Test
public void testMessageReplay() throws Exception {

    final String topicName = "persistent://prop/ns-abc/topic2";
    final String subName = "sub2";

    Message<byte[]> msg;
    int totalMessages = 10;
    int replayIndex = totalMessages / 2;

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
            .subscriptionType(SubscriptionType.Shared).receiverQueueSize(1).subscribe();
    Producer<byte[]> producer = pulsarClient.newProducer()
        .topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();

    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();
    assertNotNull(topicRef);
    PersistentSubscription subRef = topicRef.getSubscription(subName);
    PersistentDispatcherMultipleConsumers dispatcher = (PersistentDispatcherMultipleConsumers) subRef
            .getDispatcher();
    Field replayMap = PersistentDispatcherMultipleConsumers.class.getDeclaredField("messagesToRedeliver");
    replayMap.setAccessible(true);
    ConcurrentLongPairSet messagesToReplay = new ConcurrentLongPairSet(64, 1);

    assertNotNull(subRef);

    // (1) Produce messages
    for (int i = 0; i < totalMessages; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }

    MessageIdImpl firstAckedMsg = null;
    // (2) Consume and ack messages except first message
    for (int i = 0; i < totalMessages; i++) {
        msg = consumer.receive();
        consumer.acknowledge(msg);
        MessageIdImpl msgId = (MessageIdImpl) msg.getMessageId();
        if (i == 0) {
            firstAckedMsg = msgId;
        }
        if (i < replayIndex) {
            // (3) accumulate acked messages for replay
            messagesToReplay.add(msgId.getLedgerId(), msgId.getEntryId());
        }
    }

    // (4) redelivery : should redeliver only unacked messages
    Thread.sleep(1000);

    replayMap.set(dispatcher, messagesToReplay);
    // (a) redelivery with all acked-message should clear messageReply bucket
    dispatcher.redeliverUnacknowledgedMessages(dispatcher.getConsumers().get(0));
    assertEquals(messagesToReplay.size(), 0);

    // (b) fill messageReplyBucket with already acked entry again: and try to publish new msg and read it
    messagesToReplay.add(firstAckedMsg.getLedgerId(), firstAckedMsg.getEntryId());
    replayMap.set(dispatcher, messagesToReplay);
    // send new message
    final String testMsg = "testMsg";
    producer.send(testMsg.getBytes());
    // consumer should be able to receive only new message and not the
    dispatcher.consumerFlow(dispatcher.getConsumers().get(0), 1);
    msg = consumer.receive(1, TimeUnit.SECONDS);
    assertNotNull(msg);
    assertEquals(msg.getData(), testMsg.getBytes());

    consumer.close();
    producer.close();
}
 
Example 16
@Test(dataProvider =  "AllCheckSchemaCompatibilityStrategy")
public void testIsAutoUpdateSchema(SchemaCompatibilityStrategy schemaCompatibilityStrategy) throws Exception {
    final String tenant = PUBLIC_TENANT;
    final String topic = "test-consumer-compatibility";

    String namespace = "test-namespace-" + randomName(16);
    String fqtn = TopicName.get(
            TopicDomain.persistent.value(),
            tenant,
            namespace,
            topic
    ).toString();

    NamespaceName namespaceName = NamespaceName.get(tenant, namespace);

    admin.namespaces().createNamespace(
            tenant + "/" + namespace,
            Sets.newHashSet(CLUSTER_NAME)
    );

    assertEquals(admin.namespaces().getSchemaCompatibilityStrategy(namespaceName.toString()),
            SchemaCompatibilityStrategy.FULL);
    
    admin.namespaces().setSchemaCompatibilityStrategy(namespaceName.toString(), schemaCompatibilityStrategy);
    admin.schemas().createSchema(fqtn, Schema.AVRO(Schemas.PersonOne.class).getSchemaInfo());

    admin.namespaces().setIsAllowAutoUpdateSchema(namespaceName.toString(), false);
    ProducerBuilder<Schemas.PersonTwo> producerThreeBuilder = pulsarClient
            .newProducer(Schema.AVRO(SchemaDefinition.<Schemas.PersonTwo>builder().withAlwaysAllowNull
                    (false).withSupportSchemaVersioning(true).
                    withPojo(Schemas.PersonTwo.class).build()))
            .topic(fqtn);
    try {
        producerThreeBuilder.create();
    } catch (Exception e) {
        Assert.assertTrue(e.getMessage().contains("Schema not found and schema auto updating is disabled."));
    }

    admin.namespaces().setIsAllowAutoUpdateSchema(namespaceName.toString(), true);
    ConsumerBuilder<Schemas.PersonTwo> comsumerBuilder = pulsarClient.newConsumer(Schema.AVRO(
            SchemaDefinition.<Schemas.PersonTwo>builder().withAlwaysAllowNull
                    (false).withSupportSchemaVersioning(true).
                    withPojo(Schemas.PersonTwo.class).build()))
            .subscriptionName("test")
            .topic(fqtn);

    Producer<Schemas.PersonTwo> producer = producerThreeBuilder.create();
    Consumer<Schemas.PersonTwo> consumerTwo = comsumerBuilder.subscribe();

    producer.send(new Schemas.PersonTwo(2, "Lucy"));
    Message<Schemas.PersonTwo> message = consumerTwo.receive();

    Schemas.PersonTwo personTwo = message.getValue();
    consumerTwo.acknowledge(message);

    assertEquals(personTwo.getId(), 2);
    assertEquals(personTwo.getName(), "Lucy");

    producer.close();
    consumerTwo.close();

    admin.namespaces().setIsAllowAutoUpdateSchema(namespaceName.toString(), false);

    producer = producerThreeBuilder.create();
    consumerTwo = comsumerBuilder.subscribe();

    producer.send(new Schemas.PersonTwo(2, "Lucy"));
    message = consumerTwo.receive();

    personTwo = message.getValue();
    consumerTwo.acknowledge(message);

    assertEquals(personTwo.getId(), 2);
    assertEquals(personTwo.getName(), "Lucy");

    consumerTwo.close();
    producer.close();
}
 
Example 17
Source Project: pulsar   File: TopicsConsumerImplTest.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Test topic partitions auto subscribed.
 *
 * Steps:
 * 1. Create a consumer with 2 topics, and each topic has 2 partitions: xx-partition-0, xx-partition-1.
 * 2. update topics to have 3 partitions.
 * 3. trigger partitionsAutoUpdate. this should be done automatically, this is to save time to manually trigger.
 * 4. produce message to xx-partition-2 again,  and verify consumer could receive message.
 *
 */
@Test(timeOut = 30000)
public void testTopicAutoUpdatePartitions() throws Exception {
    String key = "TestTopicAutoUpdatePartitions";
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 6;

    final String topicName1 = "persistent://prop/use/ns-abc/topic-1-" + key;
    final String topicName2 = "persistent://prop/use/ns-abc/topic-2-" + key;
    List<String> topicNames = Lists.newArrayList(topicName1, topicName2);

    TenantInfo tenantInfo = createDefaultTenantInfo();
    admin.tenants().createTenant("prop", tenantInfo);
    admin.topics().createPartitionedTopic(topicName1, 2);
    admin.topics().createPartitionedTopic(topicName2, 2);

    // 1. Create a  consumer
    Consumer<byte[]> consumer = pulsarClient.newConsumer()
            .topics(topicNames)
            .subscriptionName(subscriptionName)
            .subscriptionType(SubscriptionType.Shared)
            .ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS)
            .receiverQueueSize(4)
            .autoUpdatePartitions(true)
            .subscribe();
    assertTrue(consumer instanceof MultiTopicsConsumerImpl);

    MultiTopicsConsumerImpl topicsConsumer = (MultiTopicsConsumerImpl) consumer;

    // 2. update to 3 partitions
    admin.topics().updatePartitionedTopic(topicName1, 3);
    admin.topics().updatePartitionedTopic(topicName2, 3);

    // 3. trigger partitionsAutoUpdate. this should be done automatically in 1 minutes,
    // this is to save time to manually trigger.
    log.info("trigger partitionsAutoUpdateTimerTask");
    Timeout timeout = topicsConsumer.getPartitionsAutoUpdateTimeout();
    timeout.task().run(timeout);
    Thread.sleep(200);

    // 4. produce message to xx-partition-2,  and verify consumer could receive message.
    Producer<byte[]> producer1 = pulsarClient.newProducer().topic(topicName1 + "-partition-2")
            .enableBatching(false)
            .create();
    Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topicName2 + "-partition-2")
            .enableBatching(false)
            .create();
    for (int i = 0; i < totalMessages; i++) {
        producer1.send((messagePredicate + "topic1-partition-2 index:" + i).getBytes());
        producer2.send((messagePredicate + "topic2-partition-2 index:" + i).getBytes());
        log.info("produce message to partition-2 again. messageindex: {}", i);
    }
    int messageSet = 0;
    Message<byte[]> message = consumer.receive();
    do {
        messageSet ++;
        consumer.acknowledge(message);
        log.info("4 Consumer acknowledged : " + new String(message.getData()));
        message = consumer.receive(200, TimeUnit.MILLISECONDS);
    } while (message != null);
    assertEquals(messageSet, 2 * totalMessages);

    consumer.close();
}
 
Example 18
Source Project: pulsar   File: V1_AdminApiTest2.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testPublishConsumerStats() throws Exception {
    final String topicName = "statTopic";
    final String subscriberName = topicName + "-my-sub-1";
    final String topic = "persistent://prop-xyz/use/ns1/" + topicName;
    final String producerName = "myProducer";

    @Cleanup
    PulsarClient client = PulsarClient.builder().serviceUrl(pulsar.getWebServiceAddress()).build();
    Consumer<byte[]> consumer = client.newConsumer().topic(topic).subscriptionName(subscriberName)
            .subscriptionType(SubscriptionType.Shared).subscribe();
    Producer<byte[]> producer = client.newProducer()
        .topic(topic)
        .producerName(producerName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();

    retryStrategically((test) -> {
        TopicStats stats;
        try {
            stats = admin.topics().getStats(topic);
            return stats.publishers.size() > 0 && stats.subscriptions.get(subscriberName) != null
                    && stats.subscriptions.get(subscriberName).consumers.size() > 0;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 5, 200);

    TopicStats topicStats = admin.topics().getStats(topic);
    assertEquals(topicStats.publishers.size(), 1);
    assertNotNull(topicStats.publishers.get(0).getAddress());
    assertNotNull(topicStats.publishers.get(0).getClientVersion());
    assertNotNull(topicStats.publishers.get(0).getConnectedSince());
    assertNotNull(topicStats.publishers.get(0).getProducerName());
    assertEquals(topicStats.publishers.get(0).getProducerName(), producerName);

    SubscriptionStats subscriber = topicStats.subscriptions.get(subscriberName);
    assertNotNull(subscriber);
    assertEquals(subscriber.consumers.size(), 1);
    ConsumerStats consumerStats = subscriber.consumers.get(0);
    assertNotNull(consumerStats.getAddress());
    assertNotNull(consumerStats.getClientVersion());
    assertNotNull(consumerStats.getConnectedSince());

    producer.close();
    consumer.close();
}
 
Example 19
@Test
public void testBatchMessageIndexAckForExclusiveSubscription() throws PulsarClientException, ExecutionException, InterruptedException {
    final String topic = "testBatchMessageIndexAckForExclusiveSubscription";

    @Cleanup
    Consumer<Integer> consumer = pulsarClient.newConsumer(Schema.INT32)
        .topic(topic)
        .subscriptionName("sub")
        .receiverQueueSize(100)
        .subscribe();

    @Cleanup
    Producer<Integer> producer = pulsarClient.newProducer(Schema.INT32)
        .topic(topic)
        .batchingMaxPublishDelay(50, TimeUnit.MILLISECONDS)
        .create();

    final int messages = 100;
    List<CompletableFuture<MessageId>> futures = new ArrayList<>(messages);
    for (int i = 0; i < messages; i++) {
        futures.add(producer.sendAsync(i));
    }
    FutureUtil.waitForAll(futures).get();

    for (int i = 0; i < messages; i++) {
        if (i == 49) {
            consumer.acknowledgeCumulative(consumer.receive());
        } else {
            consumer.receive();
        }
    }

    //Wait ack send.
    Thread.sleep(1000);
    consumer.close();
    consumer = pulsarClient.newConsumer(Schema.INT32)
        .topic(topic)
        .subscriptionName("sub")
        .receiverQueueSize(100)
        .subscribe();

    List<Message<Integer>> received = new ArrayList<>(100);
    for (int i = 0; i < messages; i++) {
        received.add(consumer.receive());
    }

    Assert.assertEquals(received.size(), 100);
}
 
Example 20
Source Project: pulsar   File: SchemaTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testMultiTopicSetSchemaProvider() throws Exception {
    final String tenant = PUBLIC_TENANT;
    final String namespace = "test-namespace-" + randomName(16);
    final String topicOne = "test-multi-version-schema-one";
    final String topicTwo = "test-multi-version-schema-two";
    final String fqtnOne = TopicName.get(
            TopicDomain.persistent.value(),
            tenant,
            namespace,
            topicOne
    ).toString();

    final String fqtnTwo = TopicName.get(
            TopicDomain.persistent.value(),
            tenant,
            namespace,
            topicTwo
    ).toString();


    admin.namespaces().createNamespace(
            tenant + "/" + namespace,
            Sets.newHashSet(CLUSTER_NAME)
    );

    admin.topics().createPartitionedTopic(fqtnOne, 3);
    admin.topics().createPartitionedTopic(fqtnTwo, 3);

    admin.schemas().createSchema(fqtnOne, Schema.AVRO(
            SchemaDefinition.<Schemas.PersonOne>builder().withAlwaysAllowNull
                    (false).withSupportSchemaVersioning(true).
                    withPojo(Schemas.PersonOne.class).build()).getSchemaInfo());

    admin.schemas().createSchema(fqtnOne, Schema.AVRO(
            SchemaDefinition.<Schemas.PersonTwo>builder().withAlwaysAllowNull
                    (false).withSupportSchemaVersioning(true).
                    withPojo(Schemas.PersonTwo.class).build()).getSchemaInfo());

    admin.schemas().createSchema(fqtnTwo, Schema.AVRO(
            SchemaDefinition.<Schemas.PersonTwo>builder().withAlwaysAllowNull
                    (false).withSupportSchemaVersioning(true).
                    withPojo(Schemas.PersonTwo.class).build()).getSchemaInfo());

    Producer<Schemas.PersonTwo> producer = pulsarClient.newProducer(Schema.AVRO(
            SchemaDefinition.<Schemas.PersonTwo>builder().withAlwaysAllowNull
                    (false).withSupportSchemaVersioning(true).
                    withPojo(Schemas.PersonTwo.class).build()))
            .topic(fqtnOne)
            .create();

    Schemas.PersonTwo personTwo = new Schemas.PersonTwo();
    personTwo.setId(1);
    personTwo.setName("Tom");


    Consumer<Schemas.PersonTwo> consumer = pulsarClient.newConsumer(Schema.AVRO(
            SchemaDefinition.<Schemas.PersonTwo>builder().withAlwaysAllowNull
                    (false).withSupportSchemaVersioning(true).
                    withPojo(Schemas.PersonTwo.class).build()))
            .subscriptionName("test")
            .topic(fqtnOne, fqtnTwo)
            .subscribe();

    producer.send(personTwo);

    Schemas.PersonTwo personConsume = consumer.receive().getValue();
    assertEquals("Tom", personConsume.getName());
    assertEquals(1, personConsume.getId());

    producer.close();
    consumer.close();
}