Java Code Examples for org.apache.pulsar.client.api.Consumer#redeliverUnacknowledgedMessages()

The following examples show how to use org.apache.pulsar.client.api.Consumer#redeliverUnacknowledgedMessages() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: pulsar   File: BrokerServiceTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testBrokerServicePersistentRedeliverTopicStats() throws Exception {
    final String topicName = "persistent://prop/ns-abc/successSharedTopic";
    final String subName = "successSharedSub";

    TopicStats stats;
    SubscriptionStats subStats;

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
            .subscriptionType(SubscriptionType.Shared).acknowledgmentGroupTime(0, TimeUnit.SECONDS).subscribe();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);

    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();
    assertNotNull(topicRef);

    rolloverPerIntervalStats();
    stats = topicRef.getStats(false);
    subStats = stats.subscriptions.values().iterator().next();

    // subscription stats
    assertEquals(stats.subscriptions.keySet().size(), 1);
    assertEquals(subStats.msgBacklog, 0);
    assertEquals(subStats.consumers.size(), 1);

    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);

    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);

    rolloverPerIntervalStats();
    stats = topicRef.getStats(false);
    subStats = stats.subscriptions.values().iterator().next();

    // publisher stats
    assertEquals(subStats.msgBacklog, 10);
    assertEquals(stats.publishers.size(), 1);
    assertTrue(stats.publishers.get(0).msgRateIn > 0.0);
    assertTrue(stats.publishers.get(0).msgThroughputIn > 0.0);
    assertTrue(stats.publishers.get(0).averageMsgSize > 0.0);

    // aggregated publish stats
    assertEquals(stats.msgRateIn, stats.publishers.get(0).msgRateIn);
    assertEquals(stats.msgThroughputIn, stats.publishers.get(0).msgThroughputIn);
    double diff = stats.averageMsgSize - stats.publishers.get(0).averageMsgSize;
    assertTrue(Math.abs(diff) < 0.000001);

    // consumer stats
    assertTrue(subStats.consumers.get(0).msgRateOut > 0.0);
    assertTrue(subStats.consumers.get(0).msgThroughputOut > 0.0);
    assertEquals(subStats.msgRateRedeliver, 0.0);
    assertEquals(subStats.consumers.get(0).unackedMessages, 10);

    // aggregated consumer stats
    assertEquals(subStats.msgRateOut, subStats.consumers.get(0).msgRateOut);
    assertEquals(subStats.msgThroughputOut, subStats.consumers.get(0).msgThroughputOut);
    assertEquals(subStats.msgRateRedeliver, subStats.consumers.get(0).msgRateRedeliver);
    assertEquals(stats.msgRateOut, subStats.consumers.get(0).msgRateOut);
    assertEquals(stats.msgThroughputOut, subStats.consumers.get(0).msgThroughputOut);
    assertEquals(subStats.msgRateRedeliver, subStats.consumers.get(0).msgRateRedeliver);
    assertEquals(subStats.unackedMessages, subStats.consumers.get(0).unackedMessages);

    consumer.redeliverUnacknowledgedMessages();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);

    rolloverPerIntervalStats();
    stats = topicRef.getStats(false);
    subStats = stats.subscriptions.values().iterator().next();
    assertTrue(subStats.msgRateRedeliver > 0.0);
    assertEquals(subStats.msgRateRedeliver, subStats.consumers.get(0).msgRateRedeliver);

    Message<byte[]> msg;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive();
        consumer.acknowledge(msg);
    }
    consumer.close();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);

    rolloverPerIntervalStats();
    stats = topicRef.getStats(false);
    subStats = stats.subscriptions.values().iterator().next();

    assertEquals(subStats.msgBacklog, 0);
}
 
Example 2
Source Project: pulsar   File: PersistentTopicE2ETest.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Verify: Broker should not replay already acknowledged messages again and should clear them from messageReplay
 * bucket
 *
 * 1. produce messages 2. consume messages and ack all except 1 msg 3. Verification: should replay only 1 unacked
 * message
 */
@Test
public void testMessageRedelivery() throws Exception {
    final String topicName = "persistent://prop/ns-abc/topic2";
    final String subName = "sub2";

    Message<String> msg;
    List<Message<String>> unackedMessages = new ArrayList<>();
    int totalMessages = 20;

    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING)
            .topic(topicName)
            .subscriptionName(subName)
            .subscriptionType(SubscriptionType.Shared)
            .acknowledgmentGroupTime(0, TimeUnit.SECONDS)
            .subscribe();
    Producer<String> producer = pulsarClient.newProducer(Schema.STRING)
            .topic(topicName)
            .enableBatching(false)
            .messageRoutingMode(MessageRoutingMode.SinglePartition)
            .create();

    // (1) Produce messages
    for (int i = 0; i < totalMessages; i++) {
        producer.send("my-message-" + i);
    }

    // (2) Consume and only ack last 10 messages
    for (int i = 0; i < totalMessages; i++) {
        msg = consumer.receive();
        if (i >= 10) {
            unackedMessages.add(msg);
        } else {
            consumer.acknowledge(msg);
        }
    }

    consumer.redeliverUnacknowledgedMessages();

    for (int i = 0; i < 10; i++) {
        // Verify: msg [L:0] must be redelivered
        try {
            final Message<String> redeliveredMsg = consumer.receive(1, TimeUnit.SECONDS);
            unackedMessages.removeIf(unackedMessage -> unackedMessage.getValue().equals(redeliveredMsg.getValue()));
        } catch (Exception e) {
            fail("msg should be redelivered ", e);
        }
    }
    // Make sure that first 10 messages that we didn't acknowledge get redelivered.
    assertEquals(unackedMessages.size(), 0);

    // Verify no other messages are redelivered
    msg = consumer.receive(100, TimeUnit.MILLISECONDS);
    assertNull(msg);

    consumer.close();
    producer.close();
}
 
Example 3
Source Project: pulsar   File: ResendRequestTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeOut = testTimeout)
public void testSharedSingleAckedNormalTopic() throws Exception {
    String key = "testSharedSingleAckedNormalTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-shared-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    // 1. producer connect
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);

    // 2. Create consumer
    Consumer<byte[]> consumer1 = pulsarClient.newConsumer().topic(topicName)
            .subscriptionName(subscriptionName).receiverQueueSize(totalMessages / 2)
            .subscriptionType(SubscriptionType.Shared).subscribe();

    PulsarClient newPulsarClient = newPulsarClient(lookupUrl.toString(), 0);// Creates new client connection
    Consumer<byte[]> consumer2 = newPulsarClient.newConsumer().topic(topicName)
            .subscriptionName(subscriptionName).receiverQueueSize(totalMessages / 2)
            .subscriptionType(SubscriptionType.Shared).subscribe();

    // 3. Producer publishes messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
        log.info("Producer produced " + message);
    }

    // 4. Receive messages
    int receivedConsumer1 = 0, receivedConsumer2 = 0;
    Message<byte[]> message1 = consumer1.receive();
    Message<byte[]> message2 = consumer2.receive();
    do {
        if (message1 != null) {
            log.info("Consumer 1 Received: " + new String(message1.getData()));
            receivedConsumer1 += 1;
        }

        if (message2 != null) {
            log.info("Consumer 2 Received: " + new String(message2.getData()));
            receivedConsumer2 += 1;
        }
        message1 = consumer1.receive(100, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(100, TimeUnit.MILLISECONDS);
    } while (message1 != null || message2 != null);

    log.info("Consumer 1 receives = " + receivedConsumer1);
    log.info("Consumer 2 receives = " + receivedConsumer2);
    log.info("Total receives = " + (receivedConsumer2 + receivedConsumer1));
    assertEquals(receivedConsumer2 + receivedConsumer1, totalMessages);

    // 5. Send a resend request from Consumer 1
    log.info("Consumer 1 sent a resend request");
    consumer1.redeliverUnacknowledgedMessages();

    // 6. Consumer 1's unAcked messages should be sent to Consumer 1 or 2
    int receivedMessagesAfterRedelivery = 0;
    receivedConsumer1 = 0;
    message1 = consumer1.receive(100, TimeUnit.MILLISECONDS);
    message2 = consumer2.receive(100, TimeUnit.MILLISECONDS);
    do {
        if (message1 != null) {
            log.info("Consumer 1 Received: " + new String(message1.getData()));
            receivedConsumer1 += 1;
            ++receivedMessagesAfterRedelivery;
        }

        if (message2 != null) {
            log.info("Consumer 2 Received: " + new String(message2.getData()));
            receivedConsumer2 += 1;
            ++receivedMessagesAfterRedelivery;
        }
        message1 = consumer1.receive(200, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(200, TimeUnit.MILLISECONDS);
    } while (message1 != null || message2 != null);
    log.info("Additional received = " + receivedMessagesAfterRedelivery);
    newPulsarClient.close();
    assertTrue(receivedMessagesAfterRedelivery > 0);

    assertEquals(receivedConsumer1 + receivedConsumer2, totalMessages);
}
 
Example 4
Source Project: pulsar   File: ResendRequestTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeOut = testTimeout)
public void testSharedSingleAckedPartitionedTopic() throws Exception {
    String key = "testSharedSingleAckedPartitionedTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-shared-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    final int numberOfPartitions = 3;
    TenantInfo tenantInfo = createDefaultTenantInfo();
    admin.tenants().createTenant("prop", tenantInfo);
    admin.topics().createPartitionedTopic(topicName, numberOfPartitions);
    Random rn = new Random();
    // Special step to create partitioned topic

    // 1. producer connect
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();

    // 2. Create consumer
    Consumer<byte[]> consumer1 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
            .receiverQueueSize(7).subscriptionType(SubscriptionType.Shared).subscribe();

    PulsarClient newPulsarClient = newPulsarClient(lookupUrl.toString(), 0);// Creates new client connection
    Consumer<byte[]> consumer2 = newPulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
            .receiverQueueSize(7).subscriptionType(SubscriptionType.Shared).subscribe();

    // 3. producer publish messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        log.info("Message produced: " + message);
        producer.send(message.getBytes());
    }

    // 4. Receive messages
    Message<byte[]> message1 = consumer1.receive();
    Message<byte[]> message2 = consumer2.receive();
    int messageCount1 = 0;
    int messageCount2 = 0;
    int ackCount1 = 0;
    int ackCount2 = 0;
    do {
        if (message1 != null) {
            log.info("Consumer1 received " + new String(message1.getData()));
            messageCount1 += 1;
            if (rn.nextInt() % 3 == 0) {
                consumer1.acknowledge(message1);
                log.info("Consumer1 acked " + new String(message1.getData()));
                ackCount1 += 1;
            }
        }
        if (message2 != null) {
            log.info("Consumer2 received " + new String(message2.getData()));
            messageCount2 += 1;
            if (rn.nextInt() % 3 == 0) {
                consumer2.acknowledge(message2);
                log.info("Consumer2 acked " + new String(message2.getData()));
                ackCount2 += 1;
            }
        }
        message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
    } while (message1 != null || message2 != null);
    log.info(key + " messageCount1 = " + messageCount1);
    log.info(key + " messageCount2 = " + messageCount2);
    log.info(key + " ackCount1 = " + ackCount1);
    log.info(key + " ackCount2 = " + ackCount2);
    assertEquals(messageCount1 + messageCount2, totalMessages);

    // 5. Ask for redeliver
    log.info(key + ": Sent a Redeliver Message Request");
    consumer1.redeliverUnacknowledgedMessages();
    if ((ackCount1 + ackCount2) == totalMessages) {
        return;
    }

    // 6. Check if Messages redelivered again
    message1 = consumer1.receive(5000, TimeUnit.MILLISECONDS);
    message2 = consumer2.receive(5000, TimeUnit.MILLISECONDS);
    messageCount1 = 0;
    do {
        if (message1 != null) {
            log.info("Consumer1 received " + new String(message1.getData()));
            messageCount1 += 1;
        }
        if (message2 != null) {
            log.info("Consumer2 received " + new String(message2.getData()));
            messageCount2 += 1;
        }
        message1 = consumer1.receive(1000, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(1000, TimeUnit.MILLISECONDS);
    } while (message1 != null || message2 != null);

    log.info(key + " messageCount1 = " + messageCount1);
    log.info(key + " messageCount2 = " + messageCount2);
    log.info(key + " ackCount1 = " + ackCount1);
    log.info(key + " ackCount2 = " + ackCount2);
    newPulsarClient.close();
    assertEquals(messageCount1 + messageCount2 + ackCount1, totalMessages);
}
 
Example 5
Source Project: pulsar   File: ResendRequestTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeOut = testTimeout)
public void testFailoverSingleAckedPartitionedTopic() throws Exception {
    String key = "testFailoverSingleAckedPartitionedTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-failover-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    final int numberOfPartitions = 3;
    TenantInfo tenantInfo = createDefaultTenantInfo();
    admin.tenants().createTenant("prop", tenantInfo);
    admin.topics().createPartitionedTopic(topicName, numberOfPartitions);
    Random rn = new Random();
    // Special step to create partitioned topic

    // 1. producer connect
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.RoundRobinPartition)
        .create();

    // 2. Create consumer
    ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName)
            .subscriptionName(subscriptionName).receiverQueueSize(7).subscriptionType(SubscriptionType.Failover);
    Consumer<byte[]> consumer1 = consumerBuilder.clone().consumerName("Consumer-1").subscribe();
    Consumer<byte[]> consumer2 = consumerBuilder.clone().consumerName("Consumer-2").subscribe();

    Thread.sleep(1000);
    // 3. producer publish messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        log.info("Message produced: " + message);
        producer.send(message.getBytes());
    }

    // 4. Receive messages
    Message<byte[]> message1 = consumer1.receive();
    Message<byte[]> message2 = consumer2.receive();
    int messageCount1 = 0;
    int messageCount2 = 0;
    int ackCount1 = 0;
    int ackCount2 = 0;
    do {
        if (message1 != null) {
            log.info("Consumer1 received " + new String(message1.getData()));
            messageCount1 += 1;
            if (rn.nextInt() % 3 == 0) {
                consumer1.acknowledge(message1);
                ackCount1 += 1;
            }
        }
        if (message2 != null) {
            log.info("Consumer2 received " + new String(message2.getData()));
            messageCount2 += 1;
            if (rn.nextInt() % 3 == 0) {
                consumer2.acknowledge(message2);
                ackCount2 += 1;
            }
        }
        message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
    } while (message1 != null || message2 != null);
    log.info(key + " messageCount1 = " + messageCount1);
    log.info(key + " messageCount2 = " + messageCount2);
    log.info(key + " ackCount1 = " + ackCount1);
    log.info(key + " ackCount2 = " + ackCount2);
    assertEquals(messageCount1 + messageCount2, totalMessages);
    if ((ackCount1 + ackCount2) == totalMessages) {
        return;
    }
    // 5. Ask for redeliver
    log.info(key + ": Sent a Redeliver Message Request");
    consumer1.redeliverUnacknowledgedMessages();
    consumer1.close();

    // 6. Check if Messages redelivered again
    message2 = consumer2.receive();
    messageCount1 = 0;
    do {
        if (message2 != null) {
            log.info("Consumer2 received " + new String(message2.getData()));
            messageCount2 += 1;
        }
        message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
    } while (message1 != null || message2 != null);
    log.info(key + " messageCount1 = " + messageCount1);
    log.info(key + " messageCount2 = " + messageCount2);
    log.info(key + " ackCount1 = " + ackCount1);
    log.info(key + " ackCount2 = " + ackCount2);
    assertEquals(messageCount2 + ackCount1, totalMessages);
}
 
Example 6
Source Project: pulsar   File: ResendRequestTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeOut = testTimeout)
public void testFailoverInactiveConsumer() throws Exception {
    String key = "testFailoverInactiveConsumer";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-failover-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    // 1. producer connect
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);

    // 2. Create consumer
    ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName)
            .subscriptionName(subscriptionName).receiverQueueSize(10).subscriptionType(SubscriptionType.Failover);
    Consumer<byte[]> consumer1 = consumerBuilder.clone().consumerName("Consumer-1").subscribe();
    Consumer<byte[]> consumer2 = consumerBuilder.clone().consumerName("Consumer-2").subscribe();

    // 3. Producer publishes messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
        log.info("Producer produced " + message);
    }

    // 4. Receive messages
    int receivedConsumer1 = 0, receivedConsumer2 = 0;
    Message<byte[]> message1;
    Message<byte[]> message2;
    do {
        message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
        if (message1 != null) {
            log.info("Consumer 1 Received: " + new String(message1.getData()));
            receivedConsumer1 += 1;
        }
    } while (message1 != null);
    log.info("Consumer 1 receives = " + receivedConsumer1);
    log.info("Consumer 2 receives = " + receivedConsumer2);
    log.info("Total receives = " + (receivedConsumer2 + receivedConsumer1));
    assertEquals(receivedConsumer2 + receivedConsumer1, totalMessages);
    // Consumer 2 is on Stand By
    assertEquals(receivedConsumer2, 0);

    // 5. Consumer 2 asks for a redelivery but the request is ignored
    log.info("Consumer 2 asks for resend");
    consumer2.redeliverUnacknowledgedMessages();
    Thread.sleep(1000);

    message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
    message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
    assertNull(message1);
    assertNull(message2);
}