Java Code Examples for org.apache.pulsar.client.api.Consumer#acknowledgeCumulative()

The following examples show how to use org.apache.pulsar.client.api.Consumer#acknowledgeCumulative() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
/**
 * <pre>
 * Overriden method: OffsetCommitResponse commitOffsets(OffsetCommitRequest request)
 * 
 * Note:
 * created PulsarOffsetCommitResponse as OffsetCommitRequest doesn't provide getters
 * 
 * </pre>
 */
public OffsetCommitResponse commitOffsets(PulsarOffsetCommitRequest request) {

    PulsarOffsetCommitResponse response = new PulsarOffsetCommitResponse(null);
    for (Entry<String, MessageId> topicOffset : request.getTopicOffsetMap().entrySet()) {
        final String topic = topicOffset.getKey();
        final String groupId = request.getGroupId();
        try {
            Consumer<byte[]> consumer = getConsumer(topic, groupId);
            consumer.acknowledgeCumulative(topicOffset.getValue());
        } catch (Exception e) {
            log.warn("Failed to ack message for topic {}-{}", topic, topicOffset.getValue(), e);
            response.hasError = true;
            TopicAndPartition topicPartition = new TopicAndPartition(topic, 0);
            response.errors.computeIfAbsent(topicPartition, tp -> ErrorMapping.UnknownCode());
        }
    }

    return response;
}
 
Example 2
Source Project: pulsar   File: AdminApiTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testBacklogSizeShouldBeZeroWhenConsumerAckedAllMessages() throws Exception {
    final String topic = "persistent://prop-xyz/ns1/testBacklogSizeShouldBeZeroWhenConsumerAckedAllMessages";
    Consumer<byte[]> consumer = pulsarClient.newConsumer()
            .topic(topic)
            .subscriptionName("sub-1")
            .subscribe();
    Producer<byte[]> producer = pulsarClient.newProducer()
            .topic(topic)
            .create();

    final int messages = 33;
    for (int i = 0; i < messages; i++) {
        producer.send(new byte[1024 * i * 5]);
    }

    for (int i = 0; i < messages; i++) {
        consumer.acknowledgeCumulative(consumer.receive());
    }

    // Wait ack send
    Thread.sleep(1000);

    TopicStats topicStats = admin.topics().getStats(topic);
    assertEquals(topicStats.backlogSize, 0);
}
 
Example 3
Source Project: pulsar   File: SampleConsumerWithSchema.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws PulsarClientException, JsonProcessingException {

        PulsarClient pulsarClient = PulsarClient.builder().serviceUrl("http://localhost:8080").build();

        Consumer<JsonPojo> consumer = pulsarClient.newConsumer(JSONSchema.of
                (SchemaDefinition.<JsonPojo>builder().withPojo(JsonPojo.class).build())) //
                .topic("persistent://my-property/use/my-ns/my-topic") //
                .subscriptionName("my-subscription-name").subscribe();

        Message<JsonPojo> msg = null;

        for (int i = 0; i < 100; i++) {
            msg = consumer.receive();
            // do something
            System.out.println("Received: " + msg.getValue().content);
        }

        // Acknowledge the consumption of all messages at once
        consumer.acknowledgeCumulative(msg);
        pulsarClient.close();
    }
 
Example 4
Source Project: pulsar   File: SampleConsumer.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws PulsarClientException, InterruptedException {

        PulsarClient pulsarClient = PulsarClient.builder().serviceUrl("pulsar://localhost:6650").build();

        Consumer<byte[]> consumer = pulsarClient.newConsumer() //
                .topic("persistent://my-tenant/my-ns/my-topic") //
                .subscriptionName("my-subscription-name").subscribe();

        Message<byte[]> msg = null;

        for (int i = 0; i < 100; i++) {
            msg = consumer.receive();
            // do something
            System.out.println("Received: " + new String(msg.getData()));
        }

        // Acknowledge the consumption of all messages at once
        try {
            consumer.acknowledgeCumulative(msg);
        } catch (Exception e) {
            consumer.reconsumeLater(msg, 10, TimeUnit.SECONDS);
        }
       
        pulsarClient.close();
    }
 
Example 5
Source Project: pulsar   File: V1_ProducerConsumerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test(dataProvider = "batch")
public void testSyncProducerAndConsumer(int batchMessageDelayMs) throws Exception {
    log.info("-- Starting {} test --", methodName);

    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING)
                .topic("persistent://my-property/use/my-ns/my-topic1")
                .subscriptionName("my-subscriber-name")
                .subscriptionType(SubscriptionType.Exclusive)
                .subscribe();

    ProducerBuilder<String> producerBuilder = pulsarClient.newProducer(Schema.STRING)
            .topic("persistent://my-property/use/my-ns/my-topic1");

    if (batchMessageDelayMs != 0) {
        producerBuilder.enableBatching(true)
            .batchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS)
            .batchingMaxMessages(5);
    } else {
        producerBuilder.enableBatching(false);
    }

    Producer<String> producer = producerBuilder.create();
    for (int i = 0; i < 10; i++) {
        producer.send("my-message-" + i);
    }

    Message<String> msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = msg.getValue();
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 
Example 6
Source Project: pulsar   File: V1_ProducerConsumerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testEnabledChecksumClient() throws Exception {
    log.info("-- Starting {} test --", methodName);

    final int totalMsg = 10;

    Consumer<byte[]> consumer = pulsarClient.newConsumer()
            .topic("persistent://my-property/use/my-ns/my-topic1")
            .subscriptionName("my-subscriber-name")
            .subscribe();

    final int batchMessageDelayMs = 300;
    Producer<byte[]> producer = pulsarClient.newProducer()
            .topic("persistent://my-property/use/my-ns/my-topic1")
            .enableBatching(true)
            .batchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS)
            .batchingMaxMessages(5)
            .create();

    for (int i = 0; i < totalMsg; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }

    Message<byte[]>msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < totalMsg; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 
Example 7
/**
 * verifies that messages whose size is larger than 2^14 bytes (max size of single TLS chunk) can be
 * produced/consumed
 *
 * @throws Exception
 */
@Test(timeOut = 30000)
public void testTlsLargeSizeMessage() throws Exception {
    log.info("-- Starting {} test --", methodName);

    final int MESSAGE_SIZE = 16 * 1024 + 1;
    log.info("-- message size --", MESSAGE_SIZE);
    String topicName = "persistent://my-property/use/my-ns/testTlsLargeSizeMessage"
                       + System.currentTimeMillis();

    internalSetUpForClient(true, pulsar.getBrokerServiceUrlTls());
    internalSetUpForNamespace();

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName)
            .subscriptionName("my-subscriber-name").subscribe();

    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
            .create();
    for (int i = 0; i < 10; i++) {
        byte[] message = new byte[MESSAGE_SIZE];
        Arrays.fill(message, (byte) i);
        producer.send(message);
    }

    Message<byte[]> msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        byte[] expected = new byte[MESSAGE_SIZE];
        Arrays.fill(expected, (byte) i);
        Assert.assertEquals(expected, msg.getData());
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 
Example 8
/**
 * verifies that messages whose size is larger than 2^14 bytes (max size of single TLS chunk) can be
 * produced/consumed
 *
 * @throws Exception
 */
@Test(timeOut = 30000)
public void testTlsLargeSizeMessage() throws Exception {
    log.info("-- Starting {} test --", methodName);

    final int MESSAGE_SIZE = 16 * 1024 + 1;
    log.info("-- message size --", MESSAGE_SIZE);
    String topicName = "persistent://my-property/use/my-ns/testTlsLargeSizeMessage"
                       + System.currentTimeMillis();

    internalSetUpForClient(true, pulsar.getBrokerServiceUrlTls());
    internalSetUpForNamespace();

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName)
            .subscriptionName("my-subscriber-name").subscribe();

    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
            .create();
    for (int i = 0; i < 10; i++) {
        byte[] message = new byte[MESSAGE_SIZE];
        Arrays.fill(message, (byte) i);
        producer.send(message);
    }

    Message<byte[]> msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        byte[] expected = new byte[MESSAGE_SIZE];
        Arrays.fill(expected, (byte) i);
        Assert.assertEquals(expected, msg.getData());
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 
Example 9
Source Project: pulsar   File: TlsProducerConsumerTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * verifies that messages whose size is larger than 2^14 bytes (max size of single TLS chunk) can be
 * produced/consumed
 *
 * @throws Exception
 */
@Test(timeOut = 30000)
public void testTlsLargeSizeMessage() throws Exception {
    log.info("-- Starting {} test --", methodName);

    final int MESSAGE_SIZE = 16 * 1024 + 1;
    log.info("-- message size --", MESSAGE_SIZE);

    internalSetUpForClient(true, pulsar.getBrokerServiceUrlTls());
    internalSetUpForNamespace();

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://my-property/use/my-ns/my-topic1")
            .subscriptionName("my-subscriber-name").subscribe();

    Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1")
            .create();
    for (int i = 0; i < 10; i++) {
        byte[] message = new byte[MESSAGE_SIZE];
        Arrays.fill(message, (byte) i);
        producer.send(message);
    }

    Message<byte[]> msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        byte[] expected = new byte[MESSAGE_SIZE];
        Arrays.fill(expected, (byte) i);
        Assert.assertEquals(expected, msg.getData());
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 
Example 10
Source Project: pulsar   File: TlsProducerConsumerTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * verifies that messages whose size is larger than 2^14 bytes (max size of single TLS chunk) can be
 * produced/consumed
 *
 * @throws Exception
 */
@Test(timeOut = 30000)
public void testTlsLargeSizeMessage() throws Exception {
    log.info("-- Starting {} test --", methodName);

    final int MESSAGE_SIZE = 16 * 1024 + 1;
    log.info("-- message size --", MESSAGE_SIZE);

    internalSetUpForClient(true, pulsar.getBrokerServiceUrlTls());
    internalSetUpForNamespace();

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://my-property/use/my-ns/my-topic1")
            .subscriptionName("my-subscriber-name").subscribe();

    Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1")
            .create();
    for (int i = 0; i < 10; i++) {
        byte[] message = new byte[MESSAGE_SIZE];
        Arrays.fill(message, (byte) i);
        producer.send(message);
    }

    Message<byte[]> msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        byte[] expected = new byte[MESSAGE_SIZE];
        Arrays.fill(expected, (byte) i);
        Assert.assertEquals(expected, msg.getData());
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 
Example 11
Source Project: pulsar   File: V1_ProducerConsumerTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test(dataProvider = "batch")
public void testBackoffAndReconnect(int batchMessageDelayMs) throws Exception {
    log.info("-- Starting {} test --", methodName);
    // Create consumer and producer
    Consumer<byte[]> consumer = pulsarClient.newConsumer()
            .topic("persistent://my-property/use/my-ns/my-topic4")
            .subscriptionName("my-subscriber-name")
            .subscriptionType(SubscriptionType.Exclusive)
            .startMessageIdInclusive()
            .subscribe();
    Producer<byte[]> producer = pulsarClient.newProducer()
            .topic("persistent://my-property/use/my-ns/my-topic4")
            .batchingMaxMessages(5)
            .batchingMaxPublishDelay(BATCHING_MAX_PUBLISH_DELAY_THRESHOLD, TimeUnit.MILLISECONDS)
            .enableBatching(batchMessageDelayMs != 0)
            .create();

    // Produce messages
    CompletableFuture<MessageId> lastFuture = null;
    for (int i = 0; i < 10; i++) {
        lastFuture = producer.sendAsync(("my-message-" + i).getBytes()).thenApply(msgId -> {
            log.info("Published message id: {}", msgId);
            return msgId;
        });
    }

    lastFuture.get();

    Message<byte[]> msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        log.info("Received: [{}]", new String(msg.getData()));
    }

    // Restart the broker and wait for the backoff to kick in. The client library will try to reconnect, and once
    // the broker is up, the consumer should receive the duplicate messages.
    log.info("-- Restarting broker --");
    restartBroker();

    msg = null;
    log.info("Receiving duplicate messages..");
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        log.info("Received: [{}]", new String(msg.getData()));
        Assert.assertNotNull(msg, "Message cannot be null");
    }
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 
Example 12
@Test
public void testBatchMessageIndexAckForExclusiveSubscription() throws PulsarClientException, ExecutionException, InterruptedException {
    final String topic = "testBatchMessageIndexAckForExclusiveSubscription";

    @Cleanup
    Consumer<Integer> consumer = pulsarClient.newConsumer(Schema.INT32)
        .topic(topic)
        .subscriptionName("sub")
        .receiverQueueSize(100)
        .subscribe();

    @Cleanup
    Producer<Integer> producer = pulsarClient.newProducer(Schema.INT32)
        .topic(topic)
        .batchingMaxPublishDelay(50, TimeUnit.MILLISECONDS)
        .create();

    final int messages = 100;
    List<CompletableFuture<MessageId>> futures = new ArrayList<>(messages);
    for (int i = 0; i < messages; i++) {
        futures.add(producer.sendAsync(i));
    }
    FutureUtil.waitForAll(futures).get();

    for (int i = 0; i < messages; i++) {
        if (i == 49) {
            consumer.acknowledgeCumulative(consumer.receive());
        } else {
            consumer.receive();
        }
    }

    //Wait ack send.
    Thread.sleep(1000);
    consumer.close();
    consumer = pulsarClient.newConsumer(Schema.INT32)
        .topic(topic)
        .subscriptionName("sub")
        .receiverQueueSize(100)
        .subscribe();

    List<Message<Integer>> received = new ArrayList<>(100);
    for (int i = 0; i < messages; i++) {
        received.add(consumer.receive());
    }

    Assert.assertEquals(received.size(), 100);
}
 
Example 13
@Test
public void testMultiHostUrlRetrySuccess() throws Exception {
    log.info("-- Starting {} test --", methodName);

    final String topicName = "persistent://my-property/my-ns/my-topic1";
    final String subscriptionName = "my-subscriber-name";

    // Multi hosts included an unreached port and the actual port for verify retry logic
    String urlsWithUnreached = "http://localhost:51000,localhost:" + new URI(pulsar.getWebServiceAddress()).getPort();
    if (isTcpLookup) {
        urlsWithUnreached = "pulsar://localhost:51000,localhost" + new URI(pulsar.getBrokerServiceUrl()).getPort();
    }
    PulsarClient client = newPulsarClient(urlsWithUnreached, 0);

    Consumer<byte[]> consumer = client.newConsumer().topic(topicName).subscriptionName(subscriptionName)
        .acknowledgmentGroupTime(0, TimeUnit.SECONDS).subscribe();
    Producer<byte[]> producer = client.newProducer().topic(topicName).create();

    for (int i = 0; i < 5; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        log.info("Produced message: [{}]", message);
    }

    Message<byte[]> msg = null;
    Set<String> messageSet = new HashSet();
    for (int i = 0; i < 5; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.info("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }

    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();

    producer.close();
    client.close();

    log.info("-- Exiting {} test --", methodName);
}
 
Example 14
Source Project: pulsar   File: BatchMessageIndexAckTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testBatchMessageIndexAckForExclusiveSubscription() throws PulsarClientException, ExecutionException, InterruptedException {
    final String topic = "testBatchMessageIndexAckForExclusiveSubscription";

    @Cleanup
    Consumer<Integer> consumer = pulsarClient.newConsumer(Schema.INT32)
        .topic(topic)
        .subscriptionName("sub")
        .receiverQueueSize(100)
        .subscribe();

    @Cleanup
    Producer<Integer> producer = pulsarClient.newProducer(Schema.INT32)
        .topic(topic)
        .batchingMaxPublishDelay(50, TimeUnit.MILLISECONDS)
        .create();

    final int messages = 100;
    List<CompletableFuture<MessageId>> futures = new ArrayList<>(messages);
    for (int i = 0; i < messages; i++) {
        futures.add(producer.sendAsync(i));
    }
    FutureUtil.waitForAll(futures).get();

    for (int i = 0; i < messages; i++) {
        if (i == 49) {
            consumer.acknowledgeCumulative(consumer.receive());
        } else {
            consumer.receive();
        }
    }

    //Wait ack send.
    Thread.sleep(1000);
    consumer.close();
    consumer = pulsarClient.newConsumer(Schema.INT32)
        .topic(topic)
        .subscriptionName("sub")
        .receiverQueueSize(100)
        .subscribe();

    List<Message<Integer>> received = new ArrayList<>(50);
    for (int i = 0; i < 50; i++) {
        received.add(consumer.receive());
    }

    Assert.assertEquals(received.size(), 50);

    Message<Integer> moreMessage = consumer.receive(1, TimeUnit.SECONDS);
    Assert.assertNull(moreMessage);

    futures.clear();
    for (int i = 0; i < 50; i++) {
        futures.add(producer.sendAsync(i));
    }
    FutureUtil.waitForAll(futures).get();

    for (int i = 0; i < 50; i++) {
        received.add(consumer.receive());
    }

    // Ensure the flow permit is work well since the client skip the acked batch index,
    // broker also need to handle the available permits.
    Assert.assertEquals(received.size(), 100);
}
 
Example 15
Source Project: pulsar   File: PersistentTopicE2ETest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testSimpleConsumerEvents() throws Exception {
    final String topicName = "persistent://prop/ns-abc/topic1";
    final String subName = "sub1";
    final int numMsgs = 10;

    // 1. client connect
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();

    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();
    PersistentSubscription subRef = topicRef.getSubscription(subName);

    assertNotNull(topicRef);
    assertNotNull(subRef);
    assertTrue(subRef.getDispatcher().isConsumerConnected());

    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(getAvailablePermits(subRef), 1000 /* default */);

    Producer<byte[]> producer = pulsarClient.newProducer()
        .topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();
    for (int i = 0; i < numMsgs * 2; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }

    assertTrue(subRef.getDispatcher().isConsumerConnected());
    rolloverPerIntervalStats();
    assertEquals(subRef.getNumberOfEntriesInBacklog(false), numMsgs * 2);

    // 2. messages pushed before client receive
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(getAvailablePermits(subRef), 1000 - numMsgs * 2);

    Message<byte[]> msg = null;
    for (int i = 0; i < numMsgs; i++) {
        msg = consumer.receive();
        // 3. in-order message delivery
        assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer.acknowledge(msg);
    }

    rolloverPerIntervalStats();

    // 4. messages deleted on individual acks
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(subRef.getNumberOfEntriesInBacklog(false), numMsgs);

    for (int i = 0; i < numMsgs; i++) {
        msg = consumer.receive();
        if (i == numMsgs - 1) {
            consumer.acknowledgeCumulative(msg);
        }
    }

    rolloverPerIntervalStats();

    // 5. messages deleted on cumulative acks
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(subRef.getNumberOfEntriesInBacklog(false), 0);

    // 6. consumer unsubscribe
    consumer.unsubscribe();

    // 6. consumer graceful close
    consumer.close();

    // 7. consumer unsubscribe
    try {
        consumer.unsubscribe();
        fail("Should have failed");
    } catch (PulsarClientException.AlreadyClosedException e) {
        // ok
    }

    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    subRef = topicRef.getSubscription(subName);
    assertNull(subRef);

    producer.close();

    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
}
 
Example 16
Source Project: pulsar   File: PersistentTopicE2ETest.java    License: Apache License 2.0 4 votes vote down vote up
@Test(enabled = false)
// TODO: enable this after java client supports graceful close
public void testGracefulClose() throws Exception {
    final String topicName = "persistent://prop/ns-abc/topic4";
    final String subName = "sub4";

    Producer<byte[]> producer = pulsarClient.newProducer()
        .topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);

    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();
    assertNotNull(topicRef);

    ExecutorService executor = Executors.newCachedThreadPool();
    CountDownLatch latch = new CountDownLatch(1);
    executor.submit(() -> {
        for (int i = 0; i < 10; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
        }
        latch.countDown();
        return null;
    });

    producer.close();

    // 1. verify there are no pending publish acks once the producer close
    // is completed on client
    assertEquals(topicRef.getProducers().values().iterator().next().getPendingPublishAcks(), 0);

    // safety latch in case of failure,
    // wait for the spawned thread to complete
    latch.await();

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();

    PersistentSubscription subRef = topicRef.getSubscription(subName);
    assertNotNull(subRef);

    Message<byte[]> msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive();
    }

    // 2. verify consumer close fails when there are outstanding
    // message acks
    try {
        consumer.close();
        fail("should have failed");
    } catch (IllegalStateException e) {
        // Expected - messages not acked
    }

    consumer.acknowledgeCumulative(msg);
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);

    // 3. verify consumer close succeeds once all messages are ack'ed
    consumer.close();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertTrue(subRef.getDispatcher().isConsumerConnected());

    executor.shutdown();
}
 
Example 17
@Test
void testAuthentication() throws Exception {
	log.info("-- Starting {} test --", methodName);

	// Step 1: Create Admin Client

	// create a client which connects to proxy and pass authData
	String topicName = "persistent://my-property/my-ns/my-topic1";

	ProxyConfiguration proxyConfig = new ProxyConfiguration();
	proxyConfig.setAuthenticationEnabled(true);
	proxyConfig.setServicePort(Optional.of(0));
	proxyConfig.setWebServicePort(Optional.of(0));
	proxyConfig.setBrokerServiceURL(pulsar.getBrokerServiceUrl());
	proxyConfig.setSaslJaasClientAllowedIds(".*" + localHostname + ".*");
	proxyConfig.setSaslJaasServerSectionName("PulsarProxy");

	// proxy connect to broker
	proxyConfig.setBrokerClientAuthenticationPlugin(AuthenticationSasl.class.getName());
	proxyConfig.setBrokerClientAuthenticationParameters(
		"{\"saslJaasClientSectionName\": " + "\"PulsarProxy\"," +
			"\"serverType\": " + "\"broker\"}");

	// proxy as a server, it will use sasl to authn
	Set<String> providers = new HashSet<>();
	providers.add(AuthenticationProviderSasl.class.getName());
	proxyConfig.setAuthenticationProviders(providers);

	proxyConfig.setForwardAuthorizationCredentials(true);
	AuthenticationService authenticationService = new AuthenticationService(
                       PulsarConfigurationLoader.convertFrom(proxyConfig));
	ProxyService proxyService = new ProxyService(proxyConfig, authenticationService);

	proxyService.start();
	final String proxyServiceUrl = "pulsar://localhost:" + proxyService.getListenPort().get();
	log.info("1 proxy service started {}", proxyService);

	// Step 3: Pass correct client params
	PulsarClient proxyClient = createProxyClient(proxyServiceUrl, 1);
	log.info("2 create proxy client {}, {}", proxyServiceUrl, proxyClient);

	Producer<byte[]> producer = proxyClient.newProducer(Schema.BYTES).topic(topicName).create();
	log.info("3 created producer.");

	Consumer<byte[]> consumer = proxyClient.newConsumer(Schema.BYTES).topic(topicName).subscriptionName("test-sub").subscribe();
	log.info("4 created consumer.");

	for (int i = 0; i < 10; i++) {
		String message = "my-message-" + i;
		producer.send(message.getBytes());
		log.info("Produced message: [{}]", message);
	}

	Message<byte[]> msg = null;
	Set<String> messageSet = Sets.newHashSet();
	for (int i = 0; i < 10; i++) {
		msg = consumer.receive(5, TimeUnit.SECONDS);
		String receivedMessage = new String(msg.getData());
		log.info("Received message: [{}]", receivedMessage);
		String expectedMessage = "my-message-" + i;
		testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
	}
	// Acknowledge the consumption of all messages at once
	consumer.acknowledgeCumulative(msg);
	consumer.close();

	proxyClient.close();
	proxyService.close();
}
 
Example 18
/**
 * <pre>
 * It verifies e2e tls + Authentication + Authorization (client -> proxy -> broker>
 *
 * 1. client connects to proxy over tls and pass auth-data
 * 2. proxy authenticate client and retrieve client-role
 *    and send it to broker as originalPrincipal over tls
 * 3. client creates producer/consumer via proxy
 * 4. broker authorize producer/consumer create request using originalPrincipal
 *
 * </pre>
 *
 * @throws Exception
 */
@SuppressWarnings("deprecation")
@Test
public void testTlsSyncProducerAndConsumer() throws Exception {
    log.info("-- Starting {} test --", methodName);

    final String proxyServiceUrl = proxyService.getServiceUrlTls();
    Map<String, String> authParams = Maps.newHashMap();
    authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
    authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
    Authentication authTls = new AuthenticationTls();
    authTls.configure(authParams);
    // create a client which connects to proxy over tls and pass authData
    PulsarClient proxyClient = createPulsarClient(authTls, proxyServiceUrl);

    admin.clusters().createCluster(configClusterName, new ClusterData(brokerUrl.toString(), brokerUrlTls.toString(),
            pulsar.getBrokerServiceUrl(), pulsar.getBrokerServiceUrlTls()));
    admin.tenants().createTenant("my-property",
            new TenantInfo(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("test")));
    admin.namespaces().createNamespace("my-property/my-ns", Sets.newHashSet("test"));

    Consumer<byte[]> consumer = proxyClient.newConsumer().topic("persistent://my-property/my-ns/my-topic1")
            .subscriptionName("my-subscriber-name").subscribe();
    Producer<byte[]> producer = proxyClient.newProducer(Schema.BYTES).topic("persistent://my-property/my-ns/my-topic1")
            .create();
    final int msgs = 10;
    for (int i = 0; i < msgs; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }

    Message<byte[]> msg = null;
    Set<String> messageSet = Sets.newHashSet();
    int count = 0;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
        count++;
    }
    // Acknowledge the consumption of all messages at once
    Assert.assertEquals(msgs, count);
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 
Example 19
/**
 * <pre>
 * It verifies e2e tls + Authentication + Authorization (client -> proxy -> broker>
 *
 * 1. client connects to proxy over tls and pass auth-data
 * 2. proxy authenticate client and retrieve client-role
 *    and send it to broker as originalPrincipal over tls
 * 3. client creates producer/consumer via proxy
 * 4. broker authorize producer/consumer create request using originalPrincipal
 *
 * </pre>
 *
 * @throws Exception
 */
@Test
public void testDiscoveryService() throws Exception {
    log.info("-- Starting {} test --", methodName);

    Map<String, String> authParams = Maps.newHashMap();
    authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
    authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
    Authentication authTls = new AuthenticationTls();
    authTls.configure(authParams);
    // create a client which connects to proxy over tls and pass authData
    PulsarClient proxyClient = createPulsarClient(authTls, proxyService.getServiceUrlTls());

    admin.clusters().createCluster("without-service-discovery", new ClusterData(brokerUrl.toString()));

    admin.tenants().createTenant("my-property", new TenantInfo(Sets.newHashSet("appid1", "appid2"),
            Sets.newHashSet("without-service-discovery")));
    admin.namespaces().createNamespace("my-property/without-service-discovery/my-ns");

    Consumer<byte[]> consumer = proxyClient.newConsumer()
            .topic("persistent://my-property/without-service-discovery/my-ns/my-topic1")
            .subscriptionName("my-subscriber-name").subscribe();
    Producer<byte[]> producer = proxyClient.newProducer(Schema.BYTES)
            .topic("persistent://my-property/without-service-discovery/my-ns/my-topic1").create();
    final int msgs = 10;
    for (int i = 0; i < msgs; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }

    Message<byte[]> msg = null;
    Set<String> messageSet = Sets.newHashSet();
    int count = 0;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
        count++;
    }
    // Acknowledge the consumption of all messages at once
    Assert.assertEquals(msgs, count);
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 
Example 20
/**
 * <pre>
 * It verifies e2e tls + Authentication + Authorization (client -> proxy -> broker)
 *
 * 1. client connects to proxy over tls and pass auth-data
 * 2. proxy authenticate client and retrieve client-role
 *    and send it to broker as originalPrincipal over tls
 * 3. client creates producer/consumer via proxy
 * 4. broker authorize producer/consumer create request using originalPrincipal
 *
 * </pre>
 *
 * @throws Exception
 */
@Test
public void testProxyAuthorization() throws Exception {
    log.info("-- Starting {} test --", methodName);

    startProxy();
    createAdminClient();
    // create a client which connects to proxy over tls and pass authData
    PulsarClient proxyClient = createPulsarClient(proxyService.getServiceUrlTls(), PulsarClient.builder());

    String namespaceName = "my-property/proxy-authorization/my-ns";

    admin.clusters().createCluster("proxy-authorization", new ClusterData(brokerUrl.toString()));

    admin.tenants().createTenant("my-property",
            new TenantInfo(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("proxy-authorization")));
    admin.namespaces().createNamespace(namespaceName);

    admin.namespaces().grantPermissionOnNamespace(namespaceName, "Proxy",
            Sets.newHashSet(AuthAction.consume, AuthAction.produce));
    admin.namespaces().grantPermissionOnNamespace(namespaceName, "Client",
            Sets.newHashSet(AuthAction.consume, AuthAction.produce));

    Consumer<byte[]> consumer = proxyClient.newConsumer()
            .topic("persistent://my-property/proxy-authorization/my-ns/my-topic1")
            .subscriptionName("my-subscriber-name").subscribe();

    Producer<byte[]> producer = proxyClient.newProducer(Schema.BYTES)
            .topic("persistent://my-property/proxy-authorization/my-ns/my-topic1").create();
    final int msgs = 10;
    for (int i = 0; i < msgs; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }

    Message<byte[]> msg = null;
    Set<String> messageSet = Sets.newHashSet();
    int count = 0;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
        count++;
    }
    // Acknowledge the consumption of all messages at once
    Assert.assertEquals(msgs, count);
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}