Java Code Examples for org.apache.pulsar.client.api.Consumer#receive()

The following examples show how to use org.apache.pulsar.client.api.Consumer#receive() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: pulsar   File: SourceTester.java    License: Apache License 2.0 6 votes vote down vote up
public void validateSourceResultJson(Consumer<KeyValue<byte[], byte[]>> consumer, int number, String eventType) throws Exception {
    int recordsNumber = 0;
    Message<KeyValue<byte[], byte[]>> msg = consumer.receive(2, TimeUnit.SECONDS);
    while(msg != null) {
        recordsNumber ++;
        final String key = new String(msg.getValue().getKey());
        final String value = new String(msg.getValue().getValue());
        log.info("Received message: key = {}, value = {}.", key, value);
        Assert.assertTrue(key.contains(this.keyContains()));
        Assert.assertTrue(value.contains(this.valueContains()));
        if (eventType != null) {
            Assert.assertTrue(value.contains(this.eventContains(eventType, true)));
        }
        consumer.acknowledge(msg);
        msg = consumer.receive(1, TimeUnit.SECONDS);
    }

    Assert.assertEquals(recordsNumber, number);
    log.info("Stop {} server container. topic: {} has {} records.", getSourceType(), consumer.getTopic(), recordsNumber);
}
 
Example 2
Source Project: pulsar   File: ProxyTlsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPartitions() throws Exception {
    PulsarClient client = PulsarClient.builder()
            .serviceUrl(proxyService.getServiceUrlTls())
            .allowTlsInsecureConnection(false).tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).build();
    TenantInfo tenantInfo = createDefaultTenantInfo();
    admin.tenants().createTenant("sample", tenantInfo);
    admin.topics().createPartitionedTopic("persistent://sample/test/local/partitioned-topic", 2);

    Producer<byte[]> producer = client.newProducer(Schema.BYTES).topic("persistent://sample/test/local/partitioned-topic")
            .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();

    // Create a consumer directly attached to broker
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://sample/test/local/partitioned-topic")
            .subscriptionName("my-sub").subscribe();

    for (int i = 0; i < 10; i++) {
        producer.send("test".getBytes());
    }

    for (int i = 0; i < 10; i++) {
        Message<byte[]> msg = consumer.receive(1, TimeUnit.SECONDS);
        checkNotNull(msg);
    }

    client.close();
}
 
Example 3
/**
 * verifies that messages whose size is larger than 2^14 bytes (max size of single TLS chunk) can be
 * produced/consumed
 *
 * @throws Exception
 */
@Test(timeOut = 30000)
public void testTlsLargeSizeMessage() throws Exception {
    log.info("-- Starting {} test --", methodName);

    final int MESSAGE_SIZE = 16 * 1024 + 1;
    log.info("-- message size --", MESSAGE_SIZE);
    String topicName = "persistent://my-property/use/my-ns/testTlsLargeSizeMessage"
                       + System.currentTimeMillis();

    internalSetUpForClient(true, pulsar.getBrokerServiceUrlTls());
    internalSetUpForNamespace();

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName)
            .subscriptionName("my-subscriber-name").subscribe();

    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
            .create();
    for (int i = 0; i < 10; i++) {
        byte[] message = new byte[MESSAGE_SIZE];
        Arrays.fill(message, (byte) i);
        producer.send(message);
    }

    Message<byte[]> msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        byte[] expected = new byte[MESSAGE_SIZE];
        Arrays.fill(expected, (byte) i);
        Assert.assertEquals(expected, msg.getData());
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 
Example 4
Source Project: pulsar   File: TlsProducerConsumerTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * verifies that messages whose size is larger than 2^14 bytes (max size of single TLS chunk) can be
 * produced/consumed
 *
 * @throws Exception
 */
@Test(timeOut = 30000)
public void testTlsLargeSizeMessage() throws Exception {
    log.info("-- Starting {} test --", methodName);

    final int MESSAGE_SIZE = 16 * 1024 + 1;
    log.info("-- message size --", MESSAGE_SIZE);

    internalSetUpForClient(true, pulsar.getBrokerServiceUrlTls());
    internalSetUpForNamespace();

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://my-property/use/my-ns/my-topic1")
            .subscriptionName("my-subscriber-name").subscribe();

    Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1")
            .create();
    for (int i = 0; i < 10; i++) {
        byte[] message = new byte[MESSAGE_SIZE];
        Arrays.fill(message, (byte) i);
        producer.send(message);
    }

    Message<byte[]> msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        byte[] expected = new byte[MESSAGE_SIZE];
        Arrays.fill(expected, (byte) i);
        Assert.assertEquals(expected, msg.getData());
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 
Example 5
Source Project: pulsar   File: BatchMessageTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test(dataProvider = "containerBuilder")
public void testRetrieveSequenceIdGenerated(BatcherBuilder builder) throws Exception {

    int numMsgs = 10;
    final String topicName = "persistent://prop/ns-abc/testRetrieveSequenceIdGenerated-" + UUID.randomUUID();
    final String subscriptionName = "sub-1";

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
            .subscriptionType(SubscriptionType.Shared).subscribe();

    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
            .batchingMaxPublishDelay(5, TimeUnit.SECONDS).batchingMaxMessages(numMsgs).enableBatching(true)
            .batcherBuilder(builder)
            .create();

    List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
    for (int i = 0; i < numMsgs; i++) {
        byte[] message = ("my-message-" + i).getBytes();
        sendFutureList.add(producer.sendAsync(message));
    }
    FutureUtil.waitForAll(sendFutureList).get();

    for (int i = 0; i < numMsgs; i++) {
        Message<byte[]> received = consumer.receive();
        Assert.assertEquals(received.getSequenceId(), i);
        consumer.acknowledge(received);
    }

    producer.close();
    consumer.close();
}
 
Example 6
Source Project: pulsar   File: ProxyTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPartitions() throws Exception {
    TenantInfo tenantInfo = createDefaultTenantInfo();
    admin.tenants().createTenant("sample", tenantInfo);
    @Cleanup
    PulsarClient client = PulsarClient.builder().serviceUrl(proxyService.getServiceUrl())
            .build();
    admin.topics().createPartitionedTopic("persistent://sample/test/local/partitioned-topic", 2);

    @Cleanup
    Producer<byte[]> producer = client.newProducer(Schema.BYTES)
        .topic("persistent://sample/test/local/partitioned-topic")
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();

    // Create a consumer directly attached to broker
    @Cleanup
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://sample/test/local/partitioned-topic")
            .subscriptionName("my-sub").subscribe();

    for (int i = 0; i < 10; i++) {
        producer.send("test".getBytes());
    }

    for (int i = 0; i < 10; i++) {
        Message<byte[]> msg = consumer.receive(1, TimeUnit.SECONDS);
        checkNotNull(msg);
    }
}
 
Example 7
@Test
public void testSingleMessageBatch() throws Exception {
    String topicName = "prop/ns-abc/topic-estSingleMessageBatch";

    Producer<String> producer = pulsarClient.newProducer(Schema.STRING)
            .topic(topicName)
            .enableBatching(true)
            .batchingMaxPublishDelay(10, TimeUnit.SECONDS)
            .create();

    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING)
            .topic(topicName)
            .subscriptionName("subscription")
            .ackTimeout(1, TimeUnit.HOURS)
            .subscribe();

    // Force the creation of a batch with a single message
    producer.sendAsync("hello");
    producer.flush();

    Message<String> message = consumer.receive();

    assertFalse(((ConsumerImpl<?>) consumer).getUnAckedMessageTracker().isEmpty());

    consumer.acknowledge(message);

    assertTrue(((ConsumerImpl<?>) consumer).getUnAckedMessageTracker().isEmpty());
}
 
Example 8
Source Project: pulsar   File: PythonSchemaTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Publish from Java and consume from Python
 */
@Test(dataProvider = "ServiceUrls")
public void testPythonPublishJavaConsume(String serviceUrl) throws Exception {
    String nsName = generateNamespaceName();
    pulsarCluster.createNamespace(nsName);

    String topicName = generateTopicName(nsName, "testPythonPublishJavaConsume", true);

    @Cleanup
    PulsarClient client = PulsarClient.builder()
            .serviceUrl(serviceUrl)
            .build();

    @Cleanup
    Consumer<Example2> consumer = client.newConsumer(Schema.AVRO(Example2.class))
            .topic(topicName)
            .subscriptionName("test-sub")
            .subscribe();

    // Verify Python can receive the typed message

    ContainerExecResult res = pulsarCluster.getAnyBroker()
            .execCmd("/pulsar/examples/python-examples/producer_schema.py", "pulsar://localhost:6650", topicName);
    assertEquals(res.getExitCode(), 0);

    Message<Example2> msg = consumer.receive();
    Example2 e2 = msg.getValue();
    assertEquals(e2.a, "Hello");
    assertEquals(e2.b, 1);
}
 
Example 9
Source Project: pulsar   File: SequenceIdWithErrorTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Test that sequence id from a producer is correct when there are send errors
 */
@Test
public void testCheckSequenceId() throws Exception {
    admin.namespaces().createNamespace("prop/my-test", Collections.singleton("usc"));

    String topicName = "prop/my-test/my-topic";
    int N = 10;

    PulsarClient client = PulsarClient.builder().serviceUrl(pulsar.getBrokerServiceUrl()).build();

    // Create consumer
    Consumer<String> consumer = client.newConsumer(Schema.STRING).topic(topicName).subscriptionName("sub")
            .subscribe();

    // Fence the topic by opening the ManagedLedger for the topic outside the Pulsar broker. This will cause the
    // broker to fail subsequent send operation and it will trigger a recover
    ManagedLedgerClientFactory clientFactory = new ManagedLedgerClientFactory(pulsar.getConfiguration(),
            pulsar.getZkClient(), pulsar.getBookKeeperClientFactory());
    ManagedLedgerFactory mlFactory = clientFactory.getManagedLedgerFactory();
    ManagedLedger ml = mlFactory.open(TopicName.get(topicName).getPersistenceNamingEncoding());
    ml.close();
    clientFactory.close();

    // Create a producer
    Producer<String> producer = client.newProducer(Schema.STRING).topic(topicName).create();

    for (int i = 0; i < N; i++) {
        producer.send("Hello-" + i);
    }

    for (int i = 0; i < N; i++) {
        Message<String> msg = consumer.receive();
        assertEquals(msg.getValue(), "Hello-" + i);
        assertEquals(msg.getSequenceId(), i);
        consumer.acknowledge(msg);
    }

    client.close();
}
 
Example 10
Source Project: kop   File: KafkaApisTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeOut = 20000)
public void testConsumerListOffsetLatest() throws Exception {
    String topicName = "testConsumerListOffsetLatest";
    TopicPartition tp = new TopicPartition(topicName, 0);

    // use producer to create some message to get Limit Offset.
    String pulsarTopicName = "persistent://public/default/" + topicName;

    // create partitioned topic.
    admin.topics().createPartitionedTopic(topicName, 1);

    // 1. prepare topic:
    //    use kafka producer to produce 10 messages.
    //    use pulsar consumer to get message offset.
    @Cleanup
    KProducer kProducer = new KProducer(topicName, false, getKafkaBrokerPort());
    int totalMsgs = 10;
    String messageStrPrefix = topicName + "_message_";

    for (int i = 0; i < totalMsgs; i++) {
        String messageStr = messageStrPrefix + i;
        kProducer.getProducer()
            .send(new ProducerRecord<>(
                topicName,
                i,
                messageStr))
            .get();
        log.debug("Kafka Producer Sent message: ({}, {})", i, messageStr);
    }

    @Cleanup
    Consumer<byte[]> consumer = pulsarClient.newConsumer()
        .topic(pulsarTopicName)
        .subscriptionName(topicName + "_sub")
        .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
        .subscribe();
    Message<byte[]> msg = consumer.receive(100, TimeUnit.MILLISECONDS);
    assertNotNull(msg);
    MessageIdImpl messageId = (MessageIdImpl) ((TopicMessageIdImpl) msg.getMessageId()).getInnerMessageId();
    // LAC entry should be the limit offset.
    long limitOffset = MessageIdUtils.getOffset(messageId.getLedgerId(), totalMsgs - 1);
    log.info("After create {} messages, get messageId: {} expected latest limit: {}",
        totalMsgs, messageId, limitOffset);

    // 2. real test, for ListOffset request verify Earliest get earliest
    Map<TopicPartition, Long> targetTimes = Maps.newHashMap();
    targetTimes.put(tp, ListOffsetRequest.LATEST_TIMESTAMP);

    ListOffsetRequest.Builder builder = ListOffsetRequest.Builder
        .forConsumer(true, IsolationLevel.READ_UNCOMMITTED)
        .setTargetTimes(targetTimes);

    KafkaHeaderAndRequest request = buildRequest(builder);
    CompletableFuture<AbstractResponse> responseFuture = new CompletableFuture<>();
    kafkaRequestHandler
        .handleListOffsetRequest(request, responseFuture);

    AbstractResponse response = responseFuture.get();
    ListOffsetResponse listOffsetResponse = (ListOffsetResponse) response;
    assertEquals(listOffsetResponse.responseData().get(tp).error, Errors.NONE);
    assertEquals(listOffsetResponse.responseData().get(tp).offset, Long.valueOf(limitOffset));
    assertEquals(listOffsetResponse.responseData().get(tp).timestamp, Long.valueOf(0));
}
 
Example 11
Source Project: pulsar   File: PulsarFunctionE2ETest.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeOut = 30000)
private void testPulsarSinkDLQ() throws Exception {
    final String namespacePortion = "io";
    final String replNamespace = tenant + "/" + namespacePortion;
    final String sourceTopic = "persistent://" + replNamespace + "/input";
    final String dlqTopic = sourceTopic+"-DLQ";
    final String sinkName = "PulsarSink-test";
    final String propertyKey = "key";
    final String propertyValue = "value";
    final String subscriptionName = "test-sub";
    admin.namespaces().createNamespace(replNamespace);
    Set<String> clusters = Sets.newHashSet(Lists.newArrayList("use"));
    admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
    // 1 create producer、DLQ consumer
    Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create();
    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING).topic(dlqTopic).subscriptionName(subscriptionName).subscribe();

    // 2 setup sink
    SinkConfig sinkConfig = createSinkConfig(tenant, namespacePortion, sinkName, sourceTopic, subscriptionName);
    sinkConfig.setNegativeAckRedeliveryDelayMs(1001L);
    sinkConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE);
    sinkConfig.setMaxMessageRetries(2);
    sinkConfig.setDeadLetterTopic(dlqTopic);
    sinkConfig.setInputSpecs(Collections.singletonMap(sourceTopic, ConsumerConfig.builder().receiverQueueSize(1000).build()));
    sinkConfig.setClassName(SinkForTest.class.getName());
    LocalRunner localRunner = LocalRunner.builder()
            .sinkConfig(sinkConfig)
            .clientAuthPlugin(AuthenticationTls.class.getName())
            .clientAuthParams(String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH))
            .useTls(true)
            .tlsTrustCertFilePath(TLS_TRUST_CERT_FILE_PATH)
            .tlsAllowInsecureConnection(true)
            .tlsHostNameVerificationEnabled(false)
            .brokerServiceUrl(pulsar.getBrokerServiceUrlTls()).build();

    localRunner.start(false);

    retryStrategically((test) -> {
        try {
            TopicStats topicStats = admin.topics().getStats(sourceTopic);

            return topicStats.subscriptions.containsKey(subscriptionName)
                    && topicStats.subscriptions.get(subscriptionName).consumers.size() == 1
                    && topicStats.subscriptions.get(subscriptionName).consumers.get(0).availablePermits == 1000;

        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150);

    // 3 send message
    int totalMsgs = 10;
    for (int i = 0; i < totalMsgs; i++) {
        producer.newMessage().property(propertyKey, propertyValue).value("fail" + i).sendAsync();
    }

    //4 All messages should enter DLQ
    for (int i = 0; i < totalMsgs; i++) {
        Message<String> message = consumer.receive(10, TimeUnit.SECONDS);
        assertNotNull(message);
        assertEquals(message.getValue(), "fail" + i);
    }

    //clean up
    producer.close();
    consumer.close();
}
 
Example 12
@Test
public void testBatchMessageIndexAckForExclusiveSubscription() throws PulsarClientException, ExecutionException, InterruptedException {
    final String topic = "testBatchMessageIndexAckForExclusiveSubscription";

    @Cleanup
    Consumer<Integer> consumer = pulsarClient.newConsumer(Schema.INT32)
        .topic(topic)
        .subscriptionName("sub")
        .receiverQueueSize(100)
        .subscribe();

    @Cleanup
    Producer<Integer> producer = pulsarClient.newProducer(Schema.INT32)
        .topic(topic)
        .batchingMaxPublishDelay(50, TimeUnit.MILLISECONDS)
        .create();

    final int messages = 100;
    List<CompletableFuture<MessageId>> futures = new ArrayList<>(messages);
    for (int i = 0; i < messages; i++) {
        futures.add(producer.sendAsync(i));
    }
    FutureUtil.waitForAll(futures).get();

    for (int i = 0; i < messages; i++) {
        if (i == 49) {
            consumer.acknowledgeCumulative(consumer.receive());
        } else {
            consumer.receive();
        }
    }

    //Wait ack send.
    Thread.sleep(1000);
    consumer.close();
    consumer = pulsarClient.newConsumer(Schema.INT32)
        .topic(topic)
        .subscriptionName("sub")
        .receiverQueueSize(100)
        .subscribe();

    List<Message<Integer>> received = new ArrayList<>(100);
    for (int i = 0; i < messages; i++) {
        received.add(consumer.receive());
    }

    Assert.assertEquals(received.size(), 100);
}
 
Example 13
Source Project: pulsar   File: NullValueTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void keyValueNullInlineTest() throws PulsarClientException {
    String topic = "persistent://prop/ns-abc/kv-null-value-test";

    @Cleanup
    Producer<KeyValue<String, String>> producer = pulsarClient
            .newProducer(KeyValueSchema.of(Schema.STRING, Schema.STRING))
            .topic(topic)
            .create();

    @Cleanup
    Consumer<KeyValue<String, String>> consumer = pulsarClient
            .newConsumer(KeyValueSchema.of(Schema.STRING, Schema.STRING))
            .topic(topic)
            .subscriptionName("test")
            .subscribe();

    int numMessage = 10;
    for (int i = 0; i < numMessage; i++) {
        producer.newMessage().value(new KeyValue<>(null, "test")).send();
        producer.newMessage().value(new KeyValue<>("test", null)).send();
        producer.newMessage().value(new KeyValue<>(null, null)).send();
    }

    Message<KeyValue<String, String>> message;
    KeyValue<String, String> keyValue;
    for (int i = 0; i < numMessage; i++) {
        message = consumer.receive();
        keyValue = message.getValue();
        Assert.assertNull(keyValue.getKey());
        Assert.assertEquals("test", keyValue.getValue());

        message = consumer.receive();
        keyValue = message.getValue();
        Assert.assertEquals("test", keyValue.getKey());
        Assert.assertNull(keyValue.getValue());

        message = consumer.receive();
        keyValue = message.getValue();
        Assert.assertNull(keyValue.getKey());
        Assert.assertNull(keyValue.getValue());
    }

}
 
Example 14
/**
 * <pre>
 * It verifies e2e tls + Authentication + Authorization (client -> proxy -> broker>
 *
 * 1. client connects to proxy over tls and pass auth-data
 * 2. proxy authenticate client and retrieve client-role
 *    and send it to broker as originalPrincipal over tls
 * 3. client creates producer/consumer via proxy
 * 4. broker authorize producer/consumer create request using originalPrincipal
 *
 * </pre>
 *
 * @throws Exception
 */
@SuppressWarnings("deprecation")
@Test
public void testTlsSyncProducerAndConsumer() throws Exception {
    log.info("-- Starting {} test --", methodName);

    final String proxyServiceUrl = proxyService.getServiceUrlTls();
    Map<String, String> authParams = Maps.newHashMap();
    authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
    authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
    Authentication authTls = new AuthenticationTls();
    authTls.configure(authParams);
    // create a client which connects to proxy over tls and pass authData
    PulsarClient proxyClient = createPulsarClient(authTls, proxyServiceUrl);

    admin.clusters().createCluster(configClusterName, new ClusterData(brokerUrl.toString(), brokerUrlTls.toString(),
            pulsar.getBrokerServiceUrl(), pulsar.getBrokerServiceUrlTls()));
    admin.tenants().createTenant("my-property",
            new TenantInfo(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("test")));
    admin.namespaces().createNamespace("my-property/my-ns", Sets.newHashSet("test"));

    Consumer<byte[]> consumer = proxyClient.newConsumer().topic("persistent://my-property/my-ns/my-topic1")
            .subscriptionName("my-subscriber-name").subscribe();
    Producer<byte[]> producer = proxyClient.newProducer(Schema.BYTES).topic("persistent://my-property/my-ns/my-topic1")
            .create();
    final int msgs = 10;
    for (int i = 0; i < msgs; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }

    Message<byte[]> msg = null;
    Set<String> messageSet = Sets.newHashSet();
    int count = 0;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
        count++;
    }
    // Acknowledge the consumption of all messages at once
    Assert.assertEquals(msgs, count);
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 
Example 15
Source Project: pulsar   File: BatchMessageIndexAckTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testBatchMessageIndexAckForExclusiveSubscription() throws PulsarClientException, ExecutionException, InterruptedException {
    final String topic = "testBatchMessageIndexAckForExclusiveSubscription";

    @Cleanup
    Consumer<Integer> consumer = pulsarClient.newConsumer(Schema.INT32)
        .topic(topic)
        .subscriptionName("sub")
        .receiverQueueSize(100)
        .subscribe();

    @Cleanup
    Producer<Integer> producer = pulsarClient.newProducer(Schema.INT32)
        .topic(topic)
        .batchingMaxPublishDelay(50, TimeUnit.MILLISECONDS)
        .create();

    final int messages = 100;
    List<CompletableFuture<MessageId>> futures = new ArrayList<>(messages);
    for (int i = 0; i < messages; i++) {
        futures.add(producer.sendAsync(i));
    }
    FutureUtil.waitForAll(futures).get();

    for (int i = 0; i < messages; i++) {
        if (i == 49) {
            consumer.acknowledgeCumulative(consumer.receive());
        } else {
            consumer.receive();
        }
    }

    //Wait ack send.
    Thread.sleep(1000);
    consumer.close();
    consumer = pulsarClient.newConsumer(Schema.INT32)
        .topic(topic)
        .subscriptionName("sub")
        .receiverQueueSize(100)
        .subscribe();

    List<Message<Integer>> received = new ArrayList<>(50);
    for (int i = 0; i < 50; i++) {
        received.add(consumer.receive());
    }

    Assert.assertEquals(received.size(), 50);

    Message<Integer> moreMessage = consumer.receive(1, TimeUnit.SECONDS);
    Assert.assertNull(moreMessage);

    futures.clear();
    for (int i = 0; i < 50; i++) {
        futures.add(producer.sendAsync(i));
    }
    FutureUtil.waitForAll(futures).get();

    for (int i = 0; i < 50; i++) {
        received.add(consumer.receive());
    }

    // Ensure the flow permit is work well since the client skip the acked batch index,
    // broker also need to handle the available permits.
    Assert.assertEquals(received.size(), 100);
}
 
Example 16
Source Project: pulsar   File: V1_ProducerConsumerTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test(dataProvider = "batch")
public void testBackoffAndReconnect(int batchMessageDelayMs) throws Exception {
    log.info("-- Starting {} test --", methodName);
    // Create consumer and producer
    Consumer<byte[]> consumer = pulsarClient.newConsumer()
            .topic("persistent://my-property/use/my-ns/my-topic4")
            .subscriptionName("my-subscriber-name")
            .subscriptionType(SubscriptionType.Exclusive)
            .startMessageIdInclusive()
            .subscribe();
    Producer<byte[]> producer = pulsarClient.newProducer()
            .topic("persistent://my-property/use/my-ns/my-topic4")
            .batchingMaxMessages(5)
            .batchingMaxPublishDelay(BATCHING_MAX_PUBLISH_DELAY_THRESHOLD, TimeUnit.MILLISECONDS)
            .enableBatching(batchMessageDelayMs != 0)
            .create();

    // Produce messages
    CompletableFuture<MessageId> lastFuture = null;
    for (int i = 0; i < 10; i++) {
        lastFuture = producer.sendAsync(("my-message-" + i).getBytes()).thenApply(msgId -> {
            log.info("Published message id: {}", msgId);
            return msgId;
        });
    }

    lastFuture.get();

    Message<byte[]> msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        log.info("Received: [{}]", new String(msg.getData()));
    }

    // Restart the broker and wait for the backoff to kick in. The client library will try to reconnect, and once
    // the broker is up, the consumer should receive the duplicate messages.
    log.info("-- Restarting broker --");
    restartBroker();

    msg = null;
    log.info("Receiving duplicate messages..");
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        log.info("Received: [{}]", new String(msg.getData()));
        Assert.assertNotNull(msg, "Message cannot be null");
    }
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 
Example 17
Source Project: pulsar   File: PersistentQueueE2ETest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testUnackedCountWithRedeliveries() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/testUnackedCountWithRedeliveries";
    final String subName = "sub3";
    final int numMsgs = 10;

    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();

    ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
            .receiverQueueSize(10).subscriptionType(SubscriptionType.Shared)
            .acknowledgmentGroupTime(0, TimeUnit.SECONDS);
    ConsumerImpl<byte[]> consumer1 = (ConsumerImpl<byte[]>) consumerBuilder.subscribe();

    for (int i = 0; i < numMsgs; i++) {
        producer.send(("hello-" + i).getBytes());
    }

    Set<MessageId> c1_receivedMessages = new HashSet<>();

    // C-1 gets all messages but doesn't ack
    for (int i = 0; i < numMsgs; i++) {
        c1_receivedMessages.add(consumer1.receive().getMessageId());
    }

    // C-2 will not get any message initially, since everything went to C-1 already
    Consumer<byte[]> consumer2 = consumerBuilder.subscribe();

    // Trigger C-1 to redeliver everything, half will go C-1 again and the other half to C-2
    consumer1.redeliverUnacknowledgedMessages(c1_receivedMessages);

    // Consumer 2 will also receive all message but not ack
    for (int i = 0; i < numMsgs; i++) {
        consumer2.receive();
    }

    for (MessageId msgId : c1_receivedMessages) {
        consumer1.acknowledge(msgId);
    }

    TopicStats stats = admin.topics().getStats(topicName);

    // Unacked messages count should be 0 for both consumers at this point
    SubscriptionStats subStats = stats.subscriptions.get(subName);
    assertEquals(subStats.msgBacklog, 0);

    for (ConsumerStats cs : subStats.consumers) {
        assertEquals(cs.unackedMessages, 0);
    }

    producer.close();
    consumer1.close();
    consumer2.close();

    deleteTopic(topicName);
}
 
Example 18
@Test(dataProvider =  "CanReadLastSchemaCompatibilityStrategy")
public void testConsumerCompatibilityCheckCanReadLastTest(SchemaCompatibilityStrategy schemaCompatibilityStrategy) throws Exception {
    final String tenant = PUBLIC_TENANT;
    final String topic = "test-consumer-compatibility";

        String namespace = "test-namespace-" + randomName(16);
        String fqtn = TopicName.get(
                TopicDomain.persistent.value(),
                tenant,
                namespace,
                topic
        ).toString();

        NamespaceName namespaceName = NamespaceName.get(tenant, namespace);

        admin.namespaces().createNamespace(
                tenant + "/" + namespace,
                Sets.newHashSet(CLUSTER_NAME)
        );

        admin.namespaces().setSchemaCompatibilityStrategy(namespaceName.toString(), schemaCompatibilityStrategy);
        admin.schemas().createSchema(fqtn, Schema.AVRO(Schemas.PersonOne.class).getSchemaInfo());
        admin.schemas().createSchema(fqtn, Schema.AVRO(SchemaDefinition.builder()
                .withAlwaysAllowNull(false).withPojo(Schemas.PersonTwo.class).build()).getSchemaInfo());

        Consumer<Schemas.PersonThree> consumerThree = pulsarClient.newConsumer(Schema.AVRO(
                SchemaDefinition.<Schemas.PersonThree>builder().withAlwaysAllowNull
                        (false).withSupportSchemaVersioning(true).
                        withPojo(Schemas.PersonThree.class).build()))
                .subscriptionName("test")
                .topic(fqtn)
                .subscribe();

        Producer<Schemas.PersonOne> producerOne = pulsarClient
                .newProducer(Schema.AVRO(Schemas.PersonOne.class))
                .topic(fqtn)
                .create();


        Schemas.PersonOne personOne = new Schemas.PersonOne();
        personOne.setId(1);

        producerOne.send(personOne);
        Message<Schemas.PersonThree> message = null;

        try {
            message = consumerThree.receive();
            message.getValue();
        } catch (Exception e) {
            Assert.assertTrue(e instanceof SchemaSerializationException);
            consumerThree.acknowledge(message);
        }

        Producer<Schemas.PersonTwo> producerTwo = pulsarClient
                .newProducer(Schema.AVRO(SchemaDefinition.<Schemas.PersonTwo>builder().withAlwaysAllowNull
                        (false).withSupportSchemaVersioning(true).
                        withPojo(Schemas.PersonTwo.class).build()))
                .topic(fqtn)
                .create();

        Schemas.PersonTwo personTwo = new Schemas.PersonTwo();
        personTwo.setId(1);
        personTwo.setName("Jerry");
        producerTwo.send(personTwo);

        message = consumerThree.receive();
        Schemas.PersonThree personThree = message.getValue();
        consumerThree.acknowledge(message);

        assertEquals(personThree.getId(), 1);
        assertEquals(personThree.getName(), "Jerry");

        consumerThree.close();
        producerOne.close();
        producerTwo.close();
}
 
Example 19
Source Project: pulsar   File: ResendRequestTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeOut = testTimeout)
public void testFailoverInactiveConsumer() throws Exception {
    String key = "testFailoverInactiveConsumer";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-failover-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    // 1. producer connect
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);

    // 2. Create consumer
    ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName)
            .subscriptionName(subscriptionName).receiverQueueSize(10).subscriptionType(SubscriptionType.Failover);
    Consumer<byte[]> consumer1 = consumerBuilder.clone().consumerName("Consumer-1").subscribe();
    Consumer<byte[]> consumer2 = consumerBuilder.clone().consumerName("Consumer-2").subscribe();

    // 3. Producer publishes messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
        log.info("Producer produced " + message);
    }

    // 4. Receive messages
    int receivedConsumer1 = 0, receivedConsumer2 = 0;
    Message<byte[]> message1;
    Message<byte[]> message2;
    do {
        message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
        if (message1 != null) {
            log.info("Consumer 1 Received: " + new String(message1.getData()));
            receivedConsumer1 += 1;
        }
    } while (message1 != null);
    log.info("Consumer 1 receives = " + receivedConsumer1);
    log.info("Consumer 2 receives = " + receivedConsumer2);
    log.info("Total receives = " + (receivedConsumer2 + receivedConsumer1));
    assertEquals(receivedConsumer2 + receivedConsumer1, totalMessages);
    // Consumer 2 is on Stand By
    assertEquals(receivedConsumer2, 0);

    // 5. Consumer 2 asks for a redelivery but the request is ignored
    log.info("Consumer 2 asks for resend");
    consumer2.redeliverUnacknowledgedMessages();
    Thread.sleep(1000);

    message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
    message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
    assertNull(message1);
    assertNull(message2);
}
 
Example 20
Source Project: pulsar   File: PulsarFunctionsTest.java    License: Apache License 2.0 4 votes vote down vote up
private static void publishAndConsumeMessages(String inputTopic,
                                              String outputTopic,
                                              int numMessages) throws Exception {
    @Cleanup PulsarClient client = PulsarClient.builder()
        .serviceUrl(pulsarCluster.getPlainTextServiceUrl())
        .build();

    @Cleanup Consumer<String> consumer = client.newConsumer(Schema.STRING)
        .topic(outputTopic)
        .subscriptionType(SubscriptionType.Exclusive)
        .subscriptionName("test-sub")
        .subscribe();

    if (inputTopic.endsWith(".*")) {
        @Cleanup Producer<String> producer1 = client.newProducer(Schema.STRING)
                .topic(inputTopic.substring(0, inputTopic.length() - 2) + "1")
                .create();

        @Cleanup Producer<String> producer2 = client.newProducer(Schema.STRING)
                .topic(inputTopic.substring(0, inputTopic.length() - 2) + "2")
                .create();

        for (int i = 0; i < numMessages / 2; i++) {
            producer1.send("message-" + i);
        }

        for (int i = numMessages / 2; i < numMessages; i++) {
            producer2.send("message-" + i);
        }
    } else {
        @Cleanup Producer<String> producer = client.newProducer(Schema.STRING)
                .topic(inputTopic)
                .create();

        for (int i = 0; i < numMessages; i++) {
            producer.send("message-" + i);
        }
    }

    Set<String> expectedMessages = new HashSet<>();
    for (int i = 0; i < numMessages; i++) {
        expectedMessages.add("message-" + i + "!");
    }

    for (int i = 0; i < numMessages; i++) {
        Message<String> msg = consumer.receive(30, TimeUnit.SECONDS);
        log.info("Received: {}", msg.getValue());
        assertTrue(expectedMessages.contains(msg.getValue()));
        expectedMessages.remove(msg.getValue());
    }
}