Java Code Examples for org.apache.pulsar.client.api.Consumer#acknowledge()

The following examples show how to use org.apache.pulsar.client.api.Consumer#acknowledge() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: BacklogQuotaManagerTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Test
public void testConsumerBacklogEvictionWithAck() throws Exception {
    assertEquals(admin.namespaces().getBacklogQuotaMap("prop/ns-quota"),
            ConfigHelper.backlogQuotaMap(config));
    admin.namespaces().setBacklogQuota("prop/ns-quota",
            new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.consumer_backlog_eviction));
    PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).build();

    final String topic1 = "persistent://prop/ns-quota/topic11";
    final String subName1 = "c11";
    final String subName2 = "c21";
    final int numMsgs = 20;

    Consumer<byte[]> consumer1 = client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe();
    Consumer<byte[]> consumer2 = client.newConsumer().topic(topic1).subscriptionName(subName2).subscribe();
    org.apache.pulsar.client.api.Producer<byte[]> producer = client.newProducer().topic(topic1).create();
    byte[] content = new byte[1024];
    for (int i = 0; i < numMsgs; i++) {
        producer.send(content);
        // only one consumer acknowledges the message
        consumer1.acknowledge(consumer1.receive());
        consumer2.receive();
    }

    Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
    rolloverStats();

    TopicStats stats = admin.topics().getStats(topic1);
    assertTrue(stats.backlogSize <= 10 * 1024, "Storage size is [" + stats.storageSize + "]");
    client.close();
}
 
Example 2
Source File: MessagingBase.java    From pulsar with Apache License 2.0 5 votes vote down vote up
protected <T> void receiveMessagesCheckStickyKeyAndDuplicate
        (List<Consumer<T>> consumerList, int messagesToReceive) throws PulsarClientException {
    Map<String, Set<String>> consumerKeys = Maps.newHashMap();
    Set<T> messagesReceived = Sets.newHashSet();
    for (Consumer<T> consumer : consumerList) {
        Message<T> currentReceived;
        while (true) {
            try {
                currentReceived = consumer.receive(3, TimeUnit.SECONDS);
            } catch (PulsarClientException e) {
                log.info("no more messages to receive for consumer {}", consumer.getConsumerName());
                break;
            }
            if (currentReceived != null) {
                consumer.acknowledge(currentReceived);
                assertNotNull(currentReceived.getKey());
                consumerKeys.putIfAbsent(consumer.getConsumerName(), Sets.newHashSet());
                consumerKeys.get(consumer.getConsumerName()).add(currentReceived.getKey());
                // Make sure that there are no duplicates
                assertTrue(messagesReceived.add(currentReceived.getValue()),
                        "Received duplicate message " + currentReceived.getValue());
            } else {
                break;
            }
        }
    }
    // Make sure key will not be distributed to multiple consumers
    Set<String> allKeys = Sets.newHashSet();
    consumerKeys.forEach((k, v) -> v.forEach(key -> {
        assertTrue(allKeys.add(key),
                "Key "+ key +  "is distributed to multiple consumers" );
    }));
    assertEquals(messagesReceived.size(), messagesToReceive);
}
 
Example 3
Source File: ProxyStatsTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
/**
 * Validates proxy connection stats api.
 * 
 * @throws Exception
 */
@Test
public void testConnectionsStats() throws Exception {
    final String topicName1 = "persistent://sample/test/local/connections-stats";
    PulsarClient client = PulsarClient.builder().serviceUrl(proxyService.getServiceUrl()).build();
    Producer<byte[]> producer = client.newProducer(Schema.BYTES).topic(topicName1).enableBatching(false)
            .messageRoutingMode(MessageRoutingMode.SinglePartition).create();

    // Create a consumer directly attached to broker
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName1).subscriptionName("my-sub").subscribe();

    int totalMessages = 10;
    for (int i = 0; i < totalMessages; i++) {
        producer.send("test".getBytes());
    }

    for (int i = 0; i < totalMessages; i++) {
        Message<byte[]> msg = consumer.receive(1, TimeUnit.SECONDS);
        checkNotNull(msg);
        consumer.acknowledge(msg);
    }

    Client httpClient = ClientBuilder.newClient(new ClientConfig().register(LoggingFeature.class));
    Response r = httpClient.target(proxyWebServer.getServiceUri()).path("/proxy-stats/connections").request()
            .get();
    Assert.assertEquals(r.getStatus(), Response.Status.OK.getStatusCode());
    String response = r.readEntity(String.class).trim();
    List<ConnectionStats> connectionStats = new Gson().fromJson(response, new TypeToken<List<ConnectionStats>>() {
    }.getType());

    assertNotNull(connectionStats);

    consumer.close();
    client.close();
}
 
Example 4
Source File: PersistentTopicE2ETest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Test
public void testSimpleCloseTopic() throws Exception {
    final String topicName = "persistent://prop/ns-abc/topic5";
    final String subName = "sub5";

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();
    Producer<byte[]> producer = pulsarClient.newProducer()
        .topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();

    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();
    assertNotNull(topicRef);
    PersistentSubscription subRef = topicRef.getSubscription(subName);
    assertNotNull(subRef);

    Message<byte[]> msg;
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        msg = consumer.receive();
        consumer.acknowledge(msg);
    }

    producer.close();
    consumer.close();

    topicRef.close().get();
    assertFalse(pulsar.getBrokerService().getTopicReference(topicName).isPresent());
}
 
Example 5
Source File: SourceTester.java    From pulsar with Apache License 2.0 5 votes vote down vote up
public void validateSourceResultAvro(Consumer<KeyValue<GenericRecord, GenericRecord>> consumer,
                                 int number, String eventType) throws Exception {
    int recordsNumber = 0;
    Message<KeyValue<GenericRecord, GenericRecord>> msg = consumer.receive(2, TimeUnit.SECONDS);
    while(msg != null) {
        recordsNumber ++;
        GenericRecord keyRecord = msg.getValue().getKey();
        Assert.assertNotNull(keyRecord.getFields());
        Assert.assertTrue(keyRecord.getFields().size() > 0);

        GenericRecord valueRecord = msg.getValue().getValue();
        Assert.assertNotNull(valueRecord.getFields());
        Assert.assertTrue(valueRecord.getFields().size() > 0);
        for (Field field : valueRecord.getFields()) {
            Assert.assertTrue(DEBEZIUM_FIELD_SET.contains(field.getName()));
        }

        if (eventType != null) {
            String op = valueRecord.getField("op").toString();
            Assert.assertEquals(this.eventContains(eventType, false), op);
        }
        consumer.acknowledge(msg);
        msg = consumer.receive(1, TimeUnit.SECONDS);
    }

    Assert.assertEquals(recordsNumber, number);
    log.info("Stop {} server container. topic: {} has {} records.", getSourceType(), consumer.getTopic(), recordsNumber);
}
 
Example 6
Source File: ConsumerUnitTest.java    From tutorials with MIT License 5 votes vote down vote up
public static void main(String[] args) throws IOException {
    // Create a Pulsar client instance. A single instance can be shared across many
    // producers and consumer within the same application
    PulsarClient client = PulsarClient.builder()
            .serviceUrl(SERVICE_URL)
            .build();

    //Configure consumer specific settings.
    Consumer<byte[]> consumer = client.newConsumer()
            .topic(TOPIC_NAME)
            // Allow multiple consumers to attach to the same subscription
            // and get messages dispatched as a queue
            .subscriptionType(SubscriptionType.Shared)
            .subscriptionName(SUBSCRIPTION_NAME)
            .subscribe();


    // Once the consumer is created, it can be used for the entire application lifecycle
    System.out.println("Created consumer for the topic "+ TOPIC_NAME);

    do {
        // Wait until a message is available
        Message<byte[]> msg = consumer.receive();

        // Extract the message as a printable string and then log
        String content = new String(msg.getData());
        System.out.println("Received message '"+content+"' with ID "+msg.getMessageId());

        // Acknowledge processing of the message so that it can be deleted
        consumer.acknowledge(msg);
    } while (true);
}
 
Example 7
Source File: BatchMessageIndexAckDisableTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Test
public void testBatchMessageIndexAckForSharedSubscription() throws PulsarClientException, ExecutionException, InterruptedException {
    final String topic = "testBatchMessageIndexAckForSharedSubscription";

    @Cleanup
    Consumer<Integer> consumer = pulsarClient.newConsumer(Schema.INT32)
        .topic(topic)
        .subscriptionName("sub")
        .receiverQueueSize(100)
        .subscriptionType(SubscriptionType.Shared)
        .ackTimeout(1, TimeUnit.SECONDS)
        .subscribe();

    @Cleanup
    Producer<Integer> producer = pulsarClient.newProducer(Schema.INT32)
        .topic(topic)
        .batchingMaxPublishDelay(50, TimeUnit.MILLISECONDS)
        .create();

    final int messages = 100;
    List<CompletableFuture<MessageId>> futures = new ArrayList<>(messages);
    for (int i = 0; i < messages; i++) {
        futures.add(producer.sendAsync(i));
    }
    FutureUtil.waitForAll(futures).get();

    for (int i = 0; i < messages; i++) {
        if (i % 2 == 0) {
            consumer.acknowledge(consumer.receive());
        }
    }

    List<Message<Integer>> received = new ArrayList<>(messages);
    for (int i = 0; i < messages; i++) {
        received.add(consumer.receive());
    }

    Assert.assertEquals(received.size(), 100);
}
 
Example 8
Source File: TopicsConsumerImplTest.java    From pulsar with Apache License 2.0 4 votes vote down vote up
@Test(timeOut = testTimeout)
public void multiTopicsInDifferentNameSpace() throws PulsarAdminException, PulsarClientException {
    List<String> topics = new ArrayList<>();
    topics.add("persistent://prop/use/ns-abc/topic-1");
    topics.add("persistent://prop/use/ns-abc/topic-2");
    topics.add("persistent://prop/use/ns-abc1/topic-3");
    admin.clusters().createCluster("use", new ClusterData(brokerUrl.toString()));
    admin.tenants().createTenant("prop", new TenantInfo(null, Sets.newHashSet("use")));
    admin.namespaces().createNamespace("prop/use/ns-abc");
    admin.namespaces().createNamespace("prop/use/ns-abc1");
    Consumer consumer = pulsarClient.newConsumer()
            .topics(topics)
            .subscriptionName("multiTopicSubscription")
            .subscriptionType(SubscriptionType.Exclusive)
            .subscribe();
    // create Producer
    Producer<String> producer = pulsarClient.newProducer(Schema.STRING)
            .topic("persistent://prop/use/ns-abc/topic-1")
            .producerName("producer")
            .create();
    Producer<String> producer1 = pulsarClient.newProducer(Schema.STRING)
            .topic("persistent://prop/use/ns-abc/topic-2")
            .producerName("producer1")
            .create();
    Producer<String> producer2 = pulsarClient.newProducer(Schema.STRING)
            .topic("persistent://prop/use/ns-abc1/topic-3")
            .producerName("producer2")
            .create();
    //send message
    producer.send("ns-abc/topic-1-Message1");

    producer1.send("ns-abc/topic-2-Message1");

    producer2.send("ns-abc1/topic-3-Message1");

    int messageSet = 0;
    Message<byte[]> message = consumer.receive();
    do {
        messageSet ++;
        consumer.acknowledge(message);
        log.info("Consumer acknowledged : " + new String(message.getData()));
        message = consumer.receive(200, TimeUnit.MILLISECONDS);
    } while (message != null);
    assertEquals(messageSet, 3);

    consumer.unsubscribe();
    consumer.close();
    producer.close();
    producer1.close();
    producer2.close();
}
 
Example 9
Source File: SchemaCompatibilityCheckTest.java    From pulsar with Apache License 2.0 4 votes vote down vote up
@Test(dataProvider =  "AllCheckSchemaCompatibilityStrategy")
public void testIsAutoUpdateSchema(SchemaCompatibilityStrategy schemaCompatibilityStrategy) throws Exception {
    final String tenant = PUBLIC_TENANT;
    final String topic = "test-consumer-compatibility";

    String namespace = "test-namespace-" + randomName(16);
    String fqtn = TopicName.get(
            TopicDomain.persistent.value(),
            tenant,
            namespace,
            topic
    ).toString();

    NamespaceName namespaceName = NamespaceName.get(tenant, namespace);

    admin.namespaces().createNamespace(
            tenant + "/" + namespace,
            Sets.newHashSet(CLUSTER_NAME)
    );

    assertEquals(admin.namespaces().getSchemaCompatibilityStrategy(namespaceName.toString()),
            SchemaCompatibilityStrategy.FULL);
    
    admin.namespaces().setSchemaCompatibilityStrategy(namespaceName.toString(), schemaCompatibilityStrategy);
    admin.schemas().createSchema(fqtn, Schema.AVRO(Schemas.PersonOne.class).getSchemaInfo());

    admin.namespaces().setIsAllowAutoUpdateSchema(namespaceName.toString(), false);
    ProducerBuilder<Schemas.PersonTwo> producerThreeBuilder = pulsarClient
            .newProducer(Schema.AVRO(SchemaDefinition.<Schemas.PersonTwo>builder().withAlwaysAllowNull
                    (false).withSupportSchemaVersioning(true).
                    withPojo(Schemas.PersonTwo.class).build()))
            .topic(fqtn);
    try {
        producerThreeBuilder.create();
    } catch (Exception e) {
        Assert.assertTrue(e.getMessage().contains("Schema not found and schema auto updating is disabled."));
    }

    admin.namespaces().setIsAllowAutoUpdateSchema(namespaceName.toString(), true);
    ConsumerBuilder<Schemas.PersonTwo> comsumerBuilder = pulsarClient.newConsumer(Schema.AVRO(
            SchemaDefinition.<Schemas.PersonTwo>builder().withAlwaysAllowNull
                    (false).withSupportSchemaVersioning(true).
                    withPojo(Schemas.PersonTwo.class).build()))
            .subscriptionName("test")
            .topic(fqtn);

    Producer<Schemas.PersonTwo> producer = producerThreeBuilder.create();
    Consumer<Schemas.PersonTwo> consumerTwo = comsumerBuilder.subscribe();

    producer.send(new Schemas.PersonTwo(2, "Lucy"));
    Message<Schemas.PersonTwo> message = consumerTwo.receive();

    Schemas.PersonTwo personTwo = message.getValue();
    consumerTwo.acknowledge(message);

    assertEquals(personTwo.getId(), 2);
    assertEquals(personTwo.getName(), "Lucy");

    producer.close();
    consumerTwo.close();

    admin.namespaces().setIsAllowAutoUpdateSchema(namespaceName.toString(), false);

    producer = producerThreeBuilder.create();
    consumerTwo = comsumerBuilder.subscribe();

    producer.send(new Schemas.PersonTwo(2, "Lucy"));
    message = consumerTwo.receive();

    personTwo = message.getValue();
    consumerTwo.acknowledge(message);

    assertEquals(personTwo.getId(), 2);
    assertEquals(personTwo.getName(), "Lucy");

    consumerTwo.close();
    producer.close();
}
 
Example 10
Source File: PrometheusMetricsTest.java    From pulsar with Apache License 2.0 4 votes vote down vote up
@Test
public void testPerNamespaceStats() throws Exception {
    Producer<byte[]> p1 = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1").create();
    Producer<byte[]> p2 = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic2").create();

    Consumer<byte[]> c1 = pulsarClient.newConsumer()
            .topic("persistent://my-property/use/my-ns/my-topic1")
            .subscriptionName("test")
            .subscribe();

    Consumer<byte[]> c2 = pulsarClient.newConsumer()
            .topic("persistent://my-property/use/my-ns/my-topic2")
            .subscriptionName("test")
            .subscribe();

    final int messages = 10;

    for (int i = 0; i < messages; i++) {
        String message = "my-message-" + i;
        p1.send(message.getBytes());
        p2.send(message.getBytes());
    }

    for (int i = 0; i < messages; i++) {
        c1.acknowledge(c1.receive());
        c2.acknowledge(c2.receive());
    }

    ByteArrayOutputStream statsOut = new ByteArrayOutputStream();
    PrometheusMetricsGenerator.generate(pulsar, false, false, statsOut);
    String metricsStr = new String(statsOut.toByteArray());

    Multimap<String, Metric> metrics = parseMetrics(metricsStr);

    metrics.entries().forEach(e -> {
        System.out.println(e.getKey() + ": " + e.getValue());
    });

    // There should be 1 metric aggregated per namespace
    List<Metric> cm = (List<Metric>) metrics.get("pulsar_storage_write_latency_le_1");
    assertEquals(cm.size(), 1);
    assertNull(cm.get(0).tags.get("topic"));
    assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns");

    cm = (List<Metric>) metrics.get("pulsar_producers_count");
    assertEquals(cm.size(), 2);
    assertNull(cm.get(1).tags.get("topic"));
    assertEquals(cm.get(1).tags.get("namespace"), "my-property/use/my-ns");

    cm = (List<Metric>) metrics.get("pulsar_in_bytes_total");
    assertEquals(cm.size(), 1);
    assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns");

    cm = (List<Metric>) metrics.get("pulsar_in_messages_total");
    assertEquals(cm.size(), 1);
    assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns");

    cm = (List<Metric>) metrics.get("pulsar_out_bytes_total");
    assertEquals(cm.size(), 1);
    assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns");

    cm = (List<Metric>) metrics.get("pulsar_out_messages_total");
    assertEquals(cm.size(), 1);
    assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns");

    p1.close();
    p2.close();
    c1.close();
    c2.close();
}
 
Example 11
Source File: AdminApiTest.java    From pulsar with Apache License 2.0 4 votes vote down vote up
@Test
public void persistentTopicsInvalidCursorReset() throws Exception {
    admin.namespaces().setRetention("prop-xyz/ns1", new RetentionPolicies(10, 10));

    assertEquals(admin.topics().getList("prop-xyz/ns1"), Lists.newArrayList());

    String topicName = "persistent://prop-xyz/ns1/invalidcursorreset";
    // Force to create a topic
    publishMessagesOnPersistentTopic(topicName, 0);
    assertEquals(admin.topics().getList("prop-xyz/ns1"), Lists.newArrayList(topicName));

    // create consumer and subscription
    PulsarClient client = PulsarClient.builder()
            .serviceUrl(pulsar.getWebServiceAddress())
            .statsInterval(0, TimeUnit.SECONDS)
            .build();
    Consumer<byte[]> consumer = client.newConsumer().topic(topicName).subscriptionName("my-sub")
            .subscriptionType(SubscriptionType.Exclusive).subscribe();

    assertEquals(admin.topics().getSubscriptions(topicName), Lists.newArrayList("my-sub"));

    publishMessagesOnPersistentTopic(topicName, 10);

    List<Message<byte[]>> messages = admin.topics().peekMessages(topicName, "my-sub", 10);
    assertEquals(messages.size(), 10);

    for (int i = 0; i < 10; i++) {
        Message<byte[]> message = consumer.receive();
        consumer.acknowledge(message);
    }
    // use invalid timestamp
    try {
        admin.topics().resetCursor(topicName, "my-sub", System.currentTimeMillis() - 190000);
    } catch (Exception e) {
        // fail the test
        throw e;
    }

    admin.topics().resetCursor(topicName, "my-sub", System.currentTimeMillis() + 90000);
    consumer = client.newConsumer().topic(topicName).subscriptionName("my-sub").subscribe();
    consumer.close();
    client.close();

    admin.topics().deleteSubscription(topicName, "my-sub");

    assertEquals(admin.topics().getSubscriptions(topicName), Lists.newArrayList());
    admin.topics().delete(topicName);
}
 
Example 12
Source File: V1_AdminApiTest.java    From pulsar with Apache License 2.0 4 votes vote down vote up
@Test
public void persistentTopicsInvalidCursorReset() throws Exception {
    admin.namespaces().setRetention("prop-xyz/use/ns1", new RetentionPolicies(10, 10));

    assertEquals(admin.topics().getList("prop-xyz/use/ns1"), Lists.newArrayList());

    String topicName = "persistent://prop-xyz/use/ns1/invalidcursorreset";
    // Force to create a topic
    publishMessagesOnPersistentTopic(topicName, 0);
    assertEquals(admin.topics().getList("prop-xyz/use/ns1"), Lists.newArrayList(topicName));

    // create consumer and subscription
    PulsarClient client = PulsarClient.builder()
            .serviceUrl(pulsar.getWebServiceAddress())
            .statsInterval(0, TimeUnit.SECONDS)
            .build();
    Consumer<byte[]> consumer = client.newConsumer().topic(topicName).subscriptionName("my-sub")
            .subscriptionType(SubscriptionType.Exclusive).subscribe();

    assertEquals(admin.topics().getSubscriptions(topicName), Lists.newArrayList("my-sub"));

    publishMessagesOnPersistentTopic(topicName, 10);

    List<Message<byte[]>> messages = admin.topics().peekMessages(topicName, "my-sub", 10);
    assertEquals(messages.size(), 10);

    for (int i = 0; i < 10; i++) {
        Message<byte[]> message = consumer.receive();
        consumer.acknowledge(message);
    }
    // use invalid timestamp
    try {
        admin.topics().resetCursor(topicName, "my-sub", System.currentTimeMillis() - 190000);
    } catch (Exception e) {
        // fail the test
        throw e;
    }

    admin.topics().resetCursor(topicName, "my-sub", System.currentTimeMillis() + 90000);
    consumer = client.newConsumer().topic(topicName).subscriptionName("my-sub").subscribe();
    consumer.close();
    client.close();

    admin.topics().deleteSubscription(topicName, "my-sub");

    assertEquals(admin.topics().getSubscriptions(topicName), Lists.newArrayList());
    admin.topics().delete(topicName);
}
 
Example 13
Source File: CurrentLedgerRolloverIfFullTest.java    From pulsar with Apache License 2.0 4 votes vote down vote up
@Test
public void testCurrentLedgerRolloverIfFull() throws Exception {
    super.baseSetup();
    final String topicName = "persistent://prop/ns-abc/CurrentLedgerRolloverIfFullTest";

    @Cleanup
    Producer<byte[]> producer = pulsarClient.newProducer()
            .topic(topicName)
            .producerName("CurrentLedgerRolloverIfFullTest-producer-name")
            .create();

    @Cleanup
    Consumer<byte[]> consumer = pulsarClient.newConsumer()
            .topic(topicName)
            .subscriptionName("CurrentLedgerRolloverIfFullTest-subscriber-name")
            .subscribe();

    Topic topicRef = pulsar.getBrokerService().getTopicReference(topicName).get();
    Assert.assertNotNull(topicRef);
    PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName).get();

    ManagedLedgerConfig managedLedgerConfig = persistentTopic.getManagedLedger().getConfig();
    managedLedgerConfig.setRetentionTime(1, TimeUnit.SECONDS);
    managedLedgerConfig.setMaxEntriesPerLedger(2);
    managedLedgerConfig.setMinimumRolloverTime(1, TimeUnit.MILLISECONDS);
    managedLedgerConfig.setMaximumRolloverTime(5, TimeUnit.MILLISECONDS);

    int msgNum = 10;
    for (int i = 0; i < msgNum; i++) {
        producer.send(new byte[1024 * 1024]);
    }

    ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) persistentTopic.getManagedLedger();
    Assert.assertEquals(managedLedger.getLedgersInfoAsList().size(), msgNum / 2);

    for (int i = 0; i < msgNum; i++) {
        Message<byte[]> msg = consumer.receive(2, TimeUnit.SECONDS);
        Assert.assertTrue(msg != null);
        consumer.acknowledge(msg);
    }

    // all the messages have been acknowledged
    // and all the ledgers have been removed except the the last ledger
    Thread.sleep(500);
    Assert.assertEquals(managedLedger.getLedgersInfoAsList().size(), 1);
    Assert.assertNotEquals(managedLedger.getCurrentLedgerSize(), 0);

    // trigger a ledger rollover
    // and now we have two ledgers, one with expired data and one for empty
    managedLedger.rollCurrentLedgerIfFull();
    Thread.sleep(1000);
    Assert.assertEquals(managedLedger.getLedgersInfoAsList().size(), 2);

    // trigger a ledger trimming
    // and now we only have the empty ledger
    managedLedger.trimConsumedLedgersInBackground(Futures.NULL_PROMISE);
    Assert.assertEquals(managedLedger.getCurrentLedgerSize(), 0);
}
 
Example 14
Source File: PatternTopicsConsumerImplTest.java    From pulsar with Apache License 2.0 4 votes vote down vote up
@Test(timeOut = testTimeout)
public void testStartEmptyPatternConsumer() throws Exception {
    String key = "StartEmptyPatternConsumerTest";
    String subscriptionName = "my-ex-subscription-" + key;
    String topicName1 = "persistent://my-property/my-ns/pattern-topic-1-" + key;
    String topicName2 = "persistent://my-property/my-ns/pattern-topic-2-" + key;
    String topicName3 = "persistent://my-property/my-ns/pattern-topic-3-" + key;
    Pattern pattern = Pattern.compile("persistent://my-property/my-ns/pattern-topic.*");

    // 1. create partition
    TenantInfo tenantInfo = createDefaultTenantInfo();
    admin.tenants().createTenant("prop", tenantInfo);
    admin.topics().createPartitionedTopic(topicName2, 2);
    admin.topics().createPartitionedTopic(topicName3, 3);

    // 2. Create consumer, this should success, but with empty sub-consumser internal
    Consumer<byte[]> consumer = pulsarClient.newConsumer()
        .topicsPattern(pattern)
        .patternAutoDiscoveryPeriod(2)
        .subscriptionName(subscriptionName)
        .subscriptionType(SubscriptionType.Shared)
        .ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS)
        .receiverQueueSize(4)
        .subscribe();

    // 3. verify consumer get methods, to get 5 number of partitions and topics.
    assertSame(pattern, ((PatternMultiTopicsConsumerImpl<?>) consumer).getPattern());
    assertEquals(((PatternMultiTopicsConsumerImpl<?>) consumer).getPartitionedTopics().size(), 5);
    assertEquals(((PatternMultiTopicsConsumerImpl<?>) consumer).getConsumers().size(), 5);
    assertEquals(((PatternMultiTopicsConsumerImpl<?>) consumer).getTopics().size(), 2);

    // 4. create producer
    String messagePredicate = "my-message-" + key + "-";
    int totalMessages = 30;

    Producer<byte[]> producer1 = pulsarClient.newProducer().topic(topicName1)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();
    Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topicName2)
        .enableBatching(false)
        .messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition)
        .create();
    Producer<byte[]> producer3 = pulsarClient.newProducer().topic(topicName3)
        .enableBatching(false)
        .messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition)
        .create();

    // 5. call recheckTopics to subscribe each added topics above
    log.debug("recheck topics change");
    PatternMultiTopicsConsumerImpl<byte[]> consumer1 = ((PatternMultiTopicsConsumerImpl<byte[]>) consumer);
    consumer1.run(consumer1.getRecheckPatternTimeout());
    Thread.sleep(100);

    // 6. verify consumer get methods, to get number of partitions and topics, value 6=1+2+3.
    assertSame(pattern, ((PatternMultiTopicsConsumerImpl<?>) consumer).getPattern());
    assertEquals(((PatternMultiTopicsConsumerImpl<?>) consumer).getPartitionedTopics().size(), 6);
    assertEquals(((PatternMultiTopicsConsumerImpl<?>) consumer).getConsumers().size(), 6);
    assertEquals(((PatternMultiTopicsConsumerImpl<?>) consumer).getTopics().size(), 3);


    // 7. produce data
    for (int i = 0; i < totalMessages / 3; i++) {
        producer1.send((messagePredicate + "producer1-" + i).getBytes());
        producer2.send((messagePredicate + "producer2-" + i).getBytes());
        producer3.send((messagePredicate + "producer3-" + i).getBytes());
    }

    // 8. should receive all the message
    int messageSet = 0;
    Message<byte[]> message = consumer.receive();
    do {
        assertTrue(message instanceof TopicMessageImpl);
        messageSet ++;
        consumer.acknowledge(message);
        log.debug("Consumer acknowledged : " + new String(message.getData()));
        message = consumer.receive(500, TimeUnit.MILLISECONDS);
    } while (message != null);
    assertEquals(messageSet, totalMessages);

    consumer.unsubscribe();
    consumer.close();
    producer1.close();
    producer2.close();
    producer3.close();
}
 
Example 15
Source File: ConsumedLedgersTrimTest.java    From pulsar with Apache License 2.0 4 votes vote down vote up
@Test
public void TestConsumedLedgersTrim() throws Exception {
    conf.setRetentionCheckIntervalInSeconds(1);
    super.baseSetup();
    final String topicName = "persistent://prop/ns-abc/TestConsumedLedgersTrim";
    final String subscriptionName = "my-subscriber-name";

    @Cleanup
    Producer<byte[]> producer = pulsarClient.newProducer()
            .topic(topicName)
            .producerName("producer-name")
            .create();
    @Cleanup
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
            .subscribe();
    Topic topicRef = pulsar.getBrokerService().getTopicReference(topicName).get();
    Assert.assertNotNull(topicRef);
    PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName).get();

    ManagedLedgerConfig managedLedgerConfig = persistentTopic.getManagedLedger().getConfig();
    managedLedgerConfig.setRetentionSizeInMB(1L);
    managedLedgerConfig.setRetentionTime(1, TimeUnit.SECONDS);
    managedLedgerConfig.setMaxEntriesPerLedger(2);
    managedLedgerConfig.setMinimumRolloverTime(1, TimeUnit.MILLISECONDS);

    int msgNum = 10;
    for (int i = 0; i < msgNum; i++) {
        producer.send(new byte[1024 * 1024]);
    }

    ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) persistentTopic.getManagedLedger();
    Assert.assertEquals(managedLedger.getLedgersInfoAsList().size(), msgNum / 2);

    //no traffic, unconsumed ledger will be retained
    Thread.sleep(1200);
    Assert.assertEquals(managedLedger.getLedgersInfoAsList().size(), msgNum / 2);

    for (int i = 0; i < msgNum; i++) {
        Message<byte[]> msg = consumer.receive(2, TimeUnit.SECONDS);
        Assert.assertTrue(msg != null);
        consumer.acknowledge(msg);
    }
    Assert.assertEquals(managedLedger.getLedgersInfoAsList().size(), msgNum / 2);

    //no traffic, but consumed ledger will be cleaned
    Thread.sleep(1500);
    Assert.assertEquals(managedLedger.getLedgersInfoAsList().size(), 1);
}
 
Example 16
Source File: CmdConsume.java    From pulsar with Apache License 2.0 4 votes vote down vote up
private int consume(String topic) {
    int numMessagesConsumed = 0;
    int returnCode = 0;

    try {
        PulsarClient client = clientBuilder.build();
        ConsumerBuilder<byte[]> builder = client.newConsumer()
                .subscriptionName(this.subscriptionName)
                .subscriptionType(subscriptionType)
                .subscriptionInitialPosition(subscriptionInitialPosition);

        if (isRegex) {
            builder.topicsPattern(Pattern.compile(topic));
        } else {
            builder.topic(topic);
        }

        ConsumerBuilder<byte[]> consumerBuilder = client.newConsumer().topic(topic);
        if (this.maxPendingChuckedMessage > 0) {
            consumerBuilder.maxPendingChuckedMessage(this.maxPendingChuckedMessage);
        }
        if (this.receiverQueueSize > 0) {
            consumerBuilder.maxPendingChuckedMessage(this.receiverQueueSize);
        }
        Consumer<byte[]> consumer = consumerBuilder.subscriptionName(this.subscriptionName)
                .autoAckOldestChunkedMessageOnQueueFull(this.autoAckOldestChunkedMessageOnQueueFull)
                .subscriptionType(subscriptionType).subscribe();

        RateLimiter limiter = (this.consumeRate > 0) ? RateLimiter.create(this.consumeRate) : null;
        while (this.numMessagesToConsume == 0 || numMessagesConsumed < this.numMessagesToConsume) {
            if (limiter != null) {
                limiter.acquire();
            }

            Message<byte[]> msg = consumer.receive(5, TimeUnit.SECONDS);
            if (msg == null) {
                LOG.debug("No message to consume after waiting for 5 seconds.");
            } else {
                numMessagesConsumed += 1;
                System.out.println(MESSAGE_BOUNDARY);
                String output = this.interpretMessage(msg, displayHex);
                System.out.println(output);
                consumer.acknowledge(msg);
            }
        }
        client.close();
    } catch (Exception e) {
        LOG.error("Error while consuming messages");
        LOG.error(e.getMessage(), e);
        returnCode = -1;
    } finally {
        LOG.info("{} messages successfully consumed", numMessagesConsumed);
    }

    return returnCode;

}
 
Example 17
Source File: PersistentTopicE2ETest.java    From pulsar with Apache License 2.0 4 votes vote down vote up
/**
 * Validation: 1. validates active-cursor after active subscription 2. validate active-cursor with subscription 3.
 * unconsumed messages should be present into cache 4. cache and active-cursor should be empty once subscription is
 * closed
 *
 * @throws Exception
 */
@Test
public void testActiveSubscriptionWithCache() throws Exception {
    final String topicName = "persistent://prop/ns-abc/topic2";
    final String subName = "sub2";

    Message<byte[]> msg;
    int recvQueueSize = 4;

    // (1) Create subscription
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
            .receiverQueueSize(recvQueueSize).subscribe();
    Producer<byte[]> producer = pulsarClient.newProducer()
        .topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();

    // (2) Produce Messages
    for (int i = 0; i < recvQueueSize / 2; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        msg = consumer.receive();
        consumer.acknowledge(msg);
    }

    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();

    // (3) Get Entry cache
    ManagedLedgerImpl ledger = (ManagedLedgerImpl) topicRef.getManagedLedger();
    Field cacheField = ManagedLedgerImpl.class.getDeclaredField("entryCache");
    cacheField.setAccessible(true);
    EntryCacheImpl entryCache = (EntryCacheImpl) cacheField.get(ledger);

    /************* Validation on non-empty active-cursor **************/
    // (4) Get ActiveCursor : which is list of active subscription
    Iterable<ManagedCursor> activeCursors = ledger.getActiveCursors();
    ManagedCursor curosr = activeCursors.iterator().next();
    // (4.1) Validate: active Cursor must be non-empty
    assertNotNull(curosr);
    // (4.2) Validate: validate cursor name
    assertEquals(subName, curosr.getName());

    /************* Validation on empty active-cursor **************/
    // (5) Close consumer: which (1)removes activeConsumer and (2)clears the entry-cache
    consumer.close();
    Thread.sleep(1000);
    // (5.1) Validate: active-consumer must be empty
    assertFalse(ledger.getActiveCursors().iterator().hasNext());
    // (5.2) Validate: Entry-cache must be cleared
    assertEquals(entryCache.getSize(), 0);

}
 
Example 18
Source File: BrokerServiceTest.java    From pulsar with Apache License 2.0 4 votes vote down vote up
@Test
public void testBrokerServicePersistentRedeliverTopicStats() throws Exception {
    final String topicName = "persistent://prop/ns-abc/successSharedTopic";
    final String subName = "successSharedSub";

    TopicStats stats;
    SubscriptionStats subStats;

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
            .subscriptionType(SubscriptionType.Shared).acknowledgmentGroupTime(0, TimeUnit.SECONDS).subscribe();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);

    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();
    assertNotNull(topicRef);

    rolloverPerIntervalStats();
    stats = topicRef.getStats(false);
    subStats = stats.subscriptions.values().iterator().next();

    // subscription stats
    assertEquals(stats.subscriptions.keySet().size(), 1);
    assertEquals(subStats.msgBacklog, 0);
    assertEquals(subStats.consumers.size(), 1);

    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);

    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);

    rolloverPerIntervalStats();
    stats = topicRef.getStats(false);
    subStats = stats.subscriptions.values().iterator().next();

    // publisher stats
    assertEquals(subStats.msgBacklog, 10);
    assertEquals(stats.publishers.size(), 1);
    assertTrue(stats.publishers.get(0).msgRateIn > 0.0);
    assertTrue(stats.publishers.get(0).msgThroughputIn > 0.0);
    assertTrue(stats.publishers.get(0).averageMsgSize > 0.0);

    // aggregated publish stats
    assertEquals(stats.msgRateIn, stats.publishers.get(0).msgRateIn);
    assertEquals(stats.msgThroughputIn, stats.publishers.get(0).msgThroughputIn);
    double diff = stats.averageMsgSize - stats.publishers.get(0).averageMsgSize;
    assertTrue(Math.abs(diff) < 0.000001);

    // consumer stats
    assertTrue(subStats.consumers.get(0).msgRateOut > 0.0);
    assertTrue(subStats.consumers.get(0).msgThroughputOut > 0.0);
    assertEquals(subStats.msgRateRedeliver, 0.0);
    assertEquals(subStats.consumers.get(0).unackedMessages, 10);

    // aggregated consumer stats
    assertEquals(subStats.msgRateOut, subStats.consumers.get(0).msgRateOut);
    assertEquals(subStats.msgThroughputOut, subStats.consumers.get(0).msgThroughputOut);
    assertEquals(subStats.msgRateRedeliver, subStats.consumers.get(0).msgRateRedeliver);
    assertEquals(stats.msgRateOut, subStats.consumers.get(0).msgRateOut);
    assertEquals(stats.msgThroughputOut, subStats.consumers.get(0).msgThroughputOut);
    assertEquals(subStats.msgRateRedeliver, subStats.consumers.get(0).msgRateRedeliver);
    assertEquals(subStats.unackedMessages, subStats.consumers.get(0).unackedMessages);

    consumer.redeliverUnacknowledgedMessages();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);

    rolloverPerIntervalStats();
    stats = topicRef.getStats(false);
    subStats = stats.subscriptions.values().iterator().next();
    assertTrue(subStats.msgRateRedeliver > 0.0);
    assertEquals(subStats.msgRateRedeliver, subStats.consumers.get(0).msgRateRedeliver);

    Message<byte[]> msg;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive();
        consumer.acknowledge(msg);
    }
    consumer.close();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);

    rolloverPerIntervalStats();
    stats = topicRef.getStats(false);
    subStats = stats.subscriptions.values().iterator().next();

    assertEquals(subStats.msgBacklog, 0);
}
 
Example 19
Source File: ProxyStatsTest.java    From pulsar with Apache License 2.0 4 votes vote down vote up
/**
 * Validate proxy topic stats api
 * 
 * @throws Exception
 */
@Test
public void testTopicStats() throws Exception {
    proxyService.setProxyLogLevel(2);
    final String topicName = "persistent://sample/test/local/topic-stats";
    final String topicName2 = "persistent://sample/test/local/topic-stats-2";

    PulsarClient client = PulsarClient.builder().serviceUrl(proxyService.getServiceUrl()).build();
    Producer<byte[]> producer1 = client.newProducer(Schema.BYTES).topic(topicName).enableBatching(false)
            .producerName("producer1").messageRoutingMode(MessageRoutingMode.SinglePartition).create();

    Producer<byte[]> producer2 = client.newProducer(Schema.BYTES).topic(topicName2).enableBatching(false)
            .producerName("producer2").messageRoutingMode(MessageRoutingMode.SinglePartition).create();

    // Create a consumer directly attached to broker
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").subscribe();
    Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic(topicName2).subscriptionName("my-sub")
            .subscribe();

    int totalMessages = 10;
    for (int i = 0; i < totalMessages; i++) {
        producer1.send("test".getBytes());
        producer2.send("test".getBytes());
    }

    for (int i = 0; i < totalMessages; i++) {
        Message<byte[]> msg = consumer.receive(1, TimeUnit.SECONDS);
        checkNotNull(msg);
        consumer.acknowledge(msg);
        msg = consumer2.receive(1, TimeUnit.SECONDS);
    }

    Client httpClient = ClientBuilder.newClient(new ClientConfig().register(LoggingFeature.class));
    Response r = httpClient.target(proxyWebServer.getServiceUri()).path("/proxy-stats/topics").request()
            .get();
    Assert.assertEquals(r.getStatus(), Response.Status.OK.getStatusCode());
    String response = r.readEntity(String.class).trim();
    Map<String, TopicStats> topicStats = new Gson().fromJson(response, new TypeToken<Map<String, TopicStats>>() {
    }.getType());

    assertNotNull(topicStats.get(topicName));

    consumer.close();
    consumer2.close();
    client.close();
}
 
Example 20
Source File: KafkaRequestTypeTest.java    From kop with Apache License 2.0 4 votes vote down vote up
@Test(timeOut = 20000, dataProvider = "partitionsAndBatch")
public void testKafkaProducePulsarConsume(int partitionNumber, boolean isBatch) throws Exception {
    String kafkaTopicName = "kopKafkaProducePulsarConsume" + partitionNumber;
    String pulsarTopicName = "persistent://public/default/" + kafkaTopicName;
    String key1 = "header_key1_";
    String key2 = "header_key2_";
    String value1 = "header_value1_";
    String value2 = "header_value2_";

    // create partitioned topic.
    admin.topics().createPartitionedTopic(kafkaTopicName, partitionNumber);

    @Cleanup
    Consumer<byte[]> consumer = pulsarClient.newConsumer()
        .topic(pulsarTopicName)
        .subscriptionName("test_k_producer_k_pconsumer_sub")
        .subscribe();

    // 1. produce message with Kafka producer.
    @Cleanup
    KProducer kProducer = new KProducer(kafkaTopicName, false, getKafkaBrokerPort());

    int totalMsgs = 10;
    String messageStrPrefix = "Message_Kop_KafkaProducePulsarConsume_"  + partitionNumber + "_";

    for (int i = 0; i < totalMsgs; i++) {
        String messageStr = messageStrPrefix + i;
        ProducerRecord record = new ProducerRecord<>(
            kafkaTopicName,
            i,
            messageStr);

        record.headers()
            .add(key1 + i, (value1 + i).getBytes(UTF_8))
            .add(key2 + i, (value2 + i).getBytes(UTF_8));

        if (isBatch) {
            kProducer.getProducer()
                .send(record);
        } else {
            kProducer.getProducer()
                .send(record)
                .get();
        }
        if (log.isDebugEnabled()) {
            log.debug("Kafka Producer Sent message with header: ({}, {})", i, messageStr);
        }
    }

    // 2. Consume messages use Pulsar client Consumer. verify content and key and headers
    Message<byte[]> msg = null;
    for (int i = 0; i < totalMsgs; i++) {
        msg = consumer.receive(100, TimeUnit.MILLISECONDS);
        assertNotNull(msg);
        Integer key = kafkaIntDeserialize(Base64.getDecoder().decode(msg.getKey()));
        assertEquals(messageStrPrefix + key.toString(), new String(msg.getValue()));

        // verify added 2 key-value pair
        Map<String, String> properties = msg.getProperties();
        assertEquals(properties.size(), 2);
        for (Map.Entry<String, String> kv: properties.entrySet()) {
            String k = kv.getKey();
            String v = kv.getValue();

            if (log.isDebugEnabled()) {
                log.debug("headers key: {}, value:{}", k, v);
            }

            assertTrue(k.contains(key1) || k.contains(key2));
            assertTrue(v.contains(value1) || v.contains(value2));
        }
        if (log.isDebugEnabled()) {
            log.debug("Pulsar consumer get message: {}, key: {}",
                new String(msg.getData()),
                kafkaIntDeserialize(Base64.getDecoder().decode(msg.getKey())).toString());
        }
        consumer.acknowledge(msg);
    }

    // verify have received all messages
    msg = consumer.receive(100, TimeUnit.MILLISECONDS);
    assertNull(msg);
}