Java Code Examples for org.apache.pulsar.client.api.Consumer

The following examples show how to use org.apache.pulsar.client.api.Consumer. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hazelcast-jet-contrib   Source File: PulsarTestSupport.java    License: Apache License 2.0 7 votes vote down vote up
protected static Consumer<Double> getConsumer(String topicName) throws PulsarClientException {
    if (!integerConsumerMap.containsKey(topicName)) {
        Consumer<Double> newConsumer = getClient()
                .newConsumer(Schema.DOUBLE)
                .topic(topicName)
                .consumerName("hazelcast-jet-consumer-" + topicName)
                .subscriptionName("hazelcast-jet-subscription")
                .subscriptionType(SubscriptionType.Exclusive)
                .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
                .receiverQueueSize(QUEUE_CAPACITY)
                .subscribe();
        integerConsumerMap.put(topicName, newConsumer);
        return newConsumer;
    } else {
        return integerConsumerMap.get(topicName);
    }
}
 
Example 2
Source Project: pulsar   Source File: TestCompaction.java    License: Apache License 2.0 6 votes vote down vote up
private static Map<Integer, String> consumeCompactedTopic(PulsarClient client,
                                                          String topic,
                                                          String subscription,
                                                          int numKeys) throws PulsarClientException {
    Map<Integer, String> keys = Maps.newHashMap();
    try (Consumer<byte[]> consumer = client.newConsumer()
         .readCompacted(true)
         .topic(topic)
         .subscriptionName(subscription)
         .subscribe()
    ) {
        for (int i = 0; i < numKeys; i++) {
            Message<byte[]> m = consumer.receive();
            keys.put(Integer.parseInt(m.getKey()), new String(m.getValue(), UTF_8));
        }
    }
    return keys;
}
 
Example 3
Source Project: pulsar   Source File: ConsumerImpl.java    License: Apache License 2.0 6 votes vote down vote up
static <T> ConsumerImpl<T> newConsumerImpl(PulsarClientImpl client,
                                           String topic,
                                           ConsumerConfigurationData<T> conf,
                                           ExecutorService listenerExecutor,
                                           int partitionIndex,
                                           boolean hasParentConsumer,
                                           CompletableFuture<Consumer<T>> subscribeFuture,
                                           MessageId startMessageId,
                                           Schema<T> schema,
                                           ConsumerInterceptors<T> interceptors,
                                           boolean createTopicIfDoesNotExist) {
    if (conf.getReceiverQueueSize() == 0) {
        return new ZeroQueueConsumerImpl<>(client, topic, conf, listenerExecutor, partitionIndex, hasParentConsumer,
                subscribeFuture,
                startMessageId, schema, interceptors,
                createTopicIfDoesNotExist);
    } else {
        return new ConsumerImpl<>(client, topic, conf, listenerExecutor, partitionIndex, hasParentConsumer,
                subscribeFuture, startMessageId, 0 /* rollback time in sec to start msgId */,
                schema, interceptors, createTopicIfDoesNotExist);
    }
}
 
Example 4
Source Project: pulsar   Source File: PulsarConsumerSourceTests.java    License: Apache License 2.0 6 votes vote down vote up
private TestPulsarConsumerSource createSource(Consumer<byte[]> testConsumer,
                                              long batchSize, boolean isCheckpointingEnabled) throws Exception {
    PulsarSourceBuilder<String> builder =
        PulsarSourceBuilder.builder(new SimpleStringSchema())
            .acknowledgementBatchSize(batchSize);
    TestPulsarConsumerSource source = new TestPulsarConsumerSource(builder, testConsumer, isCheckpointingEnabled);

    OperatorStateStore mockStore = mock(OperatorStateStore.class);
    FunctionInitializationContext mockContext = mock(FunctionInitializationContext.class);
    when(mockContext.getOperatorStateStore()).thenReturn(mockStore);
    when(mockStore.getSerializableListState(any(String.class))).thenReturn(null);

    source.initializeState(mockContext);

    return source;
}
 
Example 5
Source Project: pulsar   Source File: ConsumerConnector.java    License: Apache License 2.0 6 votes vote down vote up
private void resetOffsets(Consumer<byte[]> consumer, SubscriptionInitialPosition strategy) {
    if (strategy == null) {
        return;
    }
    log.info("Resetting partition {} for group-id {} and seeking to {} position", consumer.getTopic(),
            consumer.getSubscription(), strategy);
    try {
        if (strategy == SubscriptionInitialPosition.Earliest) {
            consumer.seek(MessageId.earliest);
        } else {
            consumer.seek(MessageId.latest);
        }
    } catch (PulsarClientException e) {
        log.warn("Failed to reset offset for consumer {} to {}, {}", consumer.getTopic(), strategy,
                e.getMessage(), e);
    }
}
 
Example 6
Source Project: pulsar   Source File: PulsarKafkaSimpleConsumer.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * <pre>
 * Overriden method: OffsetCommitResponse commitOffsets(OffsetCommitRequest request)
 * 
 * Note:
 * created PulsarOffsetCommitResponse as OffsetCommitRequest doesn't provide getters
 * 
 * </pre>
 */
public OffsetCommitResponse commitOffsets(PulsarOffsetCommitRequest request) {

    PulsarOffsetCommitResponse response = new PulsarOffsetCommitResponse(null);
    for (Entry<String, MessageId> topicOffset : request.getTopicOffsetMap().entrySet()) {
        final String topic = topicOffset.getKey();
        final String groupId = request.getGroupId();
        try {
            Consumer<byte[]> consumer = getConsumer(topic, groupId);
            consumer.acknowledgeCumulative(topicOffset.getValue());
        } catch (Exception e) {
            log.warn("Failed to ack message for topic {}-{}", topic, topicOffset.getValue(), e);
            response.hasError = true;
            TopicAndPartition topicPartition = new TopicAndPartition(topic, 0);
            response.errors.computeIfAbsent(topicPartition, tp -> ErrorMapping.UnknownCode());
        }
    }

    return response;
}
 
Example 7
Source Project: pulsar   Source File: CompactionTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCompactEmptyTopic() throws Exception {
    String topic = "persistent://my-property/use/my-ns/my-topic1";

    Producer<byte[]> producer = pulsarClient.newProducer()
        .topic(topic)
        .enableBatching(false)
        .create();

    pulsarClient.newConsumer().topic(topic).subscriptionName("sub1").readCompacted(true).subscribe().close();

    new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);

    producer.newMessage().key("key0").value("content0".getBytes()).send();

    try (Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("sub1")
            .readCompacted(true).subscribe()) {
        Message<byte[]> m = consumer.receive();
        Assert.assertEquals(m.getKey(), "key0");
        Assert.assertEquals(m.getData(), "content0".getBytes());
    }
}
 
Example 8
Source Project: pulsar   Source File: PulsarStateTest.java    License: Apache License 2.0 6 votes vote down vote up
private static <T> void ensureSubscriptionCreated(String inputTopicName,
                                                  String subscriptionName,
                                                  Schema<T> inputTopicSchema)
        throws Exception {
    // ensure the function subscription exists before we start producing messages
    try (PulsarClient client = PulsarClient.builder()
        .serviceUrl(container.getPlainTextServiceUrl())
        .build()) {
        try (Consumer<T> ignored = client.newConsumer(inputTopicSchema)
            .topic(inputTopicName)
            .subscriptionType(SubscriptionType.Shared)
            .subscriptionName(subscriptionName)
            .subscribe()) {
        }
    }
}
 
Example 9
Source Project: pulsar   Source File: TopicTerminationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test(timeOut = 20000)
public void testSubscribeOnTerminatedTopic() throws Exception {
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();
    /* MessageId msgId1 = */ producer.send("test-msg-1".getBytes());
    MessageId msgId2 = producer.send("test-msg-2".getBytes());

    MessageId lastMessageId = admin.topics().terminateTopicAsync(topicName).get();
    assertEquals(lastMessageId, msgId2);

    org.apache.pulsar.client.api.Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName)
            .subscriptionName("my-sub").subscribe();

    Thread.sleep(200);
    assertTrue(consumer.hasReachedEndOfTopic());
}
 
Example 10
Source Project: pulsar   Source File: ReplicatedSubscriptionConfigTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void createReplicatedSubscription() throws Exception {
    String topic = "createReplicatedSubscription-" + System.nanoTime();

    @Cleanup
    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING)
            .topic(topic)
            .subscriptionName("sub1")
            .replicateSubscriptionState(true)
            .subscribe();

    TopicStats stats = admin.topics().getStats(topic);
    assertTrue(stats.subscriptions.get("sub1").isReplicated);

    admin.topics().unload(topic);

    // Check that subscription is still marked replicated after reloading
    stats = admin.topics().getStats(topic);
    assertTrue(stats.subscriptions.get("sub1").isReplicated);
}
 
Example 11
Source Project: pulsar   Source File: SampleConsumer.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws PulsarClientException, InterruptedException {

        PulsarClient pulsarClient = PulsarClient.builder().serviceUrl("pulsar://localhost:6650").build();

        Consumer<byte[]> consumer = pulsarClient.newConsumer() //
                .topic("persistent://my-tenant/my-ns/my-topic") //
                .subscriptionName("my-subscription-name").subscribe();

        Message<byte[]> msg = null;

        for (int i = 0; i < 100; i++) {
            msg = consumer.receive();
            // do something
            System.out.println("Received: " + new String(msg.getData()));
        }

        // Acknowledge the consumption of all messages at once
        try {
            consumer.acknowledgeCumulative(msg);
        } catch (Exception e) {
            consumer.reconsumeLater(msg, 10, TimeUnit.SECONDS);
        }
       
        pulsarClient.close();
    }
 
Example 12
Source Project: pulsar   Source File: ConsumerBuilderImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Consumer<T> subscribe() throws PulsarClientException {
    try {
        return subscribeAsync().get();
    } catch (Exception e) {
        throw PulsarClientException.unwrap(e);
    }
}
 
Example 13
Source Project: pulsar   Source File: TopicsConsumerImplTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeOut = testTimeout)
public void testDifferentTopicsNameSubscribe() throws Exception {
    String key = "TopicsFromDifferentNamespace";
    final String subscriptionName = "my-ex-subscription-" + key;

    final String topicName1 = "persistent://prop/use/ns-abc1/topic-1-" + key;
    final String topicName2 = "persistent://prop/use/ns-abc2/topic-2-" + key;
    final String topicName3 = "persistent://prop/use/ns-abc3/topic-3-" + key;
    List<String> topicNames = Lists.newArrayList(topicName1, topicName2, topicName3);

    TenantInfo tenantInfo = createDefaultTenantInfo();
    admin.tenants().createTenant("prop", tenantInfo);
    admin.topics().createPartitionedTopic(topicName2, 2);
    admin.topics().createPartitionedTopic(topicName3, 3);

    // 2. Create consumer
    try {
        Consumer consumer = pulsarClient.newConsumer()
            .topics(topicNames)
            .subscriptionName(subscriptionName)
            .subscriptionType(SubscriptionType.Shared)
            .ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS)
            .subscribe();
        assertTrue(consumer instanceof MultiTopicsConsumerImpl);
    } catch (IllegalArgumentException e) {
        // expected for have different namespace
    }
}
 
Example 14
Source Project: pulsar   Source File: NamespaceServiceTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * It verifies that unloading bundle will timeout and will not hung even if one of the topic-unloading stuck.
 *
 * @throws Exception
 */
@Test(timeOut = 6000)
public void testUnloadNamespaceBundleWithStuckTopic() throws Exception {

    final String topicName = "persistent://my-property/use/my-ns/my-topic1";
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name")
            .subscribe();
    ConcurrentOpenHashMap<String, CompletableFuture<Optional<Topic>>> topics = pulsar.getBrokerService().getTopics();
    Topic spyTopic = spy(topics.get(topicName).get().get());
    topics.clear();
    CompletableFuture<Optional<Topic>> topicFuture = CompletableFuture.completedFuture(Optional.of(spyTopic));
    // add mock topic
    topics.put(topicName, topicFuture);
    // return uncompleted future as close-topic result.
    doAnswer(new Answer<CompletableFuture<Void>>() {
        @Override
        public CompletableFuture<Void> answer(InvocationOnMock invocation) throws Throwable {
            return new CompletableFuture<Void>();
        }
    }).when(spyTopic).close(false);
    NamespaceBundle bundle = pulsar.getNamespaceService().getBundle(TopicName.get(topicName));

    // try to unload bundle whose topic will be stuck
    pulsar.getNamespaceService().unloadNamespaceBundle(bundle, 1, TimeUnit.SECONDS).join();

    try {
        pulsar.getLocalZkCache().getZooKeeper().getData(ServiceUnitZkUtils.path(bundle), null, null);
        fail("it should fail as node is not present");
    } catch (org.apache.zookeeper.KeeperException.NoNodeException e) {
        // ok
    }
    consumer.close();
}
 
Example 15
Source Project: pulsar   Source File: InactiveTopicDeleteTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDeleteWhenNoBacklogs() throws Exception {
    conf.setBrokerDeleteInactiveTopicsMode(InactiveTopicDeleteMode.delete_when_subscriptions_caught_up);
    conf.setBrokerDeleteInactiveTopicsFrequencySeconds(1);
    super.baseSetup();

    final String topic = "persistent://prop/ns-abc/testDeleteWhenNoBacklogs";

    Producer<byte[]> producer = pulsarClient.newProducer()
        .topic(topic)
        .create();

    Consumer<byte[]> consumer = pulsarClient.newConsumer()
        .topic(topic)
        .subscriptionName("sub")
        .subscribe();

    for (int i = 0; i < 10; i++) {
        producer.send("Pulsar".getBytes());
    }

    consumer.close();
    producer.close();

    Thread.sleep(2000);
    Assert.assertTrue(admin.topics().getList("prop/ns-abc")
        .contains(topic));

    admin.topics().skipAllMessages(topic, "sub");
    Thread.sleep(2000);
    Assert.assertFalse(admin.topics().getList("prop/ns-abc")
        .contains(topic));

    super.internalCleanup();
}
 
Example 16
Source Project: pulsar   Source File: TopicFromMessageTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeOut = TEST_TIMEOUT)
public void testSingleTopicConsumerBatchShortName() throws Exception {
    try (Consumer<byte[]> consumer = pulsarClient.newConsumer()
            .topic("topic1").subscriptionName("sub1").subscribe();
         Producer<byte[]> producer = pulsarClient.newProducer()
            .topic("topic1").enableBatching(true).batchingMaxMessages(BATCHING_MAX_MESSAGES_THRESHOLD).create()) {
        producer.send("foobar".getBytes());

        Assert.assertEquals(consumer.receive().getTopicName(), "persistent://public/default/topic1");
    }
}
 
Example 17
Source Project: pulsar   Source File: PulsarZKDowngradeTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test(dataProvider = "ServiceUrlAndTopics")
public void testPublishAndConsume(String serviceUrl, boolean isPersistent) throws Exception {
    String topicName = generateTopicName("testpubconsume", isPersistent);

    int numMessages = 10;

    @Cleanup
    PulsarClient client = PulsarClient.builder()
            .serviceUrl(serviceUrl)
            .build();

    @Cleanup
    Consumer<String> consumer = client.newConsumer(Schema.STRING)
            .topic(topicName)
            .subscriptionName("my-sub")
            .subscribe();

    @Cleanup
    Producer<String> producer = client.newProducer(Schema.STRING)
            .topic(topicName)
            .create();

    for (int i = 0; i < numMessages; i++) {
        producer.send("smoke-message-" + i);
    }

    for (int i = 0; i < numMessages; i++) {
        Message<String> m = consumer.receive();
        assertEquals("smoke-message-" + i, m.getValue());
    }

}
 
Example 18
Source Project: pulsar   Source File: NullValueTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void nullValueBooleanSchemaTest() throws PulsarClientException {
    String topic = "persistent://prop/ns-abc/null-value-bool-test";

    @Cleanup
    Producer<Boolean> producer = pulsarClient.newProducer(Schema.BOOL)
            .topic(topic)
            .create();

    @Cleanup
    Consumer<Boolean> consumer = pulsarClient.newConsumer(Schema.BOOL)
            .topic(topic)
            .subscriptionName("test")
            .subscribe();

    int numMessage = 10;
    for (int i = 0; i < numMessage; i++) {
        producer.newMessage().value(null).sendAsync();
    }

    for (int i = 0; i < numMessage; i++) {
        Message<Boolean> message = consumer.receive();
        Assert.assertNull(message.getValue());
        Assert.assertNull(message.getData());
    }

}
 
Example 19
Source Project: pulsar   Source File: MultiTopicsConsumerImpl.java    License: Apache License 2.0 5 votes vote down vote up
public static <T> MultiTopicsConsumerImpl<T> createPartitionedConsumer(PulsarClientImpl client,
                                                                       ConsumerConfigurationData<T> conf,
                                                                       ExecutorService listenerExecutor,
                                                                       CompletableFuture<Consumer<T>> subscribeFuture,
                                                                       int numPartitions,
                                                                       Schema<T> schema, ConsumerInterceptors<T> interceptors) {
    checkArgument(conf.getTopicNames().size() == 1, "Should have only 1 topic for partitioned consumer");

    // get topic name, then remove it from conf, so constructor will create a consumer with no topic.
    ConsumerConfigurationData cloneConf = conf.clone();
    String topicName = cloneConf.getSingleTopic();
    cloneConf.getTopicNames().remove(topicName);

    CompletableFuture<Consumer> future = new CompletableFuture<>();
    MultiTopicsConsumerImpl consumer = new MultiTopicsConsumerImpl(client, topicName, cloneConf, listenerExecutor,
            future, schema, interceptors, true /* createTopicIfDoesNotExist */);

    future.thenCompose(c -> ((MultiTopicsConsumerImpl)c).subscribeAsync(topicName, numPartitions))
        .thenRun(()-> subscribeFuture.complete(consumer))
        .exceptionally(e -> {
            log.warn("Failed subscription for createPartitionedConsumer: {} {}, e:{}",
                topicName, numPartitions,  e);
            subscribeFuture.completeExceptionally(
                PulsarClientException.wrap(((Throwable) e).getCause(), String.format("Failed to subscribe %s with %d partitions", topicName, numPartitions)));
            return null;
        });
    return consumer;
}
 
Example 20
Source Project: pulsar   Source File: PersistentTopicE2ETest.java    License: Apache License 2.0 5 votes vote down vote up
@Test(dataProvider = "codec")
public void testCompression(CompressionType compressionType) throws Exception {
    final String topicName = "persistent://prop/ns-abc/topic0" + compressionType;

    // 1. producer connect
    Producer<byte[]> producer = pulsarClient.newProducer()
        .topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .compressionType(compressionType)
        .create();
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").subscribe();

    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);

    // 2. producer publish messages
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }

    for (int i = 0; i < 10; i++) {
        Message<byte[]> msg = consumer.receive(5, TimeUnit.SECONDS);
        assertNotNull(msg);
        assertEquals(msg.getData(), ("my-message-" + i).getBytes());
    }

    // 3. producer disconnect
    producer.close();
    consumer.close();
}
 
Example 21
Source Project: pulsar   Source File: SourceTester.java    License: Apache License 2.0 5 votes vote down vote up
public void validateSourceResultAvro(Consumer<KeyValue<GenericRecord, GenericRecord>> consumer,
                                 int number, String eventType) throws Exception {
    int recordsNumber = 0;
    Message<KeyValue<GenericRecord, GenericRecord>> msg = consumer.receive(2, TimeUnit.SECONDS);
    while(msg != null) {
        recordsNumber ++;
        GenericRecord keyRecord = msg.getValue().getKey();
        Assert.assertNotNull(keyRecord.getFields());
        Assert.assertTrue(keyRecord.getFields().size() > 0);

        GenericRecord valueRecord = msg.getValue().getValue();
        Assert.assertNotNull(valueRecord.getFields());
        Assert.assertTrue(valueRecord.getFields().size() > 0);
        for (Field field : valueRecord.getFields()) {
            Assert.assertTrue(DEBEZIUM_FIELD_SET.contains(field.getName()));
        }

        if (eventType != null) {
            String op = valueRecord.getField("op").toString();
            Assert.assertEquals(this.eventContains(eventType, false), op);
        }
        consumer.acknowledge(msg);
        msg = consumer.receive(1, TimeUnit.SECONDS);
    }

    Assert.assertEquals(recordsNumber, number);
    log.info("Stop {} server container. topic: {} has {} records.", getSourceType(), consumer.getTopic(), recordsNumber);
}
 
Example 22
Source Project: pulsar   Source File: InactiveTopicDeleteTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDeleteWhenNoSubscriptions() throws Exception {
    conf.setBrokerDeleteInactiveTopicsMode(InactiveTopicDeleteMode.delete_when_no_subscriptions);
    conf.setBrokerDeleteInactiveTopicsFrequencySeconds(1);
    super.baseSetup();

    final String topic = "persistent://prop/ns-abc/testDeleteWhenNoSubscriptions";

    Producer<byte[]> producer = pulsarClient.newProducer()
        .topic(topic)
        .create();

    Consumer<byte[]> consumer = pulsarClient.newConsumer()
        .topic(topic)
        .subscriptionName("sub")
        .subscribe();

    consumer.close();
    producer.close();

    Thread.sleep(2000);
    Assert.assertTrue(admin.topics().getList("prop/ns-abc")
        .contains(topic));

    admin.topics().deleteSubscription(topic, "sub");
    Thread.sleep(2000);
    Assert.assertFalse(admin.topics().getList("prop/ns-abc")
        .contains(topic));

    super.internalCleanup();
}
 
Example 23
Source Project: pulsar   Source File: BrokerServiceThrottlingTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Broker has maxConcurrentLookupRequest = 0 so, it rejects incoming lookup request and it cause consumer creation
 * failure.
 *
 * @throws Exception
 */
@Test
public void testLookupThrottlingForClientByBroker0Permit() throws Exception {

    final String topicName = "persistent://prop/ns-abc/newTopic";

    PulsarClient pulsarClient = PulsarClient.builder()
            .serviceUrl(pulsar.getBrokerServiceUrl())
            .statsInterval(0, TimeUnit.SECONDS)
            .build();

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("mysub").subscribe();
    consumer.close();

    int newPermits = 0;
    admin.brokers().updateDynamicConfiguration("maxConcurrentLookupRequest", Integer.toString(newPermits));
    // wait config to be updated
    for (int i = 0; i < 5; i++) {
        if (pulsar.getConfiguration().getMaxConcurrentLookupRequest() != newPermits) {
            Thread.sleep(100 + (i * 10));
        } else {
            break;
        }
    }

    try {
        consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("mysub").subscribe();
        consumer.close();
        fail("It should fail as throttling should not receive any request");
    } catch (org.apache.pulsar.client.api.PulsarClientException.TooManyRequestsException e) {
        // ok as throttling set to 0
    }
}
 
Example 24
/**
 * verifies that messages whose size is larger than 2^14 bytes (max size of single TLS chunk) can be
 * produced/consumed
 *
 * @throws Exception
 */
@Test(timeOut = 30000)
public void testTlsLargeSizeMessage() throws Exception {
    log.info("-- Starting {} test --", methodName);

    final int MESSAGE_SIZE = 16 * 1024 + 1;
    log.info("-- message size --", MESSAGE_SIZE);
    String topicName = "persistent://my-property/use/my-ns/testTlsLargeSizeMessage"
                       + System.currentTimeMillis();

    internalSetUpForClient(true, pulsar.getBrokerServiceUrlTls());
    internalSetUpForNamespace();

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName)
            .subscriptionName("my-subscriber-name").subscribe();

    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
            .create();
    for (int i = 0; i < 10; i++) {
        byte[] message = new byte[MESSAGE_SIZE];
        Arrays.fill(message, (byte) i);
        producer.send(message);
    }

    Message<byte[]> msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        byte[] expected = new byte[MESSAGE_SIZE];
        Arrays.fill(expected, (byte) i);
        Assert.assertEquals(expected, msg.getData());
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 
Example 25
Source Project: pulsar   Source File: PulsarClientImpl.java    License: Apache License 2.0 5 votes vote down vote up
private <T> CompletableFuture<Consumer<T>> multiTopicSubscribeAsync(ConsumerConfigurationData<T> conf, Schema<T> schema, ConsumerInterceptors<T> interceptors) {
    CompletableFuture<Consumer<T>> consumerSubscribedFuture = new CompletableFuture<>();

    ConsumerBase<T> consumer = new MultiTopicsConsumerImpl<>(PulsarClientImpl.this, conf,
            externalExecutorProvider.getExecutor(), consumerSubscribedFuture, schema, interceptors,
            true /* createTopicIfDoesNotExist */);

    synchronized (consumers) {
        consumers.put(consumer, Boolean.TRUE);
    }

    return consumerSubscribedFuture;
}
 
Example 26
Source Project: pulsar   Source File: RawReaderImpl.java    License: Apache License 2.0 5 votes vote down vote up
public RawReaderImpl(PulsarClientImpl client, String topic, String subscription,
                     CompletableFuture<Consumer<byte[]>> consumerFuture) {
    consumerConfiguration = new ConsumerConfigurationData<>();
    consumerConfiguration.getTopicNames().add(topic);
    consumerConfiguration.setSubscriptionName(subscription);
    consumerConfiguration.setSubscriptionType(SubscriptionType.Exclusive);
    consumerConfiguration.setReceiverQueueSize(DEFAULT_RECEIVER_QUEUE_SIZE);
    consumerConfiguration.setReadCompacted(true);

    consumer = new RawConsumerImpl(client, consumerConfiguration,
                                   consumerFuture);
}
 
Example 27
Source Project: pulsar   Source File: TlsProducerConsumerTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * verifies that messages whose size is larger than 2^14 bytes (max size of single TLS chunk) can be
 * produced/consumed
 *
 * @throws Exception
 */
@Test(timeOut = 30000)
public void testTlsLargeSizeMessage() throws Exception {
    log.info("-- Starting {} test --", methodName);

    final int MESSAGE_SIZE = 16 * 1024 + 1;
    log.info("-- message size --", MESSAGE_SIZE);

    internalSetUpForClient(true, pulsar.getBrokerServiceUrlTls());
    internalSetUpForNamespace();

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://my-property/use/my-ns/my-topic1")
            .subscriptionName("my-subscriber-name").subscribe();

    Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1")
            .create();
    for (int i = 0; i < 10; i++) {
        byte[] message = new byte[MESSAGE_SIZE];
        Arrays.fill(message, (byte) i);
        producer.send(message);
    }

    Message<byte[]> msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        byte[] expected = new byte[MESSAGE_SIZE];
        Arrays.fill(expected, (byte) i);
        Assert.assertEquals(expected, msg.getData());
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 
Example 28
Source Project: pulsar   Source File: CompactionTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testWholeBatchCompactedOut() throws Exception {
    String topic = "persistent://my-property/use/my-ns/my-topic1";

    // subscribe before sending anything, so that we get all messages
    pulsarClient.newConsumer().topic(topic).subscriptionName("sub1")
        .readCompacted(true).subscribe().close();

    try (Producer<byte[]> producerNormal = pulsarClient.newProducer().topic(topic)
             .enableBatching(false)
             .messageRoutingMode(MessageRoutingMode.SinglePartition)
             .create();
         Producer<byte[]> producerBatch = pulsarClient.newProducer().topic(topic)
             .maxPendingMessages(3)
             .enableBatching(true)
             .batchingMaxMessages(3)
             .batchingMaxPublishDelay(1, TimeUnit.HOURS)
             .messageRoutingMode(MessageRoutingMode.SinglePartition)
             .create()) {
        producerBatch.newMessage().key("key1").value("my-message-1".getBytes()).sendAsync();
        producerBatch.newMessage().key("key1").value("my-message-2".getBytes()).sendAsync();
        producerBatch.newMessage().key("key1").value("my-message-3".getBytes()).sendAsync();
        producerNormal.newMessage().key("key1").value("my-message-4".getBytes()).send();
    }

    // compact the topic
    Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
    compactor.compact(topic).get();

    try (Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic)
            .subscriptionName("sub1").readCompacted(true).subscribe()){
        Message<byte[]> message = consumer.receive();
        Assert.assertEquals(message.getKey(), "key1");
        Assert.assertEquals(new String(message.getData()), "my-message-4");
    }
}
 
Example 29
Source Project: pulsar   Source File: TopicTerminationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeOut = 20000)
public void testSimpleTerminationConsumer() throws Exception {
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();
    org.apache.pulsar.client.api.Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName)
            .subscriptionName("my-sub").subscribe();

    MessageId msgId1 = producer.send("test-msg-1".getBytes());
    MessageId msgId2 = producer.send("test-msg-2".getBytes());

    Message<byte[]> msg1 = consumer.receive();
    assertEquals(msg1.getMessageId(), msgId1);
    consumer.acknowledge(msg1);

    MessageId msgId3 = producer.send("test-msg-3".getBytes());

    assertFalse(consumer.hasReachedEndOfTopic());

    MessageId lastMessageId = admin.topics().terminateTopicAsync(topicName).get();
    assertEquals(lastMessageId, msgId3);

    Message<byte[]> msg2 = consumer.receive();
    assertEquals(msg2.getMessageId(), msgId2);
    consumer.acknowledge(msg2);

    Message<byte[]> msg3 = consumer.receive();
    assertEquals(msg3.getMessageId(), msgId3);
    consumer.acknowledge(msg3);

    Message<byte[]> msg4 = consumer.receive(100, TimeUnit.MILLISECONDS);
    assertNull(msg4);

    Thread.sleep(100);
    assertTrue(consumer.hasReachedEndOfTopic());
}
 
Example 30
Source Project: pulsar   Source File: SmokeTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void checkMessages() throws PulsarClientException {

    @Cleanup
    PulsarClient client = PulsarClient.builder()
            .serviceUrl(pulsarContainer.getPlainTextPulsarBrokerUrl())
            .build();

    final String inputTopic = "input";

    Producer<String> producer = client.newProducer(Schema.STRING)
            .topic(inputTopic)
            .enableBatching(false)
            .create();

    Consumer<String> consumer = client.newConsumer(Schema.STRING)
            .topic(inputTopic)
            .subscriptionName("test-subs")
            .ackTimeout(10, TimeUnit.SECONDS)
            .subscriptionType(SubscriptionType.Exclusive)
            .subscribe();

    producer.send("Hello!");
    Message<String> message = consumer.receive(10, TimeUnit.SECONDS);

    Assert.assertEquals(message.getValue(), "Hello!");

}