Java Code Examples for org.apache.pulsar.client.api.Producer#flush()

The following examples show how to use org.apache.pulsar.client.api.Producer#flush() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FlinkPulsarSinkBase.java    From pulsar-flink with Apache License 2.0 6 votes vote down vote up 
public void producerFlush() throws Exception {
    if (singleProducer != null) {
        singleProducer.flush();
    } else {
        if (topic2Producer != null) {
            for (Producer<?> p : topic2Producer.values()) {
                p.flush();
            }
        }
    }
    synchronized (pendingRecordsLock) {
        while (pendingRecords > 0) {
            try {
                pendingRecordsLock.wait();
            } catch (InterruptedException e) {
                // this can be interrupted when the Task has been cancelled.
                // by throwing an exception, we ensure that this checkpoint doesn't get confirmed
                throw new RuntimeException("Flushing got interrupted while checkpointing", e);
            }
        }
    }
}
Example 2
Source File: UnAcknowledgedMessagesTimeoutTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSingleMessageBatch() throws Exception {
    String topicName = "prop/ns-abc/topic-estSingleMessageBatch";

    Producer<String> producer = pulsarClient.newProducer(Schema.STRING)
            .topic(topicName)
            .enableBatching(true)
            .batchingMaxPublishDelay(10, TimeUnit.SECONDS)
            .create();

    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING)
            .topic(topicName)
            .subscriptionName("subscription")
            .ackTimeout(1, TimeUnit.HOURS)
            .subscribe();

    // Force the creation of a batch with a single message
    producer.sendAsync("hello");
    producer.flush();

    Message<String> message = consumer.receive();

    assertFalse(((ConsumerImpl<?>) consumer).getUnAckedMessageTracker().isEmpty());

    consumer.acknowledge(message);

    assertTrue(((ConsumerImpl<?>) consumer).getUnAckedMessageTracker().isEmpty());
}
Example 3
Source File: PulsarTestBase.java    From pulsar-flink with Apache License 2.0 4 votes vote down vote up 
public static <T> List<MessageId> sendTypedMessages(
        String topic,
        SchemaType type,
        List<T> messages,
        Optional<Integer> partition,
        Class<T> tClass) throws PulsarClientException {

    String topicName;
    if (partition.isPresent()) {
        topicName = topic + PulsarOptions.PARTITION_SUFFIX + partition.get();
    } else {
        topicName = topic;
    }

    Producer<T> producer = null;
    PulsarClient client = null;
    List<MessageId> mids = new ArrayList<>();

    try {
        client = PulsarClient.builder().serviceUrl(getServiceUrl()).build();

        switch (type) {
            case BOOLEAN:
                producer = (Producer<T>) client.newProducer(Schema.BOOL).topic(topicName).create();
                break;
            case BYTES:
                producer = (Producer<T>) client.newProducer(Schema.BYTES).topic(topicName).create();
                break;
            case DATE:
                producer = (Producer<T>) client.newProducer(Schema.DATE).topic(topicName).create();
                break;
            case STRING:
                producer = (Producer<T>) client.newProducer(Schema.STRING).topic(topicName).create();
                break;
            case TIMESTAMP:
                producer = (Producer<T>) client.newProducer(Schema.TIMESTAMP).topic(topicName).create();
                break;
            case INT8:
                producer = (Producer<T>) client.newProducer(Schema.INT8).topic(topicName).create();
                break;
            case DOUBLE:
                producer = (Producer<T>) client.newProducer(Schema.DOUBLE).topic(topicName).create();
                break;
            case FLOAT:
                producer = (Producer<T>) client.newProducer(Schema.FLOAT).topic(topicName).create();
                break;
            case INT32:
                producer = (Producer<T>) client.newProducer(Schema.INT32).topic(topicName).create();
                break;
            case INT16:
                producer = (Producer<T>) client.newProducer(Schema.INT16).topic(topicName).create();
                break;
            case INT64:
                producer = (Producer<T>) client.newProducer(Schema.INT64).topic(topicName).create();
                break;
            case AVRO:
                producer = (Producer<T>) client.newProducer(Schema.AVRO(tClass)).topic(topicName).create();
                break;
            case JSON:
                producer = (Producer<T>) client.newProducer(Schema.JSON(tClass)).topic(topicName).create();
                break;

            default:
                throw new NotImplementedException("Unsupported type " + type);
        }

        for (T message : messages) {
            MessageId mid = producer.send(message);
            log.info("Sent {} of mid: {}", message.toString(), mid.toString());
            mids.add(mid);
        }

    } finally {
        producer.flush();
        producer.close();
        client.close();
    }
    return mids;
}
Example 4
Source File: CompactionTest.java    From pulsar with Apache License 2.0 4 votes vote down vote up 
@Test(timeOut = 20000)
public void testBatchAndNonBatchWithoutEmptyPayload() throws PulsarClientException, ExecutionException, InterruptedException {
    final String topic = "persistent://my-property/use/my-ns/testBatchAndNonBatchWithoutEmptyPayload" + UUID.randomUUID().toString();

    // 1.create producer and publish message to the topic.
    Producer<byte[]> producer = pulsarClient.newProducer()
            .topic(topic)
            .enableBatching(true)
            .batchingMaxPublishDelay(1, TimeUnit.DAYS)
            .create();

    final String k1 = "k1";
    final String k2 = "k2";
    producer.newMessage().key(k1).value("0".getBytes()).send();
    List<CompletableFuture<MessageId>> futures = new ArrayList<>(7);
    for (int i = 0; i < 2; i++) {
        futures.add(producer.newMessage().key(k1).value((i + 1 + "").getBytes()).sendAsync());
    }
    producer.flush();
    producer.newMessage().key(k1).value("3".getBytes()).send();
    for (int i = 0; i < 2; i++) {
        futures.add(producer.newMessage().key(k1).value((i + 4 + "").getBytes()).sendAsync());
    }
    producer.flush();

    for (int i = 0; i < 3; i++) {
        futures.add(producer.newMessage().key(k2).value((i + "").getBytes()).sendAsync());
    }

    producer.newMessage().key(k2).value("3".getBytes()).send();
    producer.flush();
    FutureUtil.waitForAll(futures).get();

    // 2.compact the topic.
    Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
    compactor.compact(topic).get();

    // consumer with readCompacted enabled only get compacted entries
    try (Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("sub1")
            .readCompacted(true).subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe()) {
        Message<byte[]> m1 = consumer.receive(2, TimeUnit.SECONDS);
        Message<byte[]> m2 = consumer.receive(2, TimeUnit.SECONDS);
        assertNotNull(m1);
        assertNotNull(m2);
        assertEquals(m1.getKey(), k1);
        assertEquals(new String(m1.getValue()), "5");
        assertEquals(m2.getKey(), k2);
        assertEquals(new String(m2.getValue()), "3");
        Message<byte[]> none = consumer.receive(2, TimeUnit.SECONDS);
        assertNull(none);
    }
}
Example 5
Source File: CompactionTest.java    From pulsar with Apache License 2.0 4 votes vote down vote up 
@Test(timeOut = 20000)
public void testBatchAndNonBatchWithEmptyPayload() throws PulsarClientException, ExecutionException, InterruptedException {
    final String topic = "persistent://my-property/use/my-ns/testBatchAndNonBatchWithEmptyPayload" + UUID.randomUUID().toString();

    // 1.create producer and publish message to the topic.
    Producer<byte[]> producer = pulsarClient.newProducer()
            .topic(topic)
            .enableBatching(true)
            .batchingMaxPublishDelay(1, TimeUnit.DAYS)
            .create();

    final String k1 = "k1";
    final String k2 = "k2";
    final String k3 = "k3";
    producer.newMessage().key(k1).value("0".getBytes()).send();
    List<CompletableFuture<MessageId>> futures = new ArrayList<>(7);
    for (int i = 0; i < 2; i++) {
        futures.add(producer.newMessage().key(k1).value((i + 1 + "").getBytes()).sendAsync());
    }
    producer.flush();
    producer.newMessage().key(k1).value("3".getBytes()).send();
    for (int i = 0; i < 2; i++) {
        futures.add(producer.newMessage().key(k1).value((i + 4 + "").getBytes()).sendAsync());
    }
    producer.flush();

    for (int i = 0; i < 3; i++) {
        futures.add(producer.newMessage().key(k2).value((i + 10 + "").getBytes()).sendAsync());
    }
    producer.flush();

    producer.newMessage().key(k2).value("".getBytes()).send();

    producer.newMessage().key(k3).value("0".getBytes()).send();

    FutureUtil.waitForAll(futures).get();

    // 2.compact the topic.
    Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
    compactor.compact(topic).get();

    // consumer with readCompacted enabled only get compacted entries
    try (Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("sub1")
            .readCompacted(true).subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe()) {
        Message<byte[]> m1 = consumer.receive();
        Message<byte[]> m2 = consumer.receive();
        assertNotNull(m1);
        assertNotNull(m2);
        assertEquals(m1.getKey(), k1);
        assertEquals(m2.getKey(), k3);
        assertEquals(new String(m1.getValue()), "5");
        assertEquals(new String(m2.getValue()), "0");
        Message<byte[]> none = consumer.receive(2, TimeUnit.SECONDS);
        assertNull(none);
    }
}
Example 6
Source File: CompactionTest.java    From pulsar with Apache License 2.0 4 votes vote down vote up 
@Test(timeOut = 20000)
public void testBatchAndNonBatchEndOfEmptyPayload() throws PulsarClientException, ExecutionException, InterruptedException {
    final String topic = "persistent://my-property/use/my-ns/testBatchAndNonBatchWithEmptyPayload" + UUID.randomUUID().toString();

    // 1.create producer and publish message to the topic.
    Producer<byte[]> producer = pulsarClient.newProducer()
            .topic(topic)
            .enableBatching(true)
            .batchingMaxPublishDelay(1, TimeUnit.DAYS)
            .create();

    final String k1 = "k1";
    final String k2 = "k2";
    producer.newMessage().key(k1).value("0".getBytes()).send();
    List<CompletableFuture<MessageId>> futures = new ArrayList<>(7);
    for (int i = 0; i < 2; i++) {
        futures.add(producer.newMessage().key(k1).value((i + 1 + "").getBytes()).sendAsync());
    }
    producer.flush();
    producer.newMessage().key(k1).value("3".getBytes()).send();
    for (int i = 0; i < 2; i++) {
        futures.add(producer.newMessage().key(k1).value((i + 4 + "").getBytes()).sendAsync());
    }
    producer.flush();

    for (int i = 0; i < 3; i++) {
        futures.add(producer.newMessage().key(k2).value((i + 10 + "").getBytes()).sendAsync());
    }
    producer.flush();

    producer.newMessage().key(k2).value("".getBytes()).send();

    FutureUtil.waitForAll(futures).get();

    // 2.compact the topic.
    Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
    compactor.compact(topic).get();

    // consumer with readCompacted enabled only get compacted entries
    try (Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("sub1")
            .readCompacted(true).subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe()) {
        Message<byte[]> m1 = consumer.receive();
        assertNotNull(m1);
        assertEquals(m1.getKey(), k1);
        assertEquals(new String(m1.getValue()), "5");
        Message<byte[]> none = consumer.receive(2, TimeUnit.SECONDS);
        assertNull(none);
    }
}
Example 7
Source File: AdminTopicApiTest.java    From pulsar with Apache License 2.0 4 votes vote down vote up 
@Test
public void testPeekMessages() throws Exception {
    @Cleanup
    PulsarClient newPulsarClient = PulsarClient.builder()
        .serviceUrl(lookupUrl.toString())
        .build();

    final String topic = "persistent://my-property/my-ns/test-publish-timestamp";

    @Cleanup
    Consumer<byte[]> consumer = newPulsarClient.newConsumer()
        .topic(topic)
        .subscriptionName("my-sub")
        .subscribe();

    final int numMessages = 5;

    @Cleanup
    Producer<byte[]> producer = newPulsarClient.newProducer()
            .topic(topic)
            .enableBatching(true)
            .batchingMaxPublishDelay(3, TimeUnit.SECONDS)
            .batchingMaxMessages(5)
            .create();

    for (int i = 0; i < numMessages; i++) {
        producer.newMessage()
            .value(("value-" + i).getBytes(UTF_8))
            .sendAsync();
    }
    producer.flush();

    for (int i = 0; i < numMessages; i++) {
        Message<byte[]> msg = consumer.receive();
        log.info("Received message '{}'.", new String(msg.getValue(), UTF_8));
    }
    List<Message<byte[]>> messages = admin.topics().peekMessages(topic, "my-sub", 5);
    Assert.assertEquals(new String(messages.get(0).getValue(), UTF_8), "value-0");
    Assert.assertEquals(new String(messages.get(1).getValue(), UTF_8), "value-1");
    Assert.assertEquals(new String(messages.get(2).getValue(), UTF_8), "value-2");
    Assert.assertEquals(new String(messages.get(3).getValue(), UTF_8), "value-3");
    Assert.assertEquals(new String(messages.get(4).getValue(), UTF_8), "value-4");
}
Example 8
Source File: DelayedDeliveryTest.java    From pulsar with Apache License 2.0 4 votes vote down vote up 
@Test
public void testDelayedDelivery()
        throws Exception {
    String topic = "testNegativeAcks-" + System.nanoTime();

    @Cleanup
    Consumer<String> failoverConsumer = pulsarClient.newConsumer(Schema.STRING)
            .topic(topic)
            .subscriptionName("failover-sub")
            .subscriptionType(SubscriptionType.Failover)
            .subscribe();

    @Cleanup
    Consumer<String> sharedConsumer = pulsarClient.newConsumer(Schema.STRING)
            .topic(topic)
            .subscriptionName("shared-sub")
            .subscriptionType(SubscriptionType.Shared)
            .subscribe();

    @Cleanup
    Producer<String> producer = pulsarClient.newProducer(Schema.STRING)
            .topic(topic)
            .create();

    for (int i = 0; i < 10; i++) {
        producer.newMessage()
                .value("msg-" + i)
                .deliverAfter(5, TimeUnit.SECONDS)
                .sendAsync();
    }

    producer.flush();

    // Failover consumer will receive the messages immediately while
    // the shared consumer will get them after the delay
    Message<String> msg = sharedConsumer.receive(100, TimeUnit.MILLISECONDS);
    assertNull(msg);

    for (int i = 0; i < 10; i++) {
        msg = failoverConsumer.receive(100, TimeUnit.MILLISECONDS);
        assertEquals(msg.getValue(), "msg-" + i);
    }

    Set<String> receivedMsgs = new TreeSet<>();
    for (int i = 0; i < 10; i++) {
        msg = sharedConsumer.receive(10, TimeUnit.SECONDS);
        receivedMsgs.add(msg.getValue());
    }

    assertEquals(receivedMsgs.size(), 10);
    for (int i = 0; i < 10; i++) {
        assertTrue(receivedMsgs.contains("msg-" + i));
    }
}
Example 9
Source File: DelayedDeliveryTest.java    From pulsar with Apache License 2.0 4 votes vote down vote up 
@Test
public void testDelayedDeliveryWithMultipleConcurrentReadEntries()
        throws Exception {
    String topic = "persistent://public/default/testDelayedDelivery-" + System.nanoTime();

    @Cleanup
    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING)
            .topic(topic)
            .subscriptionName("shared-sub")
            .subscriptionType(SubscriptionType.Shared)
            .receiverQueueSize(1) // Use small prefecthing to simulate the multiple read batches
            .subscribe();

    // Simulate race condition with high frequency of calls to dispatcher.readMoreEntries()
    PersistentDispatcherMultipleConsumers d = (PersistentDispatcherMultipleConsumers) ((PersistentTopic) pulsar
            .getBrokerService().getTopicReference(topic).get()).getSubscription("shared-sub").getDispatcher();
    Thread t = new Thread(() -> {
        while (true) {
            synchronized (d) {
                d.readMoreEntries();
            }

            try {
                Thread.sleep(1);
            } catch (InterruptedException e) {
                return;
            }
        }
    });
    t.start();

    @Cleanup
    Producer<String> producer = pulsarClient.newProducer(Schema.STRING)
            .topic(topic)
            .create();

    final int N = 1000;

    for (int i = 0; i < N; i++) {
        producer.newMessage()
                .value("msg-" + i)
                .deliverAfter(5, TimeUnit.SECONDS)
                .sendAsync();
    }

    producer.flush();

    Message<String> msg = consumer.receive(100, TimeUnit.MILLISECONDS);
    assertNull(msg);

    Set<String> receivedMsgs = new TreeSet<>();
    for (int i = 0; i < N; i++) {
        msg = consumer.receive(10, TimeUnit.SECONDS);
        receivedMsgs.add(msg.getValue());
    }

    assertEquals(receivedMsgs.size(), N);
    for (int i = 0; i < N; i++) {
        assertTrue(receivedMsgs.contains("msg-" + i));
    }
    t.interrupt();
}