Java Code Examples for org.apache.pulsar.client.api.Producer

The following examples show how to use org.apache.pulsar.client.api.Producer. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: pulsar   Author: apache   File: PulsarStateTest.java    License: Apache License 2.0 6 votes vote down vote up
private static void publishAndConsumeMessages(String inputTopic,
                                              String outputTopic,
                                              int numMessages) throws Exception {
    @Cleanup PulsarClient client = PulsarClient.builder()
        .serviceUrl(container.getPlainTextServiceUrl())
        .build();
    @Cleanup Consumer<byte[]> consumer = client.newConsumer(Schema.BYTES)
        .topic(outputTopic)
        .subscriptionType(SubscriptionType.Exclusive)
        .subscriptionName("test-sub")
        .subscribe();
    @Cleanup Producer<byte[]> producer = client.newProducer(Schema.BYTES)
        .topic(inputTopic)
        .create();

    for (int i = 0; i < numMessages; i++) {
        producer.send(("hello test message-" + i).getBytes(UTF_8));
    }

    for (int i = 0; i < numMessages; i++) {
        Message<byte[]> msg = consumer.receive();
        assertEquals("hello test message-" + i + "!", new String(msg.getValue(), UTF_8));
    }
}
 
Example #2
Source Project: hazelcast-jet-contrib   Author: hazelcast   File: PulsarTestSupport.java    License: Apache License 2.0 6 votes vote down vote up
private static Producer<byte[]> getProducer(String topicName) throws PulsarClientException {
    // If there exists a producer with same name returns it.
    if (!producerMap.containsKey(topicName)) {
        Producer<byte[]> newProducer = getClient()
                .newProducer()
                .topic(topicName)
                .batchingMaxPublishDelay(10, TimeUnit.MILLISECONDS)
                .sendTimeout(10, TimeUnit.SECONDS)
                .blockIfQueueFull(true)
                .create();
        producerMap.put(topicName, newProducer);
        return newProducer;
    } else {
        return producerMap.get(topicName);
    }
}
 
Example #3
Source Project: pulsar   Author: apache   File: InactiveTopicDeleteTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMaxInactiveDuration() throws Exception {
    conf.setBrokerDeleteInactiveTopicsMode(InactiveTopicDeleteMode.delete_when_subscriptions_caught_up);
    conf.setBrokerDeleteInactiveTopicsFrequencySeconds(1);
    conf.setBrokerDeleteInactiveTopicsMaxInactiveDurationSeconds(5);
    super.baseSetup();

    final String topic = "persistent://prop/ns-abc/testMaxInactiveDuration";

    Producer<byte[]> producer = pulsarClient.newProducer()
        .topic(topic)
        .create();

    producer.close();
    Thread.sleep(2000);
    Assert.assertTrue(admin.topics().getList("prop/ns-abc")
        .contains(topic));

    Thread.sleep(4000);
    Assert.assertFalse(admin.topics().getList("prop/ns-abc")
        .contains(topic));

    super.internalCleanup();
}
 
Example #4
Source Project: pulsar   Author: apache   File: LoadSimulationClient.java    License: Apache License 2.0 6 votes vote down vote up
public void start() throws Exception {
    Producer<byte[]> producer = getNewProducer();
    final Consumer<byte[]> consumer = consumerFuture.get();
    while (!stop.get()) {
        final MutableBoolean wellnessFlag = new MutableBoolean();
        final Function<Throwable, ? extends MessageId> exceptionHandler = e -> {
            // Unset the well flag in the case of an exception so we can
            // try to get a new Producer.
            wellnessFlag.value = false;
            return null;
        };
        while (!stop.get() && wellnessFlag.value) {
            producer.sendAsync(payload.get()).exceptionally(exceptionHandler);
            rateLimiter.acquire();
        }
        producer.closeAsync();
        if (!stop.get()) {
            // The Producer failed due to an exception: attempt to get
            // another producer.
            producer = getNewProducer();
        } else {
            // We are finished: close the consumer.
            consumer.closeAsync();
        }
    }
}
 
Example #5
Source Project: pulsar-flink   Author: streamnative   File: FlinkPulsarSinkTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This test is meant to assure that testAtLeastOnceProducer is valid by testing that if flushing is disabled,
 * the snapshot method does indeed finishes without waiting for pending records;
 * we set a timeout because the test will not finish if the logic is broken.
 */
@SuppressWarnings("unchecked")
@Test//(timeout = 5000)
public void testDoesNotWaitForPendingRecordsIfFlushingDisabled() throws Throwable {
    Properties props = dummyProperties();
    props.setProperty("flushoncheckpoint", "false");

    final DummyFlinkPulsarSink<String> sink = new DummyFlinkPulsarSink<>(dummyClientConf(), props, mock(TopicKeyExtractor.class), null);

    final Producer mockProducer = sink.getProducer("tp");

    final OneInputStreamOperatorTestHarness<String, Object> testHarness =
            new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink));

    testHarness.open();

    testHarness.processElement(new StreamRecord<>("msg"));

    // make sure that all callbacks have not been completed
    verify(mockProducer, times(1)).newMessage();

    // should return even if there are pending records
    testHarness.snapshot(123L, 123L);

    testHarness.close();
}
 
Example #6
Source Project: pulsar   Author: apache   File: AdminApiSchemaValidationEnforced.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testEnableSchemaValidationEnforcedHasSchemaMatch() throws Exception {
    admin.namespaces().createNamespace("schema-validation-enforced/enable-has-schema-match");
    String namespace = "schema-validation-enforced/enable-has-schema-match";
    String topicName = "persistent://schema-validation-enforced/enable-has-schema-match/test";
    assertFalse(admin.namespaces().getSchemaValidationEnforced(namespace));
    try {
        admin.schemas().getSchemaInfo(topicName);
    } catch (PulsarAdminException.NotFoundException e) {
        assertTrue(e.getMessage().contains("HTTP 404 Not Found"));
    }
    admin.namespaces().setSchemaValidationEnforced(namespace,true);
    Map<String, String> properties = Maps.newHashMap();
    SchemaInfo schemaInfo = new SchemaInfo();
    schemaInfo.setType(SchemaType.STRING);
    schemaInfo.setProperties(properties);
    schemaInfo.setName("test");
    schemaInfo.setSchema("".getBytes());
    PostSchemaPayload postSchemaPayload = new PostSchemaPayload("STRING", "", properties);
    admin.schemas().createSchema(topicName, postSchemaPayload);
    try (Producer<String> p = pulsarClient.newProducer(Schema.STRING).topic(topicName).create()) {
        p.send("test schemaValidationEnforced");
    }
    assertEquals(admin.schemas().getSchemaInfo(topicName).getName(), schemaInfo.getName());
    assertEquals(admin.schemas().getSchemaInfo(topicName).getType(), schemaInfo.getType());
}
 
Example #7
Source Project: pulsar   Author: apache   File: AdminApiSchemaValidationEnforced.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testEnableSchemaValidationEnforcedNoSchema() throws Exception {
    admin.namespaces().createNamespace("schema-validation-enforced/enable-no-schema");
    String namespace = "schema-validation-enforced/enable-no-schema";
    String topicName = "persistent://schema-validation-enforced/enable-no-schema/test";
    assertFalse(admin.namespaces().getSchemaValidationEnforced(namespace));
    admin.namespaces().setSchemaValidationEnforced(namespace,true);
    try {
        admin.schemas().getSchemaInfo(topicName);
    } catch (PulsarAdminException.NotFoundException e) {
        assertTrue(e.getMessage().contains("HTTP 404 Not Found"));
    }
    try (Producer p = pulsarClient.newProducer().topic(topicName).create()) {
        p.send("test schemaValidationEnforced".getBytes());
    }
}
 
Example #8
Source Project: pulsar   Author: apache   File: AdminApiTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testBacklogSizeShouldBeZeroWhenConsumerAckedAllMessages() throws Exception {
    final String topic = "persistent://prop-xyz/ns1/testBacklogSizeShouldBeZeroWhenConsumerAckedAllMessages";
    Consumer<byte[]> consumer = pulsarClient.newConsumer()
            .topic(topic)
            .subscriptionName("sub-1")
            .subscribe();
    Producer<byte[]> producer = pulsarClient.newProducer()
            .topic(topic)
            .create();

    final int messages = 33;
    for (int i = 0; i < messages; i++) {
        producer.send(new byte[1024 * i * 5]);
    }

    for (int i = 0; i < messages; i++) {
        consumer.acknowledgeCumulative(consumer.receive());
    }

    // Wait ack send
    Thread.sleep(1000);

    TopicStats topicStats = admin.topics().getStats(topic);
    assertEquals(topicStats.backlogSize, 0);
}
 
Example #9
Source Project: pulsar   Author: apache   File: ProducerSparkReceiverData.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  if (args.length < 2) {
    System.err.println("Missing parameters!");
    System.err.println("Usage: <pulsar-service-url> <topic>");
    return;
  }

  System.out.println("Parameters:");
  System.out.println("\tServiceUrl:\t" + args[0]);
  System.out.println("\tTopic:\t" + args[1]);

  try (PulsarClient client = PulsarClient.builder().serviceUrl(args[0]).build()) {
    try (Producer<byte[]> producer = client.newProducer().topic(args[1]).create()) {
      for (int i = 0; i < 100; i++) {
        producer.send(("producer spark streaming msg").getBytes(StandardCharsets.UTF_8));
      }
    }
  }

  System.out.println("producer spark streaming msg end ...");
}
 
Example #10
Source Project: pulsar   Author: apache   File: NonPersistentReplicator.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void readEntries(Producer<byte[]> producer) {
    this.producer = (ProducerImpl) producer;

    if (STATE_UPDATER.compareAndSet(this, State.Starting, State.Started)) {
        log.info("[{}][{} -> {}] Created replicator producer", topicName, localCluster, remoteCluster);
        backOff.reset();
    } else {
        log.info(
                "[{}][{} -> {}] Replicator was stopped while creating the producer. Closing it. Replicator state: {}",
                topicName, localCluster, remoteCluster, STATE_UPDATER.get(this));
        STATE_UPDATER.set(this, State.Stopping);
        closeProducerAsync();
        return;
    }
}
 
Example #11
Source Project: pulsar   Author: apache   File: CompactionTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test(timeOut = 20000, dataProvider = "lastDeletedBatching")
public void testEmptyCompactionLedger(boolean batching) throws Exception {
    String topic = "persistent://my-property/use/my-ns/my-topic1";

    Producer<byte[]> producer = pulsarClient.newProducer().topic(topic).enableBatching(batching)
            .messageRoutingMode(MessageRoutingMode.SinglePartition).create();

    pulsarClient.newConsumer().topic(topic).subscriptionName("sub1").readCompacted(true).subscribe().close();

    producer.newMessage().key("1").value("1".getBytes()).send();
    producer.newMessage().key("2").value("2".getBytes()).send();
    producer.newMessage().key("1").value("".getBytes()).send();
    producer.newMessage().key("2").value("".getBytes()).send();

    Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
    compactor.compact(topic).get();

    // consumer with readCompacted enabled only get compacted entries
    try (Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("sub1")
            .readCompacted(true).subscribe()) {
        Message<byte[]> m = consumer.receive(2, TimeUnit.SECONDS);
        assertNull(m);
    }
}
 
Example #12
Source Project: pulsar   Author: apache   File: PartitionedProducerImpl.java    License: Apache License 2.0 6 votes vote down vote up
public PartitionedProducerImpl(PulsarClientImpl client, String topic, ProducerConfigurationData conf, int numPartitions,
        CompletableFuture<Producer<T>> producerCreatedFuture, Schema<T> schema, ProducerInterceptors interceptors) {
    super(client, topic, conf, producerCreatedFuture, schema, interceptors);
    this.producers = Lists.newArrayListWithCapacity(numPartitions);
    this.topicMetadata = new TopicMetadataImpl(numPartitions);
    this.routerPolicy = getMessageRouter();
    stats = client.getConfiguration().getStatsIntervalSeconds() > 0 ? new ProducerStatsRecorderImpl() : null;

    int maxPendingMessages = Math.min(conf.getMaxPendingMessages(),
            conf.getMaxPendingMessagesAcrossPartitions() / numPartitions);
    conf.setMaxPendingMessages(maxPendingMessages);
    start();

    // start track and auto subscribe partition increasement
    if (conf.isAutoUpdatePartitions()) {
        topicsPartitionChangedListener = new TopicsPartitionChangedListener();
        partitionsAutoUpdateTimeout = client.timer()
            .newTimeout(partitionsAutoUpdateTimerTask, 1, TimeUnit.MINUTES);
    }
}
 
Example #13
Source Project: pulsar   Author: apache   File: AdminApiTest.java    License: Apache License 2.0 6 votes vote down vote up
private void publishMessagesOnPersistentTopic(String topicName, int messages, int startIdx,
                                              boolean nullValue) throws Exception {
    Producer<byte[]> producer = pulsarClient.newProducer(Schema.BYTES)
        .topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();

    for (int i = startIdx; i < (messages + startIdx); i++) {
        if (nullValue) {
            producer.send(null);
        } else {
            String message = "message-" + i;
            producer.send(message.getBytes());
        }
    }

    producer.close();
}
 
Example #14
Source Project: pulsar   Author: apache   File: RawReaderTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testBatchingExtractKeysAndIds() throws Exception {
    String topic = "persistent://my-property/my-ns/my-raw-topic";

    try (Producer<byte[]> producer = pulsarClient.newProducer().topic(topic)
        .maxPendingMessages(3)
        .enableBatching(true)
        .batchingMaxMessages(3)
        .batchingMaxPublishDelay(1, TimeUnit.HOURS)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create()) {
        producer.newMessage().key("key1").value("my-content-1".getBytes()).sendAsync();
        producer.newMessage().key("key2").value("my-content-2".getBytes()).sendAsync();
        producer.newMessage().key("key3").value("my-content-3".getBytes()).send();
    }

    RawReader reader = RawReader.create(pulsarClient, topic, subscription).get();
    try (RawMessage m = reader.readNextAsync().get()) {
        List<ImmutableTriple<MessageId, String, Integer>> idsAndKeys = RawBatchConverter.extractIdsAndKeysAndSize(m);

        Assert.assertEquals(idsAndKeys.size(), 3);

        // assert message ids are in correct order
        Assert.assertTrue(idsAndKeys.get(0).getLeft().compareTo(idsAndKeys.get(1).getLeft()) < 0);
        Assert.assertTrue(idsAndKeys.get(1).getLeft().compareTo(idsAndKeys.get(2).getLeft()) < 0);

        // assert keys are as expected
        Assert.assertEquals(idsAndKeys.get(0).getMiddle(), "key1");
        Assert.assertEquals(idsAndKeys.get(1).getMiddle(), "key2");
        Assert.assertEquals(idsAndKeys.get(2).getMiddle(), "key3");
    } finally {
        reader.closeAsync().get();
    }
}
 
Example #15
Source Project: pulsar   Author: apache   File: PersistentTopicE2ETest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testReceiveWithTimeout() throws Exception {
    final String topicName = "persistent://prop/ns-abc/topic-receive-timeout";
    final String subName = "sub";

    ConsumerImpl<byte[]> consumer = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topicName)
            .subscriptionName(subName).receiverQueueSize(1000).subscribe();
    Producer<byte[]> producer = pulsarClient.newProducer()
        .topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();

    assertEquals(consumer.getAvailablePermits(), 0);

    Message<byte[]> msg = consumer.receive(10, TimeUnit.MILLISECONDS);
    assertNull(msg);
    assertEquals(consumer.getAvailablePermits(), 0);

    producer.send("test".getBytes());
    Thread.sleep(100);

    assertEquals(consumer.getAvailablePermits(), 0);

    msg = consumer.receive(10, TimeUnit.MILLISECONDS);
    assertNotNull(msg);
    assertEquals(consumer.getAvailablePermits(), 1);

    msg = consumer.receive(10, TimeUnit.MILLISECONDS);
    assertNull(msg);
    assertEquals(consumer.getAvailablePermits(), 1);
}
 
Example #16
Source Project: pulsar   Author: apache   File: TopicFromMessageTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeOut = TEST_TIMEOUT)
public void testSingleTopicConsumerNoBatchShortName() throws Exception {
    try (Consumer<byte[]> consumer = pulsarClient.newConsumer()
            .topic("topic1").subscriptionName("sub1").subscribe();
         Producer<byte[]> producer = pulsarClient.newProducer()
            .topic("topic1").enableBatching(false).create()) {
        producer.send("foobar".getBytes());
        Assert.assertEquals(consumer.receive().getTopicName(), "persistent://public/default/topic1");
    }
}
 
Example #17
Source Project: kop   Author: streamnative   File: GroupMetadataManager.java    License: Apache License 2.0 5 votes vote down vote up
CompletableFuture<Producer<ByteBuffer>> getOffsetsTopicProducer(int partitionId) {
    return offsetsProducers.computeIfAbsent(partitionId,
        id -> {
            if (log.isDebugEnabled()) {
                log.debug("Will create Partitioned producer: {}",
                    offsetConfig.offsetsTopicName() + PARTITIONED_TOPIC_SUFFIX + id);
            }
            return metadataTopicProducerBuilder.clone()
                .topic(offsetConfig.offsetsTopicName() + PARTITIONED_TOPIC_SUFFIX + id)
                .createAsync();
        });
}
 
Example #18
Source Project: pulsar   Author: apache   File: TestProxy.java    License: Apache License 2.0 5 votes vote down vote up
private void testProxy(String serviceUrl, String httpServiceUrl) throws Exception {
    final String tenant = "proxy-test-" + randomName(10);
    final String namespace = tenant + "/ns1";
    final String topic = "persistent://" + namespace + "/topic1";

    @Cleanup
    PulsarAdmin admin = PulsarAdmin.builder()
        .serviceHttpUrl(httpServiceUrl)
        .build();

    admin.tenants().createTenant(tenant,
            new TenantInfo(Collections.emptySet(), Collections.singleton(pulsarCluster.getClusterName())));

    admin.namespaces().createNamespace(namespace, Collections.singleton(pulsarCluster.getClusterName()));

    @Cleanup
    PulsarClient client = PulsarClient.builder()
        .serviceUrl(serviceUrl)
        .build();

    client.newConsumer()
            .topic(topic)
            .subscriptionName("sub1")
            .subscribe()
            .close();

    @Cleanup
    Producer<String> producer = client.newProducer(Schema.STRING)
            .topic(topic)
            .create();
    producer.send("content-0");
    producer.send("content-1");

    for (int i = 0; i < 10; i++) {
        // Ensure we can get the stats for the topic irrespective of which broker the proxy decides to connect to
        TopicStats stats = admin.topics().getStats(topic);
        assertEquals(stats.publishers.size(), 1);
    }
}
 
Example #19
Source Project: pulsar   Author: apache   File: BrokerClientIntegrationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testProducerConsumerWithSpecifiedReaderAndWriter() throws PulsarClientException {
    final String topicName = "persistent://my-property/my-ns/my-topic1";
    ObjectMapper mapper = new ObjectMapper();
    SchemaReader<TestMessageObject> reader = Mockito.spy(new JacksonJsonReader<>(mapper, TestMessageObject.class));
    SchemaWriter<TestMessageObject> writer = Mockito.spy(new JacksonJsonWriter<>(mapper));

    SchemaDefinition<TestMessageObject> schemaDefinition = new SchemaDefinitionBuilderImpl<TestMessageObject>()
            .withPojo(TestMessageObject.class)
            .withSchemaReader(reader)
            .withSchemaWriter(writer)
            .build();
    Schema<TestMessageObject> schema = Schema.JSON(schemaDefinition);
    PulsarClient client =  PulsarClient.builder()
            .serviceUrl(lookupUrl.toString())
            .build();

    try(Producer<TestMessageObject> producer = client.newProducer(schema).topic(topicName).create();
        Consumer<TestMessageObject> consumer = client.newConsumer(schema).topic(topicName).subscriptionName("my-subscriber-name").subscribe()) {
        assertNotNull(producer);
        assertNotNull(consumer);

        TestMessageObject object = new TestMessageObject();
        object.setValue("fooooo");
        producer.newMessage().value(object).send();

        TestMessageObject testObject = consumer.receive().getValue();
        Assert.assertEquals(object.getValue(), testObject.getValue());

        Mockito.verify(writer, Mockito.times(1)).write(Mockito.any());
        Mockito.verify(reader, Mockito.times(1)).read(Mockito.any(byte[].class));
    }
}
 
Example #20
Source Project: pulsar   Author: apache   File: MessageParserTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testWithoutBatches() throws Exception {
    String topic = "persistent://my-tenant/my-ns/my-topic";
    TopicName topicName = TopicName.get(topic);

    int n = 10;

    try (Producer<String> producer = pulsarClient.newProducer(Schema.STRING).enableBatching(false).topic(topic)
            .create()) {
        for (int i = 0; i < n; i++) {
            producer.send("hello-" + i);
        }
    }

    // Read through raw data
    ManagedCursor cursor = ((PersistentTopic) pulsar.getBrokerService().getTopicReference(topic).get())
            .getManagedLedger().newNonDurableCursor(PositionImpl.earliest);

    for (int i = 0; i < n; i++) {
        Entry entry = cursor.readEntriesOrWait(1).get(0);

        List<RawMessage> messages = Lists.newArrayList();

        try {
            MessageParser.parseMessage(topicName, entry.getLedgerId(), entry.getEntryId(), entry.getDataBuffer(),
                    (message) -> {
                        messages.add(message);
                    }, Commands.DEFAULT_MAX_MESSAGE_SIZE);
        } finally {
            entry.release();
        }

        assertEquals(messages.size(), 1);

        assertEquals(messages.get(0).getData(), Unpooled.wrappedBuffer(("hello-" + i).getBytes()));

        messages.forEach(RawMessage::release);
    }
}
 
Example #21
Source Project: pulsar   Author: apache   File: AntiAffinityNamespaceGroupTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * It verifies that load-manager::shouldAntiAffinityNamespaceUnload checks that unloading should only happen if all
 * brokers have same number of anti-affinity namespaces
 *
 * @throws Exception
 */
@Test
public void testLoadSheddingWithAntiAffinityNamespace() throws Exception {

    final String namespace = "my-tenant/use/my-ns";
    final int totalNamespaces = 5;
    final String namespaceAntiAffinityGroup = "my-antiaffinity";
    final String bundle = "0x00000000_0xffffffff";

    admin1.tenants().createTenant("my-tenant",
            new TenantInfo(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("use")));

    for (int i = 0; i < totalNamespaces; i++) {
        final String ns = namespace + i;
        admin1.namespaces().createNamespace(ns);
        admin1.namespaces().setNamespaceAntiAffinityGroup(ns, namespaceAntiAffinityGroup);
    }

    PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(pulsar1.getSafeWebServiceAddress()).build();
    Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://" + namespace + "0/my-topic1")
            .create();
    ModularLoadManagerImpl loadManager = (ModularLoadManagerImpl) ((ModularLoadManagerWrapper) pulsar1
            .getLoadManager().get()).getLoadManager();

    pulsar1.getBrokerService().updateRates();
    loadManager.updateAll();

    assertTrue(loadManager.shouldAntiAffinityNamespaceUnload(namespace + "0", bundle, primaryHost));
    producer.close();
    pulsarClient.close();
}
 
Example #22
Source Project: pulsar   Author: apache   File: ReaderTest.java    License: Apache License 2.0 5 votes vote down vote up
private Set<String> publishMessages(String topic, int count, boolean enableBatch) throws Exception {
    Set<String> keys = new HashSet<>();
    ProducerBuilder<byte[]> builder = pulsarClient.newProducer();
    builder.messageRoutingMode(MessageRoutingMode.SinglePartition);
    builder.maxPendingMessages(count);
    // disable periodical flushing
    builder.batchingMaxPublishDelay(1, TimeUnit.DAYS);
    builder.topic(topic);
    if (enableBatch) {
        builder.enableBatching(true);
        builder.batchingMaxMessages(count);
    } else {
        builder.enableBatching(false);
    }
    try (Producer<byte[]> producer = builder.create()) {
        Future<?> lastFuture = null;
        for (int i = 0; i < count; i++) {
            String key = "key"+i;
            byte[] data = ("my-message-" + i).getBytes();
            lastFuture = producer.newMessage().key(key).value(data).sendAsync();
            keys.add(key);
        }
        producer.flush();
        lastFuture.get();
    }
    return keys;
}
 
Example #23
Source Project: tutorials   Author: eugenp   File: ProducerUnitTest.java    License: MIT License 5 votes vote down vote up
public static void main(String[] args) throws IOException {
    // Create a Pulsar client instance. A single instance can be shared across many
    // producers and consumer within the same application
    PulsarClient client = PulsarClient.builder()
            .serviceUrl(SERVICE_URL)
            .build();

    // Configure producer specific settings
    Producer<byte[]> producer = client.newProducer()
            // Set the topic
            .topic(TOPIC_NAME)
            // Enable compression
            .compressionType(CompressionType.LZ4)
            .create();

    // Once the producer is created, it can be used for the entire application life-cycle
    System.out.println("Created producer for the topic "+TOPIC_NAME);

    // Send 5 test messages
    IntStream.range(1, 5).forEach(i -> {
        String content = String.format("hi-pulsar-%d", i);

        // Build a message object
        Message<byte[]> msg = MessageBuilder.create()
                .setContent(content.getBytes())
                .build();

        // Send each message and log message content and ID when successfully received
        try {
            MessageId msgId = producer.send(msg);

            System.out.println("Published message '"+content+"' with the ID "+msgId);
        } catch (PulsarClientException e) {
            System.out.println(e.getMessage());
        }
    });

    client.close();
}
 
Example #24
Source Project: pulsar   Author: apache   File: SubscriptionSeekTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSeekTime() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/testSeekTime";
    String resetTimeStr = "100s";
    long resetTimeInMillis = TimeUnit.SECONDS
            .toMillis(RelativeTimeUtil.parseRelativeTimeInSeconds(resetTimeStr));

    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();

    // Disable pre-fetch in consumer to track the messages received
    org.apache.pulsar.client.api.Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName)
            .subscriptionName("my-subscription").receiverQueueSize(0).subscribe();

    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    assertEquals(topicRef.getSubscriptions().size(), 1);
    PersistentSubscription sub = topicRef.getSubscription("my-subscription");

    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }

    assertEquals(sub.getNumberOfEntriesInBacklog(false), 10);

    long currentTimestamp = System.currentTimeMillis();
    consumer.seek(currentTimestamp);
    assertEquals(sub.getNumberOfEntriesInBacklog(false), 0);

    // Wait for consumer to reconnect
    Thread.sleep(1000);
    consumer.seek(currentTimestamp - resetTimeInMillis);
    assertEquals(sub.getNumberOfEntriesInBacklog(false), 10);
}
 
Example #25
Source Project: pulsar-flink   Author: streamnative   File: FlinkPulsarSinkBase.java    License: Apache License 2.0 5 votes vote down vote up
protected <R> Producer<R> getProducer(String topic) {
    if (forcedTopic) {
        return (Producer<R>) singleProducer;
    }

    if (topic2Producer.containsKey(topic)) {
        return (Producer<R>) topic2Producer.get(topic);
    } else {
        uploadSchema(topic);
        Producer p = createProducer(clientConfigurationData, producerConf, topic, getPulsarSchema());
        topic2Producer.put(topic, p);
        return (Producer<R>) p;
    }
}
 
Example #26
Source Project: pulsar   Author: apache   File: ProxyKeyStoreTlsTestWithoutAuth.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPartitions() throws Exception {
    @Cleanup
    PulsarClient client = internalSetUpForClient(true, proxyService.getServiceUrlTls());
    String topicName = "persistent://sample/test/local/partitioned-topic" + System.currentTimeMillis();
    TenantInfo tenantInfo = createDefaultTenantInfo();
    admin.tenants().createTenant("sample", tenantInfo);
    admin.topics().createPartitionedTopic(topicName, 2);

    @Cleanup
    Producer<byte[]> producer = client.newProducer(Schema.BYTES).topic(topicName)
            .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();

    // Create a consumer directly attached to broker
    @Cleanup
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName)
            .subscriptionName("my-sub").subscribe();

    for (int i = 0; i < 10; i++) {
        producer.send("test".getBytes());
    }

    for (int i = 0; i < 10; i++) {
        Message<byte[]> msg = consumer.receive(1, TimeUnit.SECONDS);
        checkNotNull(msg);
    }
}
 
Example #27
Source Project: pulsar   Author: apache   File: AdminApiTest2.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPreciseBacklogForPartitionedTopic() throws PulsarClientException, PulsarAdminException, InterruptedException {
    final String topic = "persistent://prop-xyz/ns1/precise-back-log-for-partitioned-topic";
    admin.topics().createPartitionedTopic(topic, 2);
    final String subName = "sub-name";

    @Cleanup
    PulsarClient client = PulsarClient.builder().serviceUrl(pulsar.getWebServiceAddress()).build();

    @Cleanup
    Consumer<byte[]> consumer = client.newConsumer()
        .topic(topic)
        .subscriptionName(subName)
        .subscribe();

    @Cleanup
    Producer<byte[]> producer = client.newProducer()
        .topic(topic)
        .enableBatching(false)
        .create();

    producer.send("message-1".getBytes(StandardCharsets.UTF_8));
    Message<byte[]> message = consumer.receive();
    assertNotNull(message);

    // Mock the entries added count. Default is disable the precise backlog, so the backlog is entries added count - consumed count
    // Since message have not acked, so the backlog is 10
    for (int i = 0; i < 2; i++) {
        PersistentSubscription subscription = (PersistentSubscription)pulsar.getBrokerService().getTopicReference(topic + "-partition-" + i).get().getSubscription(subName);
        assertNotNull(subscription);
        ((ManagedLedgerImpl)subscription.getCursor().getManagedLedger()).setEntriesAddedCounter(10L);
    }

    TopicStats topicStats = admin.topics().getPartitionedStats(topic, false);
    assertEquals(topicStats.subscriptions.get(subName).msgBacklog, 20);

    topicStats = admin.topics().getPartitionedStats(topic, false, true);
    assertEquals(topicStats.subscriptions.get(subName).msgBacklog, 1);
}
 
Example #28
Source Project: java-specialagent   Author: opentracing-contrib   File: PulsarClientTest.java    License: Apache License 2.0 5 votes vote down vote up
private static void test(final MockTracer tracer, final boolean async) throws Exception {
  if (!isJdkSupported) {
    logger.warning("jdk" + System.getProperty("java.version") + " is not supported by Pulsar");
    return;
  }

  try (
    final PulsarClient client = PulsarClient.builder().serviceUrl(pulsarService.getBrokerServiceUrl()).build();
    final Consumer<byte[]> consumer = client.newConsumer().topic("my-topic").subscriptionName("my-subscription").subscribe();
    final Producer<byte[]> producer = client.newProducer().topic("my-topic").create();
  ) {
    if (async)
      producer.sendAsync("My message".getBytes()).get(15, TimeUnit.SECONDS);
    else
      producer.send("My message".getBytes());

    final Message<byte[]> message;
    if (async)
      message = consumer.receiveAsync().get(15, TimeUnit.SECONDS);
    else
      message = consumer.receive();

    System.out.println("Message received: " + new String(message.getData()));
    consumer.acknowledge(message);
  }

  final List<MockSpan> spans = tracer.finishedSpans();
  assertEquals(2, spans.size());
  assertNull(tracer.activeSpan());
  for (final MockSpan span : spans)
    assertEquals(PulsarClientAgentIntercept.COMPONENT_NAME, span.tags().get(Tags.COMPONENT.getKey()));

  assertEquals(spans.get(0).context().traceId(), spans.get(1).context().traceId());
}
 
Example #29
Source Project: pulsar   Author: apache   File: SampleProducer.java    License: Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws PulsarClientException, InterruptedException, IOException {
    PulsarClient client = PulsarClient.builder().serviceUrl("pulsar://localhost:6650").build();

    Producer<byte[]> producer = client.newProducer().topic("persistent://my-tenant/my-ns/my-topic").create();

    for (int i = 0; i < 10; i++) {
        producer.send("my-message".getBytes());
    }

    client.close();
}
 
Example #30
Source Project: pulsar   Author: apache   File: BatchMessageTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test(dataProvider = "containerBuilder")
public void testRetrieveSequenceIdGenerated(BatcherBuilder builder) throws Exception {

    int numMsgs = 10;
    final String topicName = "persistent://prop/ns-abc/testRetrieveSequenceIdGenerated-" + UUID.randomUUID();
    final String subscriptionName = "sub-1";

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
            .subscriptionType(SubscriptionType.Shared).subscribe();

    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
            .batchingMaxPublishDelay(5, TimeUnit.SECONDS).batchingMaxMessages(numMsgs).enableBatching(true)
            .batcherBuilder(builder)
            .create();

    List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
    for (int i = 0; i < numMsgs; i++) {
        byte[] message = ("my-message-" + i).getBytes();
        sendFutureList.add(producer.sendAsync(message));
    }
    FutureUtil.waitForAll(sendFutureList).get();

    for (int i = 0; i < numMsgs; i++) {
        Message<byte[]> received = consumer.receive();
        Assert.assertEquals(received.getSequenceId(), i);
        consumer.acknowledge(received);
    }

    producer.close();
    consumer.close();
}