Java Code Examples for org.apache.pulsar.client.api.PulsarClient

The following examples show how to use org.apache.pulsar.client.api.PulsarClient. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: pulsar   Source File: ClientBuilderImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public PulsarClient build() throws PulsarClientException {
    if (StringUtils.isBlank(conf.getServiceUrl()) && conf.getServiceUrlProvider() == null) {
        throw new IllegalArgumentException("service URL or service URL provider needs to be specified on the ClientBuilder object.");
    }
    if (StringUtils.isNotBlank(conf.getServiceUrl()) && conf.getServiceUrlProvider() != null) {
        throw new IllegalArgumentException("Can only chose one way service URL or service URL provider.");
    }
    if (conf.getServiceUrlProvider() != null) {
        if (StringUtils.isBlank(conf.getServiceUrlProvider().getServiceUrl())) {
            throw new IllegalArgumentException("Cannot get service url from service url provider.");
        } else {
            conf.setServiceUrl(conf.getServiceUrlProvider().getServiceUrl());
        }
    }
    PulsarClient client = new PulsarClientImpl(conf);
    if (conf.getServiceUrlProvider() != null) {
        conf.getServiceUrlProvider().initialize(client);
    }
    return client;
}
 
Example 2
Source Project: testcontainers-java   Source File: PulsarContainerTest.java    License: MIT License 6 votes vote down vote up
protected void testPulsarFunctionality(String pulsarBrokerUrl) throws Exception {

        try (
            PulsarClient client = PulsarClient.builder()
                .serviceUrl(pulsarBrokerUrl)
                .build();
            Consumer consumer = client.newConsumer()
                .topic(TEST_TOPIC)
                .subscriptionName("test-subs")
                .subscribe();
            Producer<byte[]> producer = client.newProducer()
                .topic(TEST_TOPIC)
                .create()
        ) {

            producer.send("test containers".getBytes());
            CompletableFuture<Message> future = consumer.receiveAsync();
            Message message = future.get(5, TimeUnit.SECONDS);

            assertThat(new String(message.getData()))
                .isEqualTo("test containers");
        }
    }
 
Example 3
Source Project: hazelcast-jet-contrib   Source File: PulsarSinkBuilder.java    License: Apache License 2.0 6 votes vote down vote up
private PulsarSinkContext(
        @Nonnull ILogger logger,
        @Nonnull String topic,
        @Nonnull PulsarClient client,
        @Nonnull Map<String, Object> producerConfig,
        @Nonnull SupplierEx<Schema<M>> schemaSupplier,
        @Nonnull FunctionEx<? super E, M> extractValueFn,
        @Nullable FunctionEx<? super E, String> extractKeyFn,
        @Nullable FunctionEx<? super E, Map<String, String>> extractPropertiesFn,
        @Nullable FunctionEx<? super E, Long> extractTimestampFn
) throws PulsarClientException {
    this.logger = logger;
    this.client = client;
    this.producer = client.newProducer(schemaSupplier.get())
                          .topic(topic)
                          .loadConf(producerConfig)
                          .create();
    this.extractKeyFn = extractKeyFn;
    this.extractValueFn = extractValueFn;
    this.extractPropertiesFn = extractPropertiesFn;
    this.extractTimestampFn = extractTimestampFn;
}
 
Example 4
Source Project: pulsar   Source File: ThreadRuntimeFactory.java    License: Apache License 2.0 6 votes vote down vote up
private void initialize(String threadGroupName, PulsarClient pulsarClient, String storageServiceUrl,
                        SecretsProviderConfigurator secretsProviderConfigurator, SecretsProvider secretsProvider,
                        CollectorRegistry collectorRegistry,  String narExtractionDirectory, ClassLoader rootClassLoader) {
    if (rootClassLoader == null) {
        rootClassLoader = Thread.currentThread().getContextClassLoader();
    }

    this.rootClassLoader = rootClassLoader;
    this.secretsProviderConfigurator = secretsProviderConfigurator;
    this.defaultSecretsProvider = secretsProvider;
    this.fnCache = new FunctionCacheManagerImpl(rootClassLoader);
    this.threadGroup = new ThreadGroup(threadGroupName);
    this.pulsarClient = pulsarClient;
    this.storageServiceUrl = storageServiceUrl;
    this.collectorRegistry = collectorRegistry;
    this.narExtractionDirectory = narExtractionDirectory;
}
 
Example 5
Source Project: pulsar   Source File: PulsarSpoutTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNoSharedConsumer() throws Exception {
    TopicStats topicStats = admin.topics().getStats(topic);
    assertEquals(topicStats.subscriptions.get(subscriptionName).consumers.size(), 1);
    pulsarSpoutConf.setSharedConsumerEnabled(false);
    PulsarSpout otherSpout = new PulsarSpout(pulsarSpoutConf, PulsarClient.builder());
    MockSpoutOutputCollector otherMockCollector = new MockSpoutOutputCollector();
    SpoutOutputCollector collector = new SpoutOutputCollector(otherMockCollector);
    TopologyContext context = mock(TopologyContext.class);
    when(context.getThisComponentId()).thenReturn("test-spout-" + methodName);
    when(context.getThisTaskId()).thenReturn(1);
    otherSpout.open(Maps.newHashMap(), context, collector);

    topicStats = admin.topics().getStats(topic);
    assertEquals(topicStats.subscriptions.get(subscriptionName).consumers.size(), 2);

    otherSpout.close();

    topicStats = admin.topics().getStats(topic);
    assertEquals(topicStats.subscriptions.get(subscriptionName).consumers.size(), 1);
}
 
Example 6
Source Project: pulsar   Source File: SampleConsumerWithSchema.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws PulsarClientException, JsonProcessingException {

        PulsarClient pulsarClient = PulsarClient.builder().serviceUrl("http://localhost:8080").build();

        Consumer<JsonPojo> consumer = pulsarClient.newConsumer(JSONSchema.of
                (SchemaDefinition.<JsonPojo>builder().withPojo(JsonPojo.class).build())) //
                .topic("persistent://my-property/use/my-ns/my-topic") //
                .subscriptionName("my-subscription-name").subscribe();

        Message<JsonPojo> msg = null;

        for (int i = 0; i < 100; i++) {
            msg = consumer.receive();
            // do something
            System.out.println("Received: " + msg.getValue().content);
        }

        // Acknowledge the consumption of all messages at once
        consumer.acknowledgeCumulative(msg);
        pulsarClient.close();
    }
 
Example 7
Source Project: liiklus   Source File: PulsarRecordsStorageConfiguration.java    License: MIT License 6 votes vote down vote up
@SneakyThrows
PulsarClient createClient(PulsarProperties pulsarProperties) {
    var clientBuilder = PulsarClient.builder()
            .serviceUrl(pulsarProperties.getServiceUrl());

    pulsarProperties.getTlsTrustCertsFilePath().ifPresent(clientBuilder::tlsTrustCertsFilePath);
    pulsarProperties.getAuthPluginClassName().ifPresent(authClass -> {
        try {
            clientBuilder.authentication(authClass, pulsarProperties.getAuthPluginParams());
        } catch (PulsarClientException.UnsupportedAuthenticationException e) {
            throw new IllegalStateException(e);
        }
    });

    return clientBuilder.build();
}
 
Example 8
Source Project: pulsar   Source File: SampleConsumer.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws PulsarClientException, InterruptedException {

        PulsarClient pulsarClient = PulsarClient.builder().serviceUrl("pulsar://localhost:6650").build();

        Consumer<byte[]> consumer = pulsarClient.newConsumer() //
                .topic("persistent://my-tenant/my-ns/my-topic") //
                .subscriptionName("my-subscription-name").subscribe();

        Message<byte[]> msg = null;

        for (int i = 0; i < 100; i++) {
            msg = consumer.receive();
            // do something
            System.out.println("Received: " + new String(msg.getData()));
        }

        // Acknowledge the consumption of all messages at once
        try {
            consumer.acknowledgeCumulative(msg);
        } catch (Exception e) {
            consumer.reconsumeLater(msg, 10, TimeUnit.SECONDS);
        }
       
        pulsarClient.close();
    }
 
Example 9
Source Project: pulsar   Source File: BuildersTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void enableTlsTest() {
    ClientBuilderImpl builder = (ClientBuilderImpl)PulsarClient.builder().serviceUrl("pulsar://service:6650");
    assertFalse(builder.conf.isUseTls());
    assertEquals(builder.conf.getServiceUrl(), "pulsar://service:6650");

    builder = (ClientBuilderImpl)PulsarClient.builder().serviceUrl("http://service:6650");
    assertFalse(builder.conf.isUseTls());
    assertEquals(builder.conf.getServiceUrl(), "http://service:6650");

    builder = (ClientBuilderImpl)PulsarClient.builder().serviceUrl("pulsar+ssl://service:6650");
    assertTrue(builder.conf.isUseTls());
    assertEquals(builder.conf.getServiceUrl(), "pulsar+ssl://service:6650");

    builder = (ClientBuilderImpl)PulsarClient.builder().serviceUrl("https://service:6650");
    assertTrue(builder.conf.isUseTls());
    assertEquals(builder.conf.getServiceUrl(), "https://service:6650");

    builder = (ClientBuilderImpl)PulsarClient.builder().serviceUrl("pulsar://service:6650").enableTls(true);
    assertTrue(builder.conf.isUseTls());
    assertEquals(builder.conf.getServiceUrl(), "pulsar://service:6650");

    builder = (ClientBuilderImpl)PulsarClient.builder().serviceUrl("pulsar+ssl://service:6650").enableTls(false);
    assertTrue(builder.conf.isUseTls());
    assertEquals(builder.conf.getServiceUrl(), "pulsar+ssl://service:6650");
}
 
Example 10
Source Project: pulsar   Source File: PulsarBoltTest.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void setup() throws Exception {
    super.internalSetup();
    super.producerBaseSetup();

    serviceUrl = pulsar.getWebServiceAddress();

    pulsarBoltConf = new PulsarBoltConfiguration();
    pulsarBoltConf.setServiceUrl(serviceUrl);
    pulsarBoltConf.setTopic(topic);
    pulsarBoltConf.setTupleToMessageMapper(tupleToMessageMapper);
    pulsarBoltConf.setMetricsTimeIntervalInSecs(60);
    bolt = new PulsarBolt(pulsarBoltConf, PulsarClient.builder());
    mockCollector = new MockOutputCollector();
    OutputCollector collector = new OutputCollector(mockCollector);
    TopologyContext context = mock(TopologyContext.class);
    when(context.getThisComponentId()).thenReturn("test-bolt-" + methodName);
    when(context.getThisTaskId()).thenReturn(0);
    bolt.prepare(Maps.newHashMap(), context, collector);
    consumer = pulsarClient.newConsumer().topic(topic).subscriptionName(subscriptionName).subscribe();
}
 
Example 11
protected void internalSetUpForClient(boolean addCertificates, String lookupUrl) throws Exception {
    if (pulsarClient != null) {
        pulsarClient.close();
    }

    ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(lookupUrl)
            .enableTls(true)
            .useKeyStoreTls(true)
            .tlsTrustStorePath(BROKER_TRUSTSTORE_FILE_PATH)
            .tlsTrustStorePassword(BROKER_TRUSTSTORE_PW)
            .allowTlsInsecureConnection(false)
            .operationTimeout(1000, TimeUnit.MILLISECONDS);
    if (addCertificates) {
        Map<String, String> authParams = new HashMap<>();
        authParams.put(AuthenticationKeyStoreTls.KEYSTORE_TYPE, KEYSTORE_TYPE);
        authParams.put(AuthenticationKeyStoreTls.KEYSTORE_PATH, CLIENT_KEYSTORE_FILE_PATH);
        authParams.put(AuthenticationKeyStoreTls.KEYSTORE_PW, CLIENT_KEYSTORE_PW);
        clientBuilder.authentication(AuthenticationKeyStoreTls.class.getName(), authParams);
    }
    pulsarClient = clientBuilder.build();
}
 
Example 12
Source Project: pulsar   Source File: ProducerSparkReceiverData.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  if (args.length < 2) {
    System.err.println("Missing parameters!");
    System.err.println("Usage: <pulsar-service-url> <topic>");
    return;
  }

  System.out.println("Parameters:");
  System.out.println("\tServiceUrl:\t" + args[0]);
  System.out.println("\tTopic:\t" + args[1]);

  try (PulsarClient client = PulsarClient.builder().serviceUrl(args[0]).build()) {
    try (Producer<byte[]> producer = client.newProducer().topic(args[1]).create()) {
      for (int i = 0; i < 100; i++) {
        producer.send(("producer spark streaming msg").getBytes(StandardCharsets.UTF_8));
      }
    }
  }

  System.out.println("producer spark streaming msg end ...");
}
 
Example 13
Source Project: pulsar   Source File: PulsarSink.java    License: Apache License 2.0 6 votes vote down vote up
public Producer<T> createProducer(PulsarClient client, String topic, String producerName, Schema<T> schema)
        throws PulsarClientException {
    ProducerBuilder<T> builder = client.newProducer(schema)
            .blockIfQueueFull(true)
            .enableBatching(true)
            .batchingMaxPublishDelay(10, TimeUnit.MILLISECONDS)
            .compressionType(CompressionType.LZ4)
            .hashingScheme(HashingScheme.Murmur3_32Hash) //
            .messageRoutingMode(MessageRoutingMode.CustomPartition)
            .messageRouter(FunctionResultRouter.of())
            // set send timeout to be infinity to prevent potential deadlock with consumer
            // that might happen when consumer is blocked due to unacked messages
            .sendTimeout(0, TimeUnit.SECONDS)
            .topic(topic);
    if (producerName != null) {
        builder.producerName(producerName);
    }

    return builder.properties(properties).create();
}
 
Example 14
Source Project: pulsar   Source File: AdminApiTlsAuthTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testDeleteNamespace() throws Exception {
    try (PulsarAdmin admin = buildAdminClient("admin")) {
        log.info("Creating tenant");
        admin.tenants().createTenant("tenant1",
                                     new TenantInfo(ImmutableSet.of("admin"), ImmutableSet.of("test")));
        log.info("Creating namespace, and granting perms to user1");
        admin.namespaces().createNamespace("tenant1/ns1", ImmutableSet.of("test"));
        admin.namespaces().grantPermissionOnNamespace("tenant1/ns1", "user1", ImmutableSet.of(AuthAction.produce));

        log.info("user1 produces some messages");
        try (PulsarClient client = buildClient("user1");
             Producer<String> producer = client.newProducer(Schema.STRING).topic("tenant1/ns1/foobar").create()) {
            producer.send("foobar");
        }

        log.info("Deleting the topic");
        admin.topics().delete("tenant1/ns1/foobar", true);

        log.info("Deleting namespace");
        admin.namespaces().deleteNamespace("tenant1/ns1");
    }
}
 
Example 15
Source Project: pulsar   Source File: ProxyKeyStoreTlsTestWithoutAuth.java    License: Apache License 2.0 6 votes vote down vote up
protected PulsarClient internalSetUpForClient(boolean addCertificates, String lookupUrl) throws Exception {
    ClientBuilder clientBuilder = PulsarClient.builder()
            .serviceUrl(lookupUrl)
            .enableTls(true)
            .useKeyStoreTls(true)
            .tlsTrustStorePath(BROKER_TRUSTSTORE_FILE_PATH)
            .tlsTrustStorePassword(BROKER_TRUSTSTORE_PW)
            .allowTlsInsecureConnection(false)
            .operationTimeout(1000, TimeUnit.MILLISECONDS);
    if (addCertificates) {
        Map<String, String> authParams = new HashMap<>();
        authParams.put(AuthenticationKeyStoreTls.KEYSTORE_TYPE, KEYSTORE_TYPE);
        authParams.put(AuthenticationKeyStoreTls.KEYSTORE_PATH, CLIENT_KEYSTORE_FILE_PATH);
        authParams.put(AuthenticationKeyStoreTls.KEYSTORE_PW, CLIENT_KEYSTORE_PW);
        clientBuilder.authentication(AuthenticationKeyStoreTls.class.getName(), authParams);
    }
    return clientBuilder.build();
}
 
Example 16
protected void internalSetUpForClient(boolean addCertificates, String lookupUrl) throws Exception {
    if (pulsarClient != null) {
        pulsarClient.close();
    }

    Set<String> tlsProtocols = Sets.newConcurrentHashSet();
    tlsProtocols.add("TLSv1.2");

    ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(lookupUrl)
            .enableTls(true)
            .useKeyStoreTls(true)
            .tlsTrustStorePath(BROKER_TRUSTSTORE_FILE_PATH)
            .tlsTrustStorePassword(BROKER_TRUSTSTORE_PW)
            .allowTlsInsecureConnection(false)
            .tlsProtocols(tlsProtocols)
            .operationTimeout(1000, TimeUnit.MILLISECONDS);
    if (addCertificates) {
        Map<String, String> authParams = new HashMap<>();
        authParams.put(AuthenticationKeyStoreTls.KEYSTORE_TYPE, KEYSTORE_TYPE);
        authParams.put(AuthenticationKeyStoreTls.KEYSTORE_PATH, CLIENT_KEYSTORE_FILE_PATH);
        authParams.put(AuthenticationKeyStoreTls.KEYSTORE_PW, CLIENT_KEYSTORE_PW);
        clientBuilder.authentication(AuthenticationKeyStoreTls.class.getName(), authParams);
    }
    pulsarClient = clientBuilder.build();
}
 
Example 17
Source Project: skywalking   Source File: CaseController.java    License: Apache License 2.0 6 votes vote down vote up
@RequestMapping("/healthCheck")
@ResponseBody
public String healthCheck() throws InterruptedException {
    try (PulsarClient pulsarClient = PulsarClient.builder()
                                                 .serviceUrl(PULSAR_DOMAIN + serviceUrl)
                                                 .build(); Producer<byte[]> producer = pulsarClient.newProducer()
                                                                                                   .topic("healthCheck")
                                                                                                   .create()) {
        if (producer.isConnected()) {
            return "Success";
        } else {
            throw new RuntimeException("Health check error, the reason is test producer is disconnected!");
        }
    } catch (PulsarClientException e) {
        if (e instanceof PulsarClientException.BrokerMetadataException) {
            // Broker is not ready, retry here
            Thread.sleep(1000);
            return healthCheck();
        } else {
            throw new RuntimeException(e);
        }
    }
}
 
Example 18
Source Project: pulsar   Source File: TopicDoesNotExistsTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateConsumerOnNotExistsTopic() throws PulsarClientException, InterruptedException {
    PulsarClient pulsarClient = newPulsarClient(lookupUrl.toString(), 1);
    try {
        pulsarClient.newConsumer()
                .topic("persistent://public/default/" + UUID.randomUUID().toString())
                .subscriptionName("test")
                .subscribe();
        Assert.fail("Create consumer should failed while topic does not exists.");
    } catch (PulsarClientException ignore) {
    }
    Thread.sleep(2000);
    HashedWheelTimer timer = (HashedWheelTimer) ((PulsarClientImpl) pulsarClient).timer();
    Assert.assertEquals(timer.pendingTimeouts(), 0);
    Assert.assertEquals(((PulsarClientImpl) pulsarClient).consumersCount(), 0);
}
 
Example 19
Source Project: pulsar   Source File: KafkaApiTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSimpleProducer() throws Exception {
    String topic = "testSimpleProducer";

    @Cleanup
    PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(getPlainTextServiceUrl()).build();
    org.apache.pulsar.client.api.Consumer<byte[]> pulsarConsumer = pulsarClient.newConsumer().topic(topic)
            .subscriptionName("my-subscription")
            .subscribe();

    Properties props = new Properties();
    props.put("bootstrap.servers", getPlainTextServiceUrl());

    props.put("key.serializer", IntegerSerializer.class.getName());
    props.put("value.serializer", StringSerializer.class.getName());

    Producer<Integer, String> producer = new KafkaProducer<>(props);

    for (int i = 0; i < 10; i++) {
        producer.send(new ProducerRecord<Integer, String>(topic, i, "hello-" + i));
    }

    producer.flush();
    producer.close();

    for (int i = 0; i < 10; i++) {
        Message<byte[]> msg = pulsarConsumer.receive(1, TimeUnit.SECONDS);
        assertEquals(new String(msg.getData()), "hello-" + i);
        pulsarConsumer.acknowledge(msg);
    }
}
 
Example 20
Source Project: tutorials   Source File: ExclusiveSubscriptionUnitTest.java    License: MIT License 5 votes vote down vote up
public static void main(String[] args) throws PulsarClientException {
    PulsarClient client = PulsarClient.builder()
            .serviceUrl(SERVICE_URL)
            .build();

    Producer<byte[]> producer = client.newProducer()
            .topic(TOPIC_NAME)
            .create();

    ConsumerBuilder<byte[]> consumer1 = client.newConsumer()
            .topic(TOPIC_NAME)
            .subscriptionName(SUBSCRIPTION_NAME)
            .subscriptionType(SUBSCRIPTION_TYPE);

    ConsumerBuilder<byte[]> consumer2 = client.newConsumer()
            .topic(TOPIC_NAME)
            .subscriptionName(SUBSCRIPTION_NAME)
            .subscriptionType(SUBSCRIPTION_TYPE);

    IntStream.range(0, 999).forEach(i -> {
        Message<byte[]> msg = MessageBuilder.create()
                .setContent(String.format("message-%d", i).getBytes())
                .build();
        try {
            producer.send(msg);
        } catch (PulsarClientException e) {
            System.out.println(e.getMessage());
        }
    });

    // Consumer 1 can subscribe to the topic
    consumer1.subscribe();

    // Consumer 2 cannot due to the exclusive subscription held by consumer 1
    consumer2.subscribeAsync()
            .handle((consumer, exception) -> {
                System.out.println(exception.getMessage());
                return null;
            });
}
 
Example 21
Source Project: pulsar   Source File: ProxyParserTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPartitions() throws Exception {
    TenantInfo tenantInfo = createDefaultTenantInfo();
    admin.tenants().createTenant("sample", tenantInfo);
    PulsarClient client = PulsarClient.builder().serviceUrl(proxyService.getServiceUrl())
            .build();
    admin.topics().createPartitionedTopic("persistent://sample/test/local/partitioned-topic", 2);

    Producer<byte[]> producer = client.newProducer(Schema.BYTES)
        .topic("persistent://sample/test/local/partitioned-topic")
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();

    // Create a consumer directly attached to broker
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://sample/test/local/partitioned-topic")
            .subscriptionName("my-sub").subscribe();

    for (int i = 0; i < 10; i++) {
        producer.send("test".getBytes());
    }

    for (int i = 0; i < 10; i++) {
        Message<byte[]> msg = consumer.receive(1, TimeUnit.SECONDS);
        checkNotNull(msg);
    }

    client.close();
}
 
Example 22
Source Project: pulsar   Source File: PythonSchemaTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Publish from Java and consume from Python
 */
@Test(dataProvider = "ServiceUrls")
public void testJavaPublishPythonConsume(String serviceUrl) throws Exception {
    String nsName = generateNamespaceName();
    pulsarCluster.createNamespace(nsName);

    String topicName = generateTopicName(nsName, "testJavaPublishPythonConsume", true);

    @Cleanup
    PulsarClient client = PulsarClient.builder()
            .serviceUrl(serviceUrl)
            .build();

    // Create subscription to retain data
    client.newConsumer(Schema.JSON(Example1.class))
            .topic(topicName)
            .subscriptionName("my-subscription")
            .subscribe()
            .close();

    @Cleanup
    Producer<Example1> producer = client.newProducer(Schema.JSON(Example1.class))
            .topic(topicName)
            .create();

    Example1 e1 = new Example1();
    e1.setX(1);
    e1.setY(2L);
    producer.send(e1);

    // Verify Python can receive the typed message

    ContainerExecResult res = pulsarCluster.getAnyBroker()
            .execCmd("/pulsar/examples/python-examples/consumer_schema.py", "pulsar://localhost:6650", topicName);
    assertEquals(res.getExitCode(), 0);
}
 
Example 23
Source Project: pulsar   Source File: ReplicatorTestBase.java    License: Apache License 2.0 5 votes vote down vote up
MessageConsumer(URL url, final TopicName dest, String subId) throws Exception {
    this.url = url;
    this.namespace = dest.getNamespace();
    this.topicName = dest.toString();

    client = PulsarClient.builder().serviceUrl(url.toString()).statsInterval(0, TimeUnit.SECONDS).build();

    try {
        consumer = client.newConsumer().topic(topicName).subscriptionName(subId).subscribe();
    } catch (Exception e) {
        client.close();
        throw e;
    }
}
 
Example 24
Source Project: pulsar   Source File: ReplicatorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testReplicatedCluster() throws Exception {

    log.info("--- Starting ReplicatorTest::testReplicatedCluster ---");

    final String namespace = "pulsar/global/repl";
    final String topicName = String.format("persistent://%s/topic1-%d", namespace, System.currentTimeMillis());
    admin1.namespaces().createNamespace(namespace);
    admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r2", "r3"));
    admin1.topics().createPartitionedTopic(topicName, 4);

    PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).statsInterval(0, TimeUnit.SECONDS)
            .build();
    PulsarClient client2 = PulsarClient.builder().serviceUrl(url2.toString()).statsInterval(0, TimeUnit.SECONDS)
            .build();

    Producer<byte[]> producer1 = client1.newProducer().topic(topicName).create();
    org.apache.pulsar.client.api.Consumer<byte[]> consumer1 = client1.newConsumer().topic(topicName).subscriptionName("s1").subscribe();
    org.apache.pulsar.client.api.Consumer<byte[]> consumer2 = client2.newConsumer().topic(topicName).subscriptionName("s1").subscribe();
    byte[] value = "test".getBytes();

    // publish message local only
    TypedMessageBuilder<byte[]> msg = producer1.newMessage().replicationClusters(Lists.newArrayList("r1")).value(value);
    msg.send();
    assertEquals(consumer1.receive().getValue(), value);

    Message<byte[]> msg2 = consumer2.receive(1, TimeUnit.SECONDS);
    if (msg2 != null) {
        fail("msg should have not been replicated to remote cluster");
    }

    consumer1.close();
    consumer2.close();
    producer1.close();

}
 
Example 25
Source Project: java-specialagent   Source File: PulsarClientTest.java    License: Apache License 2.0 5 votes vote down vote up
private static void test(final MockTracer tracer, final boolean async) throws Exception {
  if (!isJdkSupported) {
    logger.warning("jdk" + System.getProperty("java.version") + " is not supported by Pulsar");
    return;
  }

  try (
    final PulsarClient client = PulsarClient.builder().serviceUrl(pulsarService.getBrokerServiceUrl()).build();
    final Consumer<byte[]> consumer = client.newConsumer().topic("my-topic").subscriptionName("my-subscription").subscribe();
    final Producer<byte[]> producer = client.newProducer().topic("my-topic").create();
  ) {
    if (async)
      producer.sendAsync("My message".getBytes()).get(15, TimeUnit.SECONDS);
    else
      producer.send("My message".getBytes());

    final Message<byte[]> message;
    if (async)
      message = consumer.receiveAsync().get(15, TimeUnit.SECONDS);
    else
      message = consumer.receive();

    System.out.println("Message received: " + new String(message.getData()));
    consumer.acknowledge(message);
  }

  final List<MockSpan> spans = tracer.finishedSpans();
  assertEquals(2, spans.size());
  assertNull(tracer.activeSpan());
  for (final MockSpan span : spans)
    assertEquals(PulsarClientAgentIntercept.COMPONENT_NAME, span.tags().get(Tags.COMPONENT.getKey()));

  assertEquals(spans.get(0).context().traceId(), spans.get(1).context().traceId());
}
 
Example 26
Source Project: Groza   Source File: producers.java    License: Apache License 2.0 5 votes vote down vote up
public static void PulsarClient() throws PulsarClientException {
    PulsarClient client = PulsarClient.builder()
            .serviceUrl("pulsar://localhost:6650")
            .build();
    Producer<byte[]> producer = client.newProducer()
            .topic("my-topic")
            .create();
    producer.send("My message".getBytes());
    client.close();
    producer.close();
}
 
Example 27
Source Project: pulsar   Source File: ProxyTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testProducer() throws Exception {
    @Cleanup
    PulsarClient client = PulsarClient.builder().serviceUrl(proxyService.getServiceUrl())
            .build();

    @Cleanup
    Producer<byte[]> producer = client.newProducer()
        .topic("persistent://sample/test/local/producer-topic")
        .create();

    for (int i = 0; i < 10; i++) {
        producer.send("test".getBytes());
    }
}
 
Example 28
public static void main(String[] args) throws PulsarClientException {
    PulsarClient client = PulsarClient.builder()
            .serviceUrl(SERVICE_URL)
            .build();

    log.info("Created a client for the Pulsar cluster running at {}", SERVICE_URL);

    client.newConsumer()
            .topic(TOPIC_NAME)
            .subscriptionName(SUBSCRIPTION_NAME)
            .subscriptionType(SubscriptionType.Shared)
            .subscribeAsync()
            .thenAccept(consumer -> {
                log.info("Consumer created asynchronously for the topic {}", TOPIC_NAME);

                do {
                    consumer.receiveAsync()
                            .handle((msg, ex) -> {
                               if (ex != null) {
                                   ex.printStackTrace();
                               }

                               log.info("Successfully received message with an ID of \"{}\" and a payload of \"{}\"",
                                       new String(msg.getMessageId().toByteArray()),
                                       new String(msg.getData()));
                               return null;
                            });
                } while (true);
            })
            .exceptionally(ex -> {
                log.error(ex.getMessage());
                return null;
            });
}
 
Example 29
Source Project: pulsar   Source File: SparkStreamingPulsarReceiver.java    License: Apache License 2.0 5 votes vote down vote up
public void onStart() {
    try {
        pulsarClient = PulsarClient.builder().serviceUrl(serviceUrl).authentication(authentication).build();
        consumer = ((PulsarClientImpl) pulsarClient).subscribeAsync(conf).join();
    } catch (Exception e) {
        LOG.error("Failed to start subscription : {}", e.getMessage());
        restart("Restart a consumer");
    }
}
 
Example 30
Source Project: pulsar   Source File: ReplicatorGlobalNSTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * If local cluster is removed from the global namespace then all topics under that namespace should be deleted from
 * the cluster.
 *
 * @throws Exception
 */
@Test
public void testRemoveLocalClusterOnGlobalNamespace() throws Exception {
    log.info("--- Starting ReplicatorTest::testRemoveLocalClusterOnGlobalNamespace ---");

    final String namespace = "pulsar/global/removeClusterTest";
    admin1.namespaces().createNamespace(namespace);
    admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r2", "r3"));

    final String topicName = "persistent://" + namespace + "/topic";

    PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).statsInterval(0, TimeUnit.SECONDS)
            .build();
    PulsarClient client2 = PulsarClient.builder().serviceUrl(url2.toString()).statsInterval(0, TimeUnit.SECONDS)
            .build();

    ProducerImpl<byte[]> producer1 = (ProducerImpl<byte[]>) client1.newProducer().topic(topicName)
            .enableBatching(false).messageRoutingMode(MessageRoutingMode.SinglePartition).create();
    ConsumerImpl<byte[]> consumer1 = (ConsumerImpl<byte[]>) client1.newConsumer().topic(topicName)
            .subscriptionName("sub1").subscribe();
    ConsumerImpl<byte[]> consumer2 = (ConsumerImpl<byte[]>) client2.newConsumer().topic(topicName)
            .subscriptionName("sub1").subscribe();

    admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r2", "r3"));

    MockedPulsarServiceBaseTest
            .retryStrategically((test) -> !pulsar1.getBrokerService().getTopics().containsKey(topicName), 50, 150);

    Assert.assertFalse(pulsar1.getBrokerService().getTopics().containsKey(topicName));
    Assert.assertFalse(producer1.isConnected());
    Assert.assertFalse(consumer1.isConnected());
    Assert.assertTrue(consumer2.isConnected());

    client1.close();
    client2.close();
}