org.apache.pulsar.client.api.PulsarClientException Java Examples

@Override
void receiveIndividualMessagesFromBatch(MessageMetadata msgMetadata, int redeliveryCount,
        List<Long> ackSet,
        ByteBuf uncompressedPayload, MessageIdData messageId, ClientCnx cnx) {
    log.warn(
            "Closing consumer [{}]-[{}] due to unsupported received batch-message with zero receiver queue size",
            subscription, consumerName);
    // close connection
    closeAsync().handle((ok, e) -> {
        // notify callback with failure result
        notifyPendingReceivedCallback(null,
                new PulsarClientException.InvalidMessageException(
                        format("Unsupported Batch message with 0 size receiver queue for [%s]-[%s] ",
                                subscription, consumerName)));
        return null;
    });
}
 

@Override
public void reconsumeLater(Message<?> message, long delayTime, TimeUnit unit) throws PulsarClientException {
    if (!conf.isRetryEnable()) {
        throw new PulsarClientException("reconsumeLater method not support!");
    }
    try {
        reconsumeLaterAsync(message, delayTime, unit).get();
    } catch (Exception e) {
        Throwable t = e.getCause();
        if (t instanceof PulsarClientException) {
            throw (PulsarClientException) t;
        } else {
            throw new PulsarClientException(t);
        }
    }
}
 

public ConnectionPool(ClientConfigurationData conf, EventLoopGroup eventLoopGroup,
        Supplier<ClientCnx> clientCnxSupplier) throws PulsarClientException {
    this.eventLoopGroup = eventLoopGroup;
    this.clientConfig = conf;
    this.maxConnectionsPerHosts = conf.getConnectionsPerBroker();

    pool = new ConcurrentHashMap<>();
    bootstrap = new Bootstrap();
    bootstrap.group(eventLoopGroup);
    bootstrap.channel(EventLoopUtil.getClientSocketChannelClass(eventLoopGroup));

    bootstrap.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, conf.getConnectionTimeoutMs());
    bootstrap.option(ChannelOption.TCP_NODELAY, conf.isUseTcpNoDelay());
    bootstrap.option(ChannelOption.ALLOCATOR, PulsarByteBufAllocator.DEFAULT);

    try {
        channelInitializerHandler = new PulsarChannelInitializer(conf, clientCnxSupplier);
        bootstrap.handler(channelInitializerHandler);
    } catch (Exception e) {
        log.error("Failed to create channel initializer");
        throw new PulsarClientException(e);
    }

    this.dnsResolver = new DnsNameResolverBuilder(eventLoopGroup.next()).traceEnabled(true)
            .channelType(EventLoopUtil.getDatagramChannelClass(eventLoopGroup)).build();
}
 

/**
 * Receive the messages as a batch. The {@link BatchReceivePolicy} is
 * configured while creating the Pulsar {@link Consumer}.
 * In this method, emitted items are created by applying the projection function
 * to the messages received from Pulsar client. If there is an event time
 * associated with the message, it sets the event time as the timestamp of the
 * emitted item. Otherwise, it sets the publish time(which always exists)
 * of the message as the timestamp.
 */
private void fillBuffer(SourceBuilder.TimestampedSourceBuffer<T> sourceBuffer) throws PulsarClientException {
    Messages<M> messages = consumer.batchReceive();
    for (Message<M> message : messages) {
        if (message.getEventTime() != 0) {
            sourceBuffer.add(projectionFn.apply(message), message.getEventTime());
        } else {
            sourceBuffer.add(projectionFn.apply(message), message.getPublishTime());
        }
    }
    consumer.acknowledgeAsync(messages)
            .exceptionally(t -> {
                logger.warning(buildLogMessage(messages));
                return null;
            });
}
 

@Test
public void testAutoSubscriptionCreationDisable() throws Exception {
    pulsar.getConfiguration().setAllowAutoSubscriptionCreation(false);

    final String topicName = "persistent://prop/ns-abc/test-subtopic";
    final String subscriptionName = "test-subtopic-sub";

    admin.topics().createNonPartitionedTopic(topicName);

    try {
        pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).subscribe();
        fail("Subscribe operation should have failed");
    } catch (Exception e) {
        assertTrue(e instanceof PulsarClientException);
    }
    assertFalse(admin.topics().getSubscriptions(topicName).contains(subscriptionName));
}
 

private void failPendingReceive() {
    lock.readLock().lock();
    try {
        if (listenerExecutor != null && !listenerExecutor.isShutdown()) {
            while (!pendingReceives.isEmpty()) {
                CompletableFuture<Message<T>> receiveFuture = pendingReceives.poll();
                if (receiveFuture != null) {
                    receiveFuture.completeExceptionally(
                            new PulsarClientException.AlreadyClosedException("Consumer is already closed"));
                } else {
                    break;
                }
            }
        }
    } finally {
        lock.readLock().unlock();
    }
}
 

private void batchMessageAndSend() {
    if (log.isTraceEnabled()) {
        log.trace("[{}] [{}] Batching the messages from the batch container with {} messages", topic, producerName,
                batchMessageContainer.getNumMessagesInBatch());
    }
    if (!batchMessageContainer.isEmpty()) {
        try {
            List<OpSendMsg> opSendMsgs;
            if (batchMessageContainer.isMultiBatches()) {
                opSendMsgs = batchMessageContainer.createOpSendMsgs();
            } else {
                opSendMsgs = Collections.singletonList(batchMessageContainer.createOpSendMsg());
            }
            batchMessageContainer.clear();
            for (OpSendMsg opSendMsg : opSendMsgs) {
                processOpSendMsg(opSendMsg);
            }
        } catch (PulsarClientException e) {
            Thread.currentThread().interrupt();
            semaphore.release(batchMessageContainer.getNumMessagesInBatch());
        } catch (Throwable t) {
            semaphore.release(batchMessageContainer.getNumMessagesInBatch());
            log.warn("[{}] [{}] error while create opSendMsg by batch message container", topic, producerName, t);
        }
    }
}
 

@Override
public Void process(String input, Context context) {
    String publishTopic = (String) context.getUserConfigValueOrDefault("publish-topic", "publishtopic");
    String output = String.format("%s!", input);

    Map<String, String> properties = new HashMap<>();
    properties.put("input_topic", context.getCurrentRecord().getTopicName().get());
    properties.putAll(context.getCurrentRecord().getProperties());

    try {
        TypedMessageBuilder messageBuilder = context.newOutputMessage(publishTopic, Schema.STRING).
                value(output).properties(properties);
        if (context.getCurrentRecord().getKey().isPresent()){
            messageBuilder.key(context.getCurrentRecord().getKey().get());
        }
        messageBuilder.eventTime(System.currentTimeMillis()).sendAsync();
    } catch (PulsarClientException e) {
        context.getLogger().error(e.toString());
    }
    return null;
}
 

@Test
public void testPulsarSink() throws PulsarClientException {
    String topicName = randomName();
    Sink<Integer> pulsarSink = setupSink(topicName); // Its projection function -> Integer::doubleValue

    Pipeline p = Pipeline.create();
    List<Integer> numbers = IntStream.range(0, ITEM_COUNT).boxed().collect(Collectors.toList());
    p.readFrom(TestSources.items(numbers))
     .writeTo(pulsarSink);

    createJetMember().newJob(p).join();
    List<Double> list = consumeMessages(topicName, ITEM_COUNT);

    assertTrueEventually(() -> {
        Assert.assertEquals(ITEM_COUNT, list.size());
        for (double i = 0; i < ITEM_COUNT; i++) {
            assertTrue("missing entry: " + i, list.contains(i));
        }
    }, 10);
}
 

@SuppressWarnings({ "rawtypes" })
@Override
public void prepare(Map conf, TopologyContext context, OutputCollector collector) {
    this.componentId = context.getThisComponentId();
    this.boltId = String.format("%s-%s", componentId, context.getThisTaskId());
    this.collector = collector;
    try {
        sharedPulsarClient = SharedPulsarClient.get(componentId, clientConf);
        producer = sharedPulsarClient.getSharedProducer(producerConf);
        LOG.info("[{}] Created a pulsar producer on topic {} to send messages", boltId, pulsarBoltConf.getTopic());
    } catch (PulsarClientException e) {
        LOG.error("[{}] Error initializing pulsar producer on topic {}", boltId, pulsarBoltConf.getTopic(), e);
        throw new IllegalStateException(
                format("Failed to initialize producer for %s : %s", pulsarBoltConf.getTopic(), e.getMessage()), e);
    }
    context.registerMetric(String.format("PulsarBoltMetrics-%s-%s", componentId, context.getThisTaskIndex()), this,
            pulsarBoltConf.getMetricsTimeIntervalInSecs());
}
 

@Test(expected = StageException.class)
public void testTakeStageExceptionPulsarMessageConverterConvert() throws StageException {
  createPulsarMessageConsumerImplNoIssues();

  List<Stage.ConfigIssue> issues = pulsarMessageConsumerImplMock.init(contextMock);
  Assert.assertEquals(0, issues.size());

  try {
    Mockito.when(messageConsumerMock.receive(Mockito.anyInt(), Mockito.any()))
           .thenReturn(TestUtilsPulsar.getPulsarMessage());
    Mockito.when(pulsarMessageConverterMock.convert(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()))
           .thenThrow(new StageException(PulsarErrors.PULSAR_09, "messageId", "param1", "param2"));
  } catch (PulsarClientException e) {
    Assert.fail("Error mocking Consumer.receive method in testTakeStageExceptionPulsarMessageConverterConvert");
  }
  pulsarMessageConsumerImplMock.take(Mockito.mock(BatchMaker.class), contextMock, 1);
  Assert.fail();
}
 

public <K, V> Map<String, List<PulsarKafkaStream<K, V>>> createMessageStreamsByFilter(TopicFilter topicFilter,
        Map<String, Integer> topicCountMap, Decoder<K> keyDecoder, Decoder<V> valueDecoder) {

    Map<String, List<PulsarKafkaStream<K, V>>> streams = Maps.newHashMap();

    topicCountMap.forEach((topic, count) -> {
        try {
            Consumer<byte[]> consumer = consumerBuilder.topic(topic).subscribe();
            resetOffsets(consumer, strategy);
            log.info("Creating stream for {}-{} with config {}", topic, groupId, consumerBuilder.toString());
            for (int i = 0; i < count; i++) {
                PulsarKafkaStream<K, V> stream = new PulsarKafkaStream<>(keyDecoder, valueDecoder, consumer,
                        isAutoCommit, clientId);
                // if multiple thread-count present then client expects multiple streams reading from the same
                // topic. so, create multiple stream using the same consumer
                streams.computeIfAbsent(topic, key -> Lists.newArrayList()).add(stream);
                topicStreams.add(stream);
            }
        } catch (PulsarClientException e) {
            log.error("Failed to subscribe on topic {} with group-id {}, {}", topic, groupId, e.getMessage(), e);
            throw new RuntimeException("Failed to subscribe on topic " + topic, e);
        }
    });
    return streams;
}
 

private org.apache.pulsar.client.api.Producer<byte[]> createNewProducer(String topic) {
    try {
        pulsarProducerBuilder.messageRoutingMode(MessageRoutingMode.CustomPartition);
        pulsarProducerBuilder.messageRouter(new MessageRouter() {
            private static final long serialVersionUID = 1L;

            @Override
            public int choosePartition(Message<?> msg, TopicMetadata metadata) {
                // https://kafka.apache.org/08/documentation.html#producerapi
                // The default partitioner is based on the hash of the key.
                return partitioner.partition(msg.getKey(), metadata.numPartitions());
            }
        });
        log.info("Creating producer for topic {} with config {}", topic, pulsarProducerBuilder.toString());
        return pulsarProducerBuilder.clone().topic(topic).create();
    } catch (PulsarClientException e) {
        throw new RuntimeException(e);
    }
}
 

@Test
public void testClientStart() throws PulsarClientException, TransactionCoordinatorClientException, InterruptedException {
    try {
        transactionCoordinatorClient.start();
        Assert.fail("should failed here because the transaction metas store already started!");
    } catch (TransactionCoordinatorClientException e) {
        // ok here
    }

    Assert.assertNotNull(transactionCoordinatorClient);
    Assert.assertEquals(transactionCoordinatorClient.getState(), State.READY);
}
 

public PulsarLogger(Map<String, Object> props) throws PulsarClientException {
    
    String brokerHost = (String) props.get(PULSAR_SERVICE_URL_PROP_NAME);
    Integer brokerPort = (Integer) props.get(PULSAR_SERVICE_URL_PORT_PROP_NAME);

    pulsarURL.append(brokerHost);
    pulsarURL.append(":");
    pulsarURL.append(brokerPort);
    
    this.topic = (String) props.get("topic");
    
    if (props.get("sync") != null) {
        this.sync = (Boolean) props.get("sync");
    } else {
        this.sync = false;
    }
    
    if (props.get("flatten") != null) {
        this.flatten = (Boolean) props.get("flatten");
    } else {
        this.flatten = false;
    }
    
    this.pulsarClient = new PulsarClientImpl(pulsarURL.toString(), new ClientConfiguration());
    this.producer = pulsarClient.createProducer(topic);
    
    this.jsonUtils = new JsonUtils();
    
}
 

public static void main(String[] args) throws PulsarClientException, InterruptedException, IOException {
    PulsarClient client = PulsarClient.builder().serviceUrl("pulsar://localhost:6650").build();

    Producer<byte[]> producer = client.newProducer().topic("persistent://my-tenant/my-ns/my-topic").create();

    for (int i = 0; i < 10; i++) {
        producer.send("my-message".getBytes());
    }

    client.close();
}
 

@Test
public void testZeroQueueSizeMessageRedeliveryForAsyncReceive() throws PulsarClientException, ExecutionException, InterruptedException {
    final String topic = "persistent://prop/ns-abc/testZeroQueueSizeMessageRedeliveryForAsyncReceive";
    Consumer<Integer> consumer = pulsarClient.newConsumer(Schema.INT32)
        .topic(topic)
        .receiverQueueSize(0)
        .subscriptionName("sub")
        .subscriptionType(SubscriptionType.Shared)
        .ackTimeout(1, TimeUnit.SECONDS)
        .subscribe();

    final int messages = 10;
    Producer<Integer> producer = pulsarClient.newProducer(Schema.INT32)
        .topic(topic)
        .enableBatching(false)
        .create();

    for (int i = 0; i < messages; i++) {
        producer.send(i);
    }

    Set<Integer> receivedMessages = new HashSet<>();
    List<CompletableFuture<Message<Integer>>> futures = new ArrayList<>(20);
    for (int i = 0; i < messages * 2; i++) {
        futures.add(consumer.receiveAsync());
    }
    for (CompletableFuture<Message<Integer>> future : futures) {
        receivedMessages.add(future.get().getValue());
    }

    Assert.assertEquals(receivedMessages.size(), messages);

    consumer.close();
    producer.close();
}
 

@SuppressWarnings("unchecked")
protected <T> CompletableFuture<Schema<T>> preProcessSchemaBeforeSubscribe(PulsarClientImpl pulsarClientImpl,
                                                                  Schema<T> schema,
                                                                  String topicName) {
    if (schema != null && schema.supportSchemaVersioning()) {
        final SchemaInfoProvider schemaInfoProvider;
        try {
            schemaInfoProvider = pulsarClientImpl.getSchemaProviderLoadingCache().get(topicName);
        } catch (ExecutionException e) {
            log.error("Failed to load schema info provider for topic {}", topicName, e);
            return FutureUtil.failedFuture(e.getCause());
        }
        schema = schema.clone();
        if (schema.requireFetchingSchemaInfo()) {
            Schema finalSchema = schema;
            return schemaInfoProvider.getLatestSchema().thenCompose(schemaInfo -> {
                if (null == schemaInfo) {
                    if (!(finalSchema instanceof AutoConsumeSchema)) {
                        // no schema info is found
                        return FutureUtil.failedFuture(
                                new PulsarClientException.NotFoundException(
                                        "No latest schema found for topic " + topicName));
                    }
                }
                try {
                    log.info("Configuring schema for topic {} : {}", topicName, schemaInfo);
                    finalSchema.configureSchemaInfo(topicName, "topic", schemaInfo);
                } catch (RuntimeException re) {
                    return FutureUtil.failedFuture(re);
                }
                finalSchema.setSchemaInfoProvider(schemaInfoProvider);
                return CompletableFuture.completedFuture(finalSchema);
            });
        } else {
            schema.setSchemaInfoProvider(schemaInfoProvider);
        }
    }
    return CompletableFuture.completedFuture(schema);
}
 

@Override
public void seek(MessageId messageId) throws PulsarClientException {
    try {
        seekAsync(messageId).get();
    } catch (Exception e) {
        throw PulsarClientException.unwrap(e);
    }
}
 

@Test
public void testSeekOnPartitionedTopic() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/testSeekPartitions";

    admin.topics().createPartitionedTopic(topicName, 2);
    org.apache.pulsar.client.api.Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName)
            .subscriptionName("my-subscription").subscribe();

    try {
        consumer.seek(MessageId.latest);
        fail("Should not have succeeded");
    } catch (PulsarClientException e) {
        // Expected
    }
}
 

@Override
public void acknowledge(Message<?> message) throws PulsarClientException {
    try {
        acknowledge(message.getMessageId());
    } catch (NullPointerException npe) {
        throw new PulsarClientException.InvalidMessageException(npe.getMessage());
    }
}
 

@Override
protected Message<T> internalReceive() throws PulsarClientException {
    Message<T> message;
    try {
        message = incomingMessages.take();
        messageProcessed(message);
        return beforeConsume(message);
    } catch (InterruptedException e) {
        stats.incrementNumReceiveFailed();
        throw PulsarClientException.unwrap(e);
    }
}
 

public void close() throws PulsarClientException {
    if (counter.decrementAndGet() <= 0) {
        if (client != null) {
            client.close();
            instances.remove(componentId);
            LOG.info("[{}] Closed Pulsar Client", componentId);
        }
    }
}
 

public static void main(String[] args) throws IOException {
    // Create a Pulsar client instance. A single instance can be shared across many
    // producers and consumer within the same application
    PulsarClient client = PulsarClient.builder()
            .serviceUrl(SERVICE_URL)
            .build();

    // Here you get the chance to configure producer specific settings
    Producer<byte[]> producer = client.newProducer()
            // Set the topic
            .topic(TOPIC_NAME)
            // Enable compression
            .compressionType(CompressionType.LZ4)
            .create();

    // Once the producer is created, it can be used for the entire application life-cycle
    log.info("Created producer for the topic {}", TOPIC_NAME);

    // Send 10 test messages
    IntStream.range(1, 11).forEach(i -> {
        String content = String.format("hello-pulsar-%d", i);

        // Build a message object
        Message<byte[]> msg = MessageBuilder.create()
                .setContent(content.getBytes())
                .build();

        // Send each message and log message content and ID when successfully received
        try {
            MessageId msgId = producer.send(msg);

            log.info("Published message '{}' with the ID {}", content, msgId);
        } catch (PulsarClientException e) {
            log.error(e.getMessage());
        }
    });

    client.close();
}
 

public static Reader<byte[]> createReader(ReaderBuilder readerBuilder,
                                          String readerName,
                                          String topic,
                                          MessageId startMessageId) throws PulsarClientException {
    return readerBuilder
            .subscriptionRolePrefix(readerName)
            .readerName(readerName)
            .topic(topic)
            .readCompacted(true)
            .startMessageId(startMessageId)
            .create();
}
 

@Test
public void testConfigBuilder() throws PulsarClientException {
    ClientConfigurationData clientConfig = new ClientConfigurationData();
    clientConfig.setServiceUrl("pulsar://unknown:6650");
    clientConfig.setStatsIntervalSeconds(80);

    PulsarClientImpl pulsarClient = new PulsarClientImpl(clientConfig);
    assertNotNull(pulsarClient, "Pulsar client built using config should not be null");

    assertEquals(pulsarClient.getConfiguration().getServiceUrl(), "pulsar://unknown:6650");
    assertEquals(pulsarClient.getConfiguration().getNumListenerThreads(), 1, "builder default not set properly");
    assertEquals(pulsarClient.getConfiguration().getStatsIntervalSeconds(), 80,
            "builder default should overrite if set explicitly");
}
 

public void close() {
    try {
        client.close();
    } catch (PulsarClientException e) {
        log.warn("Failed to close client", e);
    }
}
 

@Override
public CompletableFuture<Void> unsubscribeAsync() {
    if (getState() == State.Closing || getState() == State.Closed) {
        return FutureUtil
                .failedFuture(new PulsarClientException.AlreadyClosedException("Consumer was already closed"));
    }
    final CompletableFuture<Void> unsubscribeFuture = new CompletableFuture<>();
    if (isConnected()) {
        setState(State.Closing);
        long requestId = client.newRequestId();
        ByteBuf unsubscribe = Commands.newUnsubscribe(consumerId, requestId);
        ClientCnx cnx = cnx();
        cnx.sendRequestWithId(unsubscribe, requestId).thenRun(() -> {
            cnx.removeConsumer(consumerId);
            unAckedMessageTracker.close();
            if (possibleSendToDeadLetterTopicMessages != null) {
                possibleSendToDeadLetterTopicMessages.clear();
            }
            client.cleanupConsumer(ConsumerImpl.this);
            log.info("[{}][{}] Successfully unsubscribed from topic", topic, subscription);
            setState(State.Closed);
            unsubscribeFuture.complete(null);
        }).exceptionally(e -> {
            log.error("[{}][{}] Failed to unsubscribe: {}", topic, subscription, e.getCause().getMessage());
            setState(State.Ready);
            unsubscribeFuture.completeExceptionally(
                PulsarClientException.wrap(e.getCause(),
                    String.format("Failed to unsubscribe the subscription %s of topic %s",
                        topicName.toString(), subscription)));
            return null;
        });
    } else {
        unsubscribeFuture.completeExceptionally(
            new PulsarClientException(
                String.format("The client is not connected to the broker when unsubscribing the " +
                    "subscription %s of the topic %s", subscription, topicName.toString())));
    }
    return unsubscribeFuture;
}
 

@Override
public AuthenticationDataProvider getAuthData() throws PulsarClientException {
    try {
        return new AuthenticationDataBasic(userId, password);
    } catch (Exception e) {
        throw PulsarClientException.unwrap(e);
    }
}
 

@PreDestroy
public void destroy() {
	if (client != null) {
		try {
			client.close();
		} catch (PulsarClientException e) {
			log.warn("couldn't close Pulsar client", e);
		} finally {
			client = null;
			blockEventProducer = null;
			contractEventProducer = null;
		}
	}
}