Java Code Examples for org.apache.pulsar.client.api.Consumer#acknowledgeCumulative()

/**
 * <pre>
 * Overriden method: OffsetCommitResponse commitOffsets(OffsetCommitRequest request)
 * 
 * Note:
 * created PulsarOffsetCommitResponse as OffsetCommitRequest doesn't provide getters
 * 
 * </pre>
 */
public OffsetCommitResponse commitOffsets(PulsarOffsetCommitRequest request) {

    PulsarOffsetCommitResponse response = new PulsarOffsetCommitResponse(null);
    for (Entry<String, MessageId> topicOffset : request.getTopicOffsetMap().entrySet()) {
        final String topic = topicOffset.getKey();
        final String groupId = request.getGroupId();
        try {
            Consumer<byte[]> consumer = getConsumer(topic, groupId);
            consumer.acknowledgeCumulative(topicOffset.getValue());
        } catch (Exception e) {
            log.warn("Failed to ack message for topic {}-{}", topic, topicOffset.getValue(), e);
            response.hasError = true;
            TopicAndPartition topicPartition = new TopicAndPartition(topic, 0);
            response.errors.computeIfAbsent(topicPartition, tp -> ErrorMapping.UnknownCode());
        }
    }

    return response;
}
 

public static void main(String[] args) throws PulsarClientException, InterruptedException {

        PulsarClient pulsarClient = PulsarClient.builder().serviceUrl("pulsar://localhost:6650").build();

        Consumer<byte[]> consumer = pulsarClient.newConsumer() //
                .topic("persistent://my-tenant/my-ns/my-topic") //
                .subscriptionName("my-subscription-name").subscribe();

        Message<byte[]> msg = null;

        for (int i = 0; i < 100; i++) {
            msg = consumer.receive();
            // do something
            System.out.println("Received: " + new String(msg.getData()));
        }

        // Acknowledge the consumption of all messages at once
        try {
            consumer.acknowledgeCumulative(msg);
        } catch (Exception e) {
            consumer.reconsumeLater(msg, 10, TimeUnit.SECONDS);
        }
       
        pulsarClient.close();
    }
 

public static void main(String[] args) throws PulsarClientException, JsonProcessingException {

        PulsarClient pulsarClient = PulsarClient.builder().serviceUrl("http://localhost:8080").build();

        Consumer<JsonPojo> consumer = pulsarClient.newConsumer(JSONSchema.of
                (SchemaDefinition.<JsonPojo>builder().withPojo(JsonPojo.class).build())) //
                .topic("persistent://my-property/use/my-ns/my-topic") //
                .subscriptionName("my-subscription-name").subscribe();

        Message<JsonPojo> msg = null;

        for (int i = 0; i < 100; i++) {
            msg = consumer.receive();
            // do something
            System.out.println("Received: " + msg.getValue().content);
        }

        // Acknowledge the consumption of all messages at once
        consumer.acknowledgeCumulative(msg);
        pulsarClient.close();
    }
 

@Test
public void testBacklogSizeShouldBeZeroWhenConsumerAckedAllMessages() throws Exception {
    final String topic = "persistent://prop-xyz/ns1/testBacklogSizeShouldBeZeroWhenConsumerAckedAllMessages";
    Consumer<byte[]> consumer = pulsarClient.newConsumer()
            .topic(topic)
            .subscriptionName("sub-1")
            .subscribe();
    Producer<byte[]> producer = pulsarClient.newProducer()
            .topic(topic)
            .create();

    final int messages = 33;
    for (int i = 0; i < messages; i++) {
        producer.send(new byte[1024 * i * 5]);
    }

    for (int i = 0; i < messages; i++) {
        consumer.acknowledgeCumulative(consumer.receive());
    }

    // Wait ack send
    Thread.sleep(1000);

    TopicStats topicStats = admin.topics().getStats(topic);
    assertEquals(topicStats.backlogSize, 0);
}
 

@Test(dataProvider = "batch")
public void testSyncProducerAndConsumer(int batchMessageDelayMs) throws Exception {
    log.info("-- Starting {} test --", methodName);

    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING)
                .topic("persistent://my-property/use/my-ns/my-topic1")
                .subscriptionName("my-subscriber-name")
                .subscriptionType(SubscriptionType.Exclusive)
                .subscribe();

    ProducerBuilder<String> producerBuilder = pulsarClient.newProducer(Schema.STRING)
            .topic("persistent://my-property/use/my-ns/my-topic1");

    if (batchMessageDelayMs != 0) {
        producerBuilder.enableBatching(true)
            .batchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS)
            .batchingMaxMessages(5);
    } else {
        producerBuilder.enableBatching(false);
    }

    Producer<String> producer = producerBuilder.create();
    for (int i = 0; i < 10; i++) {
        producer.send("my-message-" + i);
    }

    Message<String> msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = msg.getValue();
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 

/**
 * verifies that messages whose size is larger than 2^14 bytes (max size of single TLS chunk) can be
 * produced/consumed
 *
 * @throws Exception
 */
@Test(timeOut = 30000)
public void testTlsLargeSizeMessage() throws Exception {
    log.info("-- Starting {} test --", methodName);

    final int MESSAGE_SIZE = 16 * 1024 + 1;
    log.info("-- message size --", MESSAGE_SIZE);

    internalSetUpForClient(true, pulsar.getBrokerServiceUrlTls());
    internalSetUpForNamespace();

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://my-property/use/my-ns/my-topic1")
            .subscriptionName("my-subscriber-name").subscribe();

    Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1")
            .create();
    for (int i = 0; i < 10; i++) {
        byte[] message = new byte[MESSAGE_SIZE];
        Arrays.fill(message, (byte) i);
        producer.send(message);
    }

    Message<byte[]> msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        byte[] expected = new byte[MESSAGE_SIZE];
        Arrays.fill(expected, (byte) i);
        Assert.assertEquals(expected, msg.getData());
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 

/**
 * verifies that messages whose size is larger than 2^14 bytes (max size of single TLS chunk) can be
 * produced/consumed
 *
 * @throws Exception
 */
@Test(timeOut = 30000)
public void testTlsLargeSizeMessage() throws Exception {
    log.info("-- Starting {} test --", methodName);

    final int MESSAGE_SIZE = 16 * 1024 + 1;
    log.info("-- message size --", MESSAGE_SIZE);

    internalSetUpForClient(true, pulsar.getBrokerServiceUrlTls());
    internalSetUpForNamespace();

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://my-property/use/my-ns/my-topic1")
            .subscriptionName("my-subscriber-name").subscribe();

    Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1")
            .create();
    for (int i = 0; i < 10; i++) {
        byte[] message = new byte[MESSAGE_SIZE];
        Arrays.fill(message, (byte) i);
        producer.send(message);
    }

    Message<byte[]> msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        byte[] expected = new byte[MESSAGE_SIZE];
        Arrays.fill(expected, (byte) i);
        Assert.assertEquals(expected, msg.getData());
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 

/**
 * verifies that messages whose size is larger than 2^14 bytes (max size of single TLS chunk) can be
 * produced/consumed
 *
 * @throws Exception
 */
@Test(timeOut = 30000)
public void testTlsLargeSizeMessage() throws Exception {
    log.info("-- Starting {} test --", methodName);

    final int MESSAGE_SIZE = 16 * 1024 + 1;
    log.info("-- message size --", MESSAGE_SIZE);
    String topicName = "persistent://my-property/use/my-ns/testTlsLargeSizeMessage"
                       + System.currentTimeMillis();

    internalSetUpForClient(true, pulsar.getBrokerServiceUrlTls());
    internalSetUpForNamespace();

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName)
            .subscriptionName("my-subscriber-name").subscribe();

    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
            .create();
    for (int i = 0; i < 10; i++) {
        byte[] message = new byte[MESSAGE_SIZE];
        Arrays.fill(message, (byte) i);
        producer.send(message);
    }

    Message<byte[]> msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        byte[] expected = new byte[MESSAGE_SIZE];
        Arrays.fill(expected, (byte) i);
        Assert.assertEquals(expected, msg.getData());
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 

/**
 * verifies that messages whose size is larger than 2^14 bytes (max size of single TLS chunk) can be
 * produced/consumed
 *
 * @throws Exception
 */
@Test(timeOut = 30000)
public void testTlsLargeSizeMessage() throws Exception {
    log.info("-- Starting {} test --", methodName);

    final int MESSAGE_SIZE = 16 * 1024 + 1;
    log.info("-- message size --", MESSAGE_SIZE);
    String topicName = "persistent://my-property/use/my-ns/testTlsLargeSizeMessage"
                       + System.currentTimeMillis();

    internalSetUpForClient(true, pulsar.getBrokerServiceUrlTls());
    internalSetUpForNamespace();

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName)
            .subscriptionName("my-subscriber-name").subscribe();

    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
            .create();
    for (int i = 0; i < 10; i++) {
        byte[] message = new byte[MESSAGE_SIZE];
        Arrays.fill(message, (byte) i);
        producer.send(message);
    }

    Message<byte[]> msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        byte[] expected = new byte[MESSAGE_SIZE];
        Arrays.fill(expected, (byte) i);
        Assert.assertEquals(expected, msg.getData());
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 

@Test
public void testEnabledChecksumClient() throws Exception {
    log.info("-- Starting {} test --", methodName);

    final int totalMsg = 10;

    Consumer<byte[]> consumer = pulsarClient.newConsumer()
            .topic("persistent://my-property/use/my-ns/my-topic1")
            .subscriptionName("my-subscriber-name")
            .subscribe();

    final int batchMessageDelayMs = 300;
    Producer<byte[]> producer = pulsarClient.newProducer()
            .topic("persistent://my-property/use/my-ns/my-topic1")
            .enableBatching(true)
            .batchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS)
            .batchingMaxMessages(5)
            .create();

    for (int i = 0; i < totalMsg; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }

    Message<byte[]>msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < totalMsg; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 

@Test
public void testMultiHostUrlRetrySuccess() throws Exception {
    log.info("-- Starting {} test --", methodName);

    final String topicName = "persistent://my-property/my-ns/my-topic1";
    final String subscriptionName = "my-subscriber-name";

    // Multi hosts included an unreached port and the actual port for verify retry logic
    String urlsWithUnreached = "http://localhost:51000,localhost:" + new URI(pulsar.getWebServiceAddress()).getPort();
    if (isTcpLookup) {
        urlsWithUnreached = "pulsar://localhost:51000,localhost" + new URI(pulsar.getBrokerServiceUrl()).getPort();
    }
    PulsarClient client = newPulsarClient(urlsWithUnreached, 0);

    Consumer<byte[]> consumer = client.newConsumer().topic(topicName).subscriptionName(subscriptionName)
        .acknowledgmentGroupTime(0, TimeUnit.SECONDS).subscribe();
    Producer<byte[]> producer = client.newProducer().topic(topicName).create();

    for (int i = 0; i < 5; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        log.info("Produced message: [{}]", message);
    }

    Message<byte[]> msg = null;
    Set<String> messageSet = new HashSet();
    for (int i = 0; i < 5; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.info("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }

    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();

    producer.close();
    client.close();

    log.info("-- Exiting {} test --", methodName);
}
 

@Test
public void testBatchMessageIndexAckForExclusiveSubscription() throws PulsarClientException, ExecutionException, InterruptedException {
    final String topic = "testBatchMessageIndexAckForExclusiveSubscription";

    @Cleanup
    Consumer<Integer> consumer = pulsarClient.newConsumer(Schema.INT32)
        .topic(topic)
        .subscriptionName("sub")
        .receiverQueueSize(100)
        .subscribe();

    @Cleanup
    Producer<Integer> producer = pulsarClient.newProducer(Schema.INT32)
        .topic(topic)
        .batchingMaxPublishDelay(50, TimeUnit.MILLISECONDS)
        .create();

    final int messages = 100;
    List<CompletableFuture<MessageId>> futures = new ArrayList<>(messages);
    for (int i = 0; i < messages; i++) {
        futures.add(producer.sendAsync(i));
    }
    FutureUtil.waitForAll(futures).get();

    for (int i = 0; i < messages; i++) {
        if (i == 49) {
            consumer.acknowledgeCumulative(consumer.receive());
        } else {
            consumer.receive();
        }
    }

    //Wait ack send.
    Thread.sleep(1000);
    consumer.close();
    consumer = pulsarClient.newConsumer(Schema.INT32)
        .topic(topic)
        .subscriptionName("sub")
        .receiverQueueSize(100)
        .subscribe();

    List<Message<Integer>> received = new ArrayList<>(50);
    for (int i = 0; i < 50; i++) {
        received.add(consumer.receive());
    }

    Assert.assertEquals(received.size(), 50);

    Message<Integer> moreMessage = consumer.receive(1, TimeUnit.SECONDS);
    Assert.assertNull(moreMessage);

    futures.clear();
    for (int i = 0; i < 50; i++) {
        futures.add(producer.sendAsync(i));
    }
    FutureUtil.waitForAll(futures).get();

    for (int i = 0; i < 50; i++) {
        received.add(consumer.receive());
    }

    // Ensure the flow permit is work well since the client skip the acked batch index,
    // broker also need to handle the available permits.
    Assert.assertEquals(received.size(), 100);
}
 

@Test
public void testBatchMessageIndexAckForExclusiveSubscription() throws PulsarClientException, ExecutionException, InterruptedException {
    final String topic = "testBatchMessageIndexAckForExclusiveSubscription";

    @Cleanup
    Consumer<Integer> consumer = pulsarClient.newConsumer(Schema.INT32)
        .topic(topic)
        .subscriptionName("sub")
        .receiverQueueSize(100)
        .subscribe();

    @Cleanup
    Producer<Integer> producer = pulsarClient.newProducer(Schema.INT32)
        .topic(topic)
        .batchingMaxPublishDelay(50, TimeUnit.MILLISECONDS)
        .create();

    final int messages = 100;
    List<CompletableFuture<MessageId>> futures = new ArrayList<>(messages);
    for (int i = 0; i < messages; i++) {
        futures.add(producer.sendAsync(i));
    }
    FutureUtil.waitForAll(futures).get();

    for (int i = 0; i < messages; i++) {
        if (i == 49) {
            consumer.acknowledgeCumulative(consumer.receive());
        } else {
            consumer.receive();
        }
    }

    //Wait ack send.
    Thread.sleep(1000);
    consumer.close();
    consumer = pulsarClient.newConsumer(Schema.INT32)
        .topic(topic)
        .subscriptionName("sub")
        .receiverQueueSize(100)
        .subscribe();

    List<Message<Integer>> received = new ArrayList<>(100);
    for (int i = 0; i < messages; i++) {
        received.add(consumer.receive());
    }

    Assert.assertEquals(received.size(), 100);
}
 

@Test
public void testSimpleConsumerEvents() throws Exception {
    final String topicName = "persistent://prop/ns-abc/topic1";
    final String subName = "sub1";
    final int numMsgs = 10;

    // 1. client connect
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();

    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();
    PersistentSubscription subRef = topicRef.getSubscription(subName);

    assertNotNull(topicRef);
    assertNotNull(subRef);
    assertTrue(subRef.getDispatcher().isConsumerConnected());

    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(getAvailablePermits(subRef), 1000 /* default */);

    Producer<byte[]> producer = pulsarClient.newProducer()
        .topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();
    for (int i = 0; i < numMsgs * 2; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }

    assertTrue(subRef.getDispatcher().isConsumerConnected());
    rolloverPerIntervalStats();
    assertEquals(subRef.getNumberOfEntriesInBacklog(false), numMsgs * 2);

    // 2. messages pushed before client receive
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(getAvailablePermits(subRef), 1000 - numMsgs * 2);

    Message<byte[]> msg = null;
    for (int i = 0; i < numMsgs; i++) {
        msg = consumer.receive();
        // 3. in-order message delivery
        assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer.acknowledge(msg);
    }

    rolloverPerIntervalStats();

    // 4. messages deleted on individual acks
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(subRef.getNumberOfEntriesInBacklog(false), numMsgs);

    for (int i = 0; i < numMsgs; i++) {
        msg = consumer.receive();
        if (i == numMsgs - 1) {
            consumer.acknowledgeCumulative(msg);
        }
    }

    rolloverPerIntervalStats();

    // 5. messages deleted on cumulative acks
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(subRef.getNumberOfEntriesInBacklog(false), 0);

    // 6. consumer unsubscribe
    consumer.unsubscribe();

    // 6. consumer graceful close
    consumer.close();

    // 7. consumer unsubscribe
    try {
        consumer.unsubscribe();
        fail("Should have failed");
    } catch (PulsarClientException.AlreadyClosedException e) {
        // ok
    }

    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    subRef = topicRef.getSubscription(subName);
    assertNull(subRef);

    producer.close();

    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
}
 

@Test(enabled = false)
// TODO: enable this after java client supports graceful close
public void testGracefulClose() throws Exception {
    final String topicName = "persistent://prop/ns-abc/topic4";
    final String subName = "sub4";

    Producer<byte[]> producer = pulsarClient.newProducer()
        .topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);

    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();
    assertNotNull(topicRef);

    ExecutorService executor = Executors.newCachedThreadPool();
    CountDownLatch latch = new CountDownLatch(1);
    executor.submit(() -> {
        for (int i = 0; i < 10; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
        }
        latch.countDown();
        return null;
    });

    producer.close();

    // 1. verify there are no pending publish acks once the producer close
    // is completed on client
    assertEquals(topicRef.getProducers().values().iterator().next().getPendingPublishAcks(), 0);

    // safety latch in case of failure,
    // wait for the spawned thread to complete
    latch.await();

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();

    PersistentSubscription subRef = topicRef.getSubscription(subName);
    assertNotNull(subRef);

    Message<byte[]> msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive();
    }

    // 2. verify consumer close fails when there are outstanding
    // message acks
    try {
        consumer.close();
        fail("should have failed");
    } catch (IllegalStateException e) {
        // Expected - messages not acked
    }

    consumer.acknowledgeCumulative(msg);
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);

    // 3. verify consumer close succeeds once all messages are ack'ed
    consumer.close();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertTrue(subRef.getDispatcher().isConsumerConnected());

    executor.shutdown();
}
 

@Test
void testAuthentication() throws Exception {
	log.info("-- Starting {} test --", methodName);

	// Step 1: Create Admin Client

	// create a client which connects to proxy and pass authData
	String topicName = "persistent://my-property/my-ns/my-topic1";

	ProxyConfiguration proxyConfig = new ProxyConfiguration();
	proxyConfig.setAuthenticationEnabled(true);
	proxyConfig.setServicePort(Optional.of(0));
	proxyConfig.setWebServicePort(Optional.of(0));
	proxyConfig.setBrokerServiceURL(pulsar.getBrokerServiceUrl());
	proxyConfig.setSaslJaasClientAllowedIds(".*" + localHostname + ".*");
	proxyConfig.setSaslJaasServerSectionName("PulsarProxy");

	// proxy connect to broker
	proxyConfig.setBrokerClientAuthenticationPlugin(AuthenticationSasl.class.getName());
	proxyConfig.setBrokerClientAuthenticationParameters(
		"{\"saslJaasClientSectionName\": " + "\"PulsarProxy\"," +
			"\"serverType\": " + "\"broker\"}");

	// proxy as a server, it will use sasl to authn
	Set<String> providers = new HashSet<>();
	providers.add(AuthenticationProviderSasl.class.getName());
	proxyConfig.setAuthenticationProviders(providers);

	proxyConfig.setForwardAuthorizationCredentials(true);
	AuthenticationService authenticationService = new AuthenticationService(
                       PulsarConfigurationLoader.convertFrom(proxyConfig));
	ProxyService proxyService = new ProxyService(proxyConfig, authenticationService);

	proxyService.start();
	final String proxyServiceUrl = "pulsar://localhost:" + proxyService.getListenPort().get();
	log.info("1 proxy service started {}", proxyService);

	// Step 3: Pass correct client params
	PulsarClient proxyClient = createProxyClient(proxyServiceUrl, 1);
	log.info("2 create proxy client {}, {}", proxyServiceUrl, proxyClient);

	Producer<byte[]> producer = proxyClient.newProducer(Schema.BYTES).topic(topicName).create();
	log.info("3 created producer.");

	Consumer<byte[]> consumer = proxyClient.newConsumer(Schema.BYTES).topic(topicName).subscriptionName("test-sub").subscribe();
	log.info("4 created consumer.");

	for (int i = 0; i < 10; i++) {
		String message = "my-message-" + i;
		producer.send(message.getBytes());
		log.info("Produced message: [{}]", message);
	}

	Message<byte[]> msg = null;
	Set<String> messageSet = Sets.newHashSet();
	for (int i = 0; i < 10; i++) {
		msg = consumer.receive(5, TimeUnit.SECONDS);
		String receivedMessage = new String(msg.getData());
		log.info("Received message: [{}]", receivedMessage);
		String expectedMessage = "my-message-" + i;
		testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
	}
	// Acknowledge the consumption of all messages at once
	consumer.acknowledgeCumulative(msg);
	consumer.close();

	proxyClient.close();
	proxyService.close();
}
 

/**
 * <pre>
 * It verifies e2e tls + Authentication + Authorization (client -> proxy -> broker>
 *
 * 1. client connects to proxy over tls and pass auth-data
 * 2. proxy authenticate client and retrieve client-role
 *    and send it to broker as originalPrincipal over tls
 * 3. client creates producer/consumer via proxy
 * 4. broker authorize producer/consumer create request using originalPrincipal
 *
 * </pre>
 *
 * @throws Exception
 */
@SuppressWarnings("deprecation")
@Test
public void testTlsSyncProducerAndConsumer() throws Exception {
    log.info("-- Starting {} test --", methodName);

    final String proxyServiceUrl = proxyService.getServiceUrlTls();
    Map<String, String> authParams = Maps.newHashMap();
    authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
    authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
    Authentication authTls = new AuthenticationTls();
    authTls.configure(authParams);
    // create a client which connects to proxy over tls and pass authData
    PulsarClient proxyClient = createPulsarClient(authTls, proxyServiceUrl);

    admin.clusters().createCluster(configClusterName, new ClusterData(brokerUrl.toString(), brokerUrlTls.toString(),
            pulsar.getBrokerServiceUrl(), pulsar.getBrokerServiceUrlTls()));
    admin.tenants().createTenant("my-property",
            new TenantInfo(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("test")));
    admin.namespaces().createNamespace("my-property/my-ns", Sets.newHashSet("test"));

    Consumer<byte[]> consumer = proxyClient.newConsumer().topic("persistent://my-property/my-ns/my-topic1")
            .subscriptionName("my-subscriber-name").subscribe();
    Producer<byte[]> producer = proxyClient.newProducer(Schema.BYTES).topic("persistent://my-property/my-ns/my-topic1")
            .create();
    final int msgs = 10;
    for (int i = 0; i < msgs; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }

    Message<byte[]> msg = null;
    Set<String> messageSet = Sets.newHashSet();
    int count = 0;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
        count++;
    }
    // Acknowledge the consumption of all messages at once
    Assert.assertEquals(msgs, count);
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 

/**
 * <pre>
 * It verifies e2e tls + Authentication + Authorization (client -> proxy -> broker>
 *
 * 1. client connects to proxy over tls and pass auth-data
 * 2. proxy authenticate client and retrieve client-role
 *    and send it to broker as originalPrincipal over tls
 * 3. client creates producer/consumer via proxy
 * 4. broker authorize producer/consumer create request using originalPrincipal
 *
 * </pre>
 *
 * @throws Exception
 */
@Test
public void testDiscoveryService() throws Exception {
    log.info("-- Starting {} test --", methodName);

    Map<String, String> authParams = Maps.newHashMap();
    authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
    authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
    Authentication authTls = new AuthenticationTls();
    authTls.configure(authParams);
    // create a client which connects to proxy over tls and pass authData
    PulsarClient proxyClient = createPulsarClient(authTls, proxyService.getServiceUrlTls());

    admin.clusters().createCluster("without-service-discovery", new ClusterData(brokerUrl.toString()));

    admin.tenants().createTenant("my-property", new TenantInfo(Sets.newHashSet("appid1", "appid2"),
            Sets.newHashSet("without-service-discovery")));
    admin.namespaces().createNamespace("my-property/without-service-discovery/my-ns");

    Consumer<byte[]> consumer = proxyClient.newConsumer()
            .topic("persistent://my-property/without-service-discovery/my-ns/my-topic1")
            .subscriptionName("my-subscriber-name").subscribe();
    Producer<byte[]> producer = proxyClient.newProducer(Schema.BYTES)
            .topic("persistent://my-property/without-service-discovery/my-ns/my-topic1").create();
    final int msgs = 10;
    for (int i = 0; i < msgs; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }

    Message<byte[]> msg = null;
    Set<String> messageSet = Sets.newHashSet();
    int count = 0;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
        count++;
    }
    // Acknowledge the consumption of all messages at once
    Assert.assertEquals(msgs, count);
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 

/**
 * <pre>
 * It verifies e2e tls + Authentication + Authorization (client -> proxy -> broker)
 *
 * 1. client connects to proxy over tls and pass auth-data
 * 2. proxy authenticate client and retrieve client-role
 *    and send it to broker as originalPrincipal over tls
 * 3. client creates producer/consumer via proxy
 * 4. broker authorize producer/consumer create request using originalPrincipal
 *
 * </pre>
 *
 * @throws Exception
 */
@Test
public void testProxyAuthorization() throws Exception {
    log.info("-- Starting {} test --", methodName);

    startProxy();
    createAdminClient();
    // create a client which connects to proxy over tls and pass authData
    PulsarClient proxyClient = createPulsarClient(proxyService.getServiceUrlTls(), PulsarClient.builder());

    String namespaceName = "my-property/proxy-authorization/my-ns";

    admin.clusters().createCluster("proxy-authorization", new ClusterData(brokerUrl.toString()));

    admin.tenants().createTenant("my-property",
            new TenantInfo(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("proxy-authorization")));
    admin.namespaces().createNamespace(namespaceName);

    admin.namespaces().grantPermissionOnNamespace(namespaceName, "Proxy",
            Sets.newHashSet(AuthAction.consume, AuthAction.produce));
    admin.namespaces().grantPermissionOnNamespace(namespaceName, "Client",
            Sets.newHashSet(AuthAction.consume, AuthAction.produce));

    Consumer<byte[]> consumer = proxyClient.newConsumer()
            .topic("persistent://my-property/proxy-authorization/my-ns/my-topic1")
            .subscriptionName("my-subscriber-name").subscribe();

    Producer<byte[]> producer = proxyClient.newProducer(Schema.BYTES)
            .topic("persistent://my-property/proxy-authorization/my-ns/my-topic1").create();
    final int msgs = 10;
    for (int i = 0; i < msgs; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }

    Message<byte[]> msg = null;
    Set<String> messageSet = Sets.newHashSet();
    int count = 0;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
        count++;
    }
    // Acknowledge the consumption of all messages at once
    Assert.assertEquals(msgs, count);
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 

@Test(dataProvider = "batch")
public void testBackoffAndReconnect(int batchMessageDelayMs) throws Exception {
    log.info("-- Starting {} test --", methodName);
    // Create consumer and producer
    Consumer<byte[]> consumer = pulsarClient.newConsumer()
            .topic("persistent://my-property/use/my-ns/my-topic4")
            .subscriptionName("my-subscriber-name")
            .subscriptionType(SubscriptionType.Exclusive)
            .startMessageIdInclusive()
            .subscribe();
    Producer<byte[]> producer = pulsarClient.newProducer()
            .topic("persistent://my-property/use/my-ns/my-topic4")
            .batchingMaxMessages(5)
            .batchingMaxPublishDelay(BATCHING_MAX_PUBLISH_DELAY_THRESHOLD, TimeUnit.MILLISECONDS)
            .enableBatching(batchMessageDelayMs != 0)
            .create();

    // Produce messages
    CompletableFuture<MessageId> lastFuture = null;
    for (int i = 0; i < 10; i++) {
        lastFuture = producer.sendAsync(("my-message-" + i).getBytes()).thenApply(msgId -> {
            log.info("Published message id: {}", msgId);
            return msgId;
        });
    }

    lastFuture.get();

    Message<byte[]> msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        log.info("Received: [{}]", new String(msg.getData()));
    }

    // Restart the broker and wait for the backoff to kick in. The client library will try to reconnect, and once
    // the broker is up, the consumer should receive the duplicate messages.
    log.info("-- Restarting broker --");
    restartBroker();

    msg = null;
    log.info("Receiving duplicate messages..");
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        log.info("Received: [{}]", new String(msg.getData()));
        Assert.assertNotNull(msg, "Message cannot be null");
    }
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}