Java Code Examples for org.apache.pulsar.client.api.Message#getValue()

@Test(dataProvider = "batch")
public void testSyncProducerAndConsumer(int batchMessageDelayMs) throws Exception {
    log.info("-- Starting {} test --", methodName);

    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING)
                .topic("persistent://my-property/use/my-ns/my-topic1")
                .subscriptionName("my-subscriber-name")
                .subscriptionType(SubscriptionType.Exclusive)
                .subscribe();

    ProducerBuilder<String> producerBuilder = pulsarClient.newProducer(Schema.STRING)
            .topic("persistent://my-property/use/my-ns/my-topic1");

    if (batchMessageDelayMs != 0) {
        producerBuilder.enableBatching(true)
            .batchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS)
            .batchingMaxMessages(5);
    } else {
        producerBuilder.enableBatching(false);
    }

    Producer<String> producer = producerBuilder.create();
    for (int i = 0; i < 10; i++) {
        producer.send("my-message-" + i);
    }

    Message<String> msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = msg.getValue();
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 

/**
 * Publish from Java and consume from Python
 */
@Test(dataProvider = "ServiceUrls")
public void testPythonPublishJavaConsume(String serviceUrl) throws Exception {
    String nsName = generateNamespaceName();
    pulsarCluster.createNamespace(nsName);

    String topicName = generateTopicName(nsName, "testPythonPublishJavaConsume", true);

    @Cleanup
    PulsarClient client = PulsarClient.builder()
            .serviceUrl(serviceUrl)
            .build();

    @Cleanup
    Consumer<Example2> consumer = client.newConsumer(Schema.AVRO(Example2.class))
            .topic(topicName)
            .subscriptionName("test-sub")
            .subscribe();

    // Verify Python can receive the typed message

    ContainerExecResult res = pulsarCluster.getAnyBroker()
            .execCmd("/pulsar/examples/python-examples/producer_schema.py", "pulsar://localhost:6650", topicName);
    assertEquals(res.getExitCode(), 0);

    Message<Example2> msg = consumer.receive();
    Example2 e2 = msg.getValue();
    assertEquals(e2.a, "Hello");
    assertEquals(e2.b, 1);
}
 

@Test
public void testStoreEmptyGroup() throws Exception {
    @Cleanup
    Consumer<ByteBuffer> consumer = pulsarClient.newConsumer(Schema.BYTEBUFFER)
        .topic(groupMetadataManager.getTopicPartitionName())
        .subscriptionName("test-sub")
        .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
        .subscribe();

    int generation = 27;
    String protocolType = "consumer";
    GroupMetadata group = GroupMetadata.loadGroup(
        groupId,
        Empty,
        generation,
        protocolType,
        null,
        null,
        Collections.emptyList()
    );
    groupMetadataManager.addGroup(group);

    Errors errors = groupMetadataManager.storeGroup(group, Collections.emptyMap()).get();
    assertEquals(Errors.NONE, errors);

    Message<ByteBuffer> message = consumer.receive();
    while (message.getValue().array().length == 0) {
        // bypass above place holder message.
        message = consumer.receive();
    }
    assertTrue(message.getEventTime() > 0L);
    assertTrue(message.hasKey());
    byte[] key = message.getKeyBytes();
    BaseKey groupKey = GroupMetadataConstants.readMessageKey(ByteBuffer.wrap(key));
    assertTrue(groupKey instanceof GroupMetadataKey);
    GroupMetadataKey groupMetadataKey = (GroupMetadataKey) groupKey;
    assertEquals(groupId, groupMetadataKey.key());

    ByteBuffer value = message.getValue();
    MemoryRecords memRecords = MemoryRecords.readableRecords(value);
    AtomicBoolean verified = new AtomicBoolean(false);
    memRecords.batches().forEach(batch -> {
        for (Record record : batch) {
            assertFalse(verified.get());
            BaseKey bk = GroupMetadataConstants.readMessageKey(record.key());
            assertTrue(bk instanceof GroupMetadataKey);
            GroupMetadataKey gmk = (GroupMetadataKey) bk;
            assertEquals(groupId, gmk.key());

            GroupMetadata gm = GroupMetadataConstants.readGroupMessageValue(
                groupId, record.value()
            );
            assertTrue(gm.is(Empty));
            assertEquals(generation, gm.generationId());
            assertEquals(Optional.of(protocolType), gm.protocolType());
            verified.set(true);
        }
    });
    assertTrue(verified.get());

}
 

@Test
public void testStoreEmptySimpleGroup() throws Exception {
    @Cleanup
    Consumer<ByteBuffer> consumer = pulsarClient.newConsumer(Schema.BYTEBUFFER)
        .topic(groupMetadataManager.getTopicPartitionName())
        .subscriptionName("test-sub")
        .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
        .subscribe();
    GroupMetadata group = new GroupMetadata(groupId, Empty);
    groupMetadataManager.addGroup(group);

    Errors errors = groupMetadataManager.storeGroup(group, Collections.emptyMap()).get();
    assertEquals(Errors.NONE, errors);

    Message<ByteBuffer> message = consumer.receive();
    while (message.getValue().array().length == 0) {
        // bypass above place holder message.
        message = consumer.receive();
    }
    assertTrue(message.getEventTime() > 0L);
    assertTrue(message.hasKey());
    byte[] key = message.getKeyBytes();

    BaseKey groupKey = GroupMetadataConstants.readMessageKey(ByteBuffer.wrap(key));
    assertTrue(groupKey instanceof GroupMetadataKey);
    GroupMetadataKey groupMetadataKey = (GroupMetadataKey) groupKey;
    assertEquals(groupId, groupMetadataKey.key());

    ByteBuffer value = message.getValue();
    MemoryRecords memRecords = MemoryRecords.readableRecords(value);
    AtomicBoolean verified = new AtomicBoolean(false);
    memRecords.batches().forEach(batch -> {
        for (Record record : batch) {
            assertFalse(verified.get());
            BaseKey bk = GroupMetadataConstants.readMessageKey(record.key());
            assertTrue(bk instanceof GroupMetadataKey);
            GroupMetadataKey gmk = (GroupMetadataKey) bk;
            assertEquals(groupId, gmk.key());

            GroupMetadata gm = GroupMetadataConstants.readGroupMessageValue(
                groupId, record.value()
            );
            assertTrue(gm.is(Empty));
            assertEquals(0, gm.generationId());
            assertEquals(Optional.empty(), gm.protocolType());
            verified.set(true);
        }
    });
    assertTrue(verified.get());
}
 

@Test
public void testStoreNoneEmptyGroup() throws Exception {
    @Cleanup
    Consumer<ByteBuffer> consumer = pulsarClient.newConsumer(Schema.BYTEBUFFER)
        .topic(groupMetadataManager.getTopicPartitionName())
        .subscriptionName("test-sub")
        .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
        .subscribe();
    String memberId = "memberId";
    String clientId = "clientId";
    String clientHost = "localhost";

    GroupMetadata group = new GroupMetadata(groupId, Empty);
    groupMetadataManager.addGroup(group);

    Map<String, byte[]> protocols = new HashMap<>();
    protocols.put("protocol", new byte[0]);
    MemberMetadata member = new MemberMetadata(
        memberId,
        groupId,
        clientId,
        clientHost,
        rebalanceTimeout,
        sessionTimeout,
        protocolType,
        protocols
    );
    CompletableFuture<JoinGroupResult> joinFuture = new CompletableFuture<>();
    member.awaitingJoinCallback(joinFuture);
    group.add(member);
    group.transitionTo(GroupState.PreparingRebalance);
    group.initNextGeneration();

    Map<String, byte[]> assignments = new HashMap<>();
    assignments.put(memberId, new byte[0]);
    Errors errors = groupMetadataManager.storeGroup(group, assignments).get();
    assertEquals(Errors.NONE, errors);

    Message<ByteBuffer> message = consumer.receive();
    while (message.getValue().array().length == 0) {
        // bypass above place holder message.
        message = consumer.receive();
    }
    assertTrue(message.getEventTime() > 0L);
    assertTrue(message.hasKey());
    byte[] key = message.getKeyBytes();
    BaseKey groupKey = GroupMetadataConstants.readMessageKey(ByteBuffer.wrap(key));
    assertTrue(groupKey instanceof GroupMetadataKey);
    GroupMetadataKey groupMetadataKey = (GroupMetadataKey) groupKey;
    assertEquals(groupId, groupMetadataKey.key());

    ByteBuffer value = message.getValue();
    MemoryRecords memRecords = MemoryRecords.readableRecords(value);
    AtomicBoolean verified = new AtomicBoolean(false);
    memRecords.batches().forEach(batch -> {
        for (Record record : batch) {
            assertFalse(verified.get());
            BaseKey bk = GroupMetadataConstants.readMessageKey(record.key());
            assertTrue(bk instanceof GroupMetadataKey);
            GroupMetadataKey gmk = (GroupMetadataKey) bk;
            assertEquals(groupId, gmk.key());

            GroupMetadata gm = GroupMetadataConstants.readGroupMessageValue(
                groupId, record.value()
            );
            assertEquals(Stable, gm.currentState());
            assertEquals(1, gm.generationId());
            assertEquals(Optional.of(protocolType), gm.protocolType());
            assertEquals("protocol", gm.protocolOrNull());
            assertTrue(gm.has(memberId));
            verified.set(true);
        }
    });
    assertTrue(verified.get());
}
 

@Test
public void testCommitOffset() throws Exception {
    @Cleanup
    Consumer<ByteBuffer> consumer = pulsarClient.newConsumer(Schema.BYTEBUFFER)
        .topic(groupMetadataManager.getTopicPartitionName())
        .subscriptionName("test-sub")
        .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
        .subscribe();
    String memberId = "fakeMemberId";
    TopicPartition topicPartition = new TopicPartition("foo", 0);
    groupMetadataManager.addPartitionOwnership(groupPartitionId);
    long offset = 37L;

    GroupMetadata group = new GroupMetadata(groupId, Empty);
    groupMetadataManager.addGroup(group);

    Map<TopicPartition, OffsetAndMetadata> offsets = ImmutableMap.<TopicPartition, OffsetAndMetadata>builder()
        .put(topicPartition, OffsetAndMetadata.apply(offset))
        .build();

    Map<TopicPartition, Errors> commitErrors = groupMetadataManager.storeOffsets(
        group, memberId, offsets
    ).get();

    assertTrue(group.hasOffsets());
    assertFalse(commitErrors.isEmpty());
    Errors maybeError = commitErrors.get(topicPartition);
    assertEquals(Errors.NONE, maybeError);
    assertTrue(group.hasOffsets());

    Map<TopicPartition, PartitionData> cachedOffsets = groupMetadataManager.getOffsets(
        groupId,
        Optional.of(Lists.newArrayList(topicPartition))
    );
    PartitionData maybePartitionResponse = cachedOffsets.get(topicPartition);
    assertNotNull(maybePartitionResponse);

    assertEquals(Errors.NONE, maybePartitionResponse.error);
    assertEquals(offset, maybePartitionResponse.offset);

    Message<ByteBuffer> message = consumer.receive();
    while (message.getValue().array().length == 0) {
        // bypass above place holder message.
        message = consumer.receive();
    }
    assertTrue(message.getEventTime() > 0L);
    assertTrue(message.hasKey());
    byte[] key = message.getKeyBytes();
    BaseKey groupKey = GroupMetadataConstants.readMessageKey(ByteBuffer.wrap(key));
    assertTrue(groupKey instanceof OffsetKey);

    ByteBuffer value = message.getValue();
    MemoryRecords memRecords = MemoryRecords.readableRecords(value);
    AtomicBoolean verified = new AtomicBoolean(false);
    memRecords.batches().forEach(batch -> {
        for (Record record : batch) {
            assertFalse(verified.get());
            BaseKey bk = GroupMetadataConstants.readMessageKey(record.key());
            assertTrue(bk instanceof OffsetKey);
            OffsetKey ok = (OffsetKey) bk;
            GroupTopicPartition gtp = ok.key();
            assertEquals(groupId, gtp.group());
            assertEquals(topicPartition, gtp.topicPartition());

            OffsetAndMetadata gm = GroupMetadataConstants.readOffsetMessageValue(
                record.value()
            );
            assertEquals(offset, gm.offset());
            verified.set(true);
        }
    });
    assertTrue(verified.get());
}
 

@Test
public void testGroupMetadataRemoval() throws Exception {
    @Cleanup
    Consumer<ByteBuffer> consumer = pulsarClient.newConsumer(Schema.BYTEBUFFER)
        .topic(groupMetadataManager.getTopicPartitionName())
        .subscriptionName("test-sub")
        .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
        .subscribe();
    TopicPartition topicPartition1 = new TopicPartition("foo", 0);
    TopicPartition topicPartition2 = new TopicPartition("foo", 1);

    groupMetadataManager.addPartitionOwnership(groupPartitionId);

    GroupMetadata group = new GroupMetadata(groupId, Empty);
    groupMetadataManager.addGroup(group);
    group.generationId(5);

    groupMetadataManager.cleanupGroupMetadata().get();

    Message<ByteBuffer> message = consumer.receive();
    while (message.getValue().array().length == 0) {
        // bypass above place holder message.
        message = consumer.receive();
    }
    assertTrue(message.getEventTime() > 0L);
    assertTrue(message.hasKey());
    byte[] key = message.getKeyBytes();

    BaseKey groupKey = GroupMetadataConstants.readMessageKey(ByteBuffer.wrap(key));
    assertTrue(groupKey instanceof GroupMetadataKey);
    GroupMetadataKey groupMetadataKey = (GroupMetadataKey) groupKey;
    assertEquals(groupId, groupMetadataKey.key());

    ByteBuffer value = message.getValue();
    MemoryRecords memRecords = MemoryRecords.readableRecords(value);
    AtomicBoolean verified = new AtomicBoolean(false);
    memRecords.batches().forEach(batch -> {
        assertEquals(RecordBatch.CURRENT_MAGIC_VALUE, batch.magic());
        assertEquals(TimestampType.CREATE_TIME, batch.timestampType());
        for (Record record : batch) {
            assertFalse(verified.get());
            assertTrue(record.hasKey());
            assertFalse(record.hasValue());
            assertTrue(record.timestamp() > 0);
            BaseKey bk = GroupMetadataConstants.readMessageKey(record.key());
            assertTrue(bk instanceof GroupMetadataKey);
            GroupMetadataKey gmk = (GroupMetadataKey) bk;
            assertEquals(groupId, gmk.key());
            verified.set(true);
        }
    });
    assertTrue(verified.get());
    assertEquals(Optional.empty(), groupMetadataManager.getGroup(groupId));
    Map<TopicPartition, PartitionData> cachedOffsets = groupMetadataManager.getOffsets(
        groupId,
        Optional.of(Lists.newArrayList(
            topicPartition1,
            topicPartition2
        ))
    );
    assertEquals(
        OffsetFetchResponse.INVALID_OFFSET,
        cachedOffsets.get(topicPartition1).offset);
    assertEquals(
        OffsetFetchResponse.INVALID_OFFSET,
        cachedOffsets.get(topicPartition2).offset);

}
 

@Test
public void testExpireGroupWithOffsetsOnly() throws Exception {
    @Cleanup
    Consumer<ByteBuffer> consumer = pulsarClient.newConsumer(Schema.BYTEBUFFER)
        .topic(groupMetadataManager.getTopicPartitionName())
        .subscriptionName("test-sub")
        .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
        .subscribe();
    // verify that the group is removed properly, but no tombstone is written if
    // this is a group which is only using kafka for offset storage

    String memberId = "";
    TopicPartition topicPartition1 = new TopicPartition("foo", 0);
    TopicPartition topicPartition2 = new TopicPartition("foo", 1);
    long offset = 37;

    groupMetadataManager.addPartitionOwnership(groupPartitionId);

    GroupMetadata group = new GroupMetadata(groupId, Empty);
    groupMetadataManager.addGroup(group);

    long startMs = Time.SYSTEM.milliseconds();
    Map<TopicPartition, OffsetAndMetadata> offsets = ImmutableMap.<TopicPartition, OffsetAndMetadata>builder()
        .put(topicPartition1, OffsetAndMetadata.apply(offset, "", startMs, startMs + 1))
        .put(topicPartition2, OffsetAndMetadata.apply(offset, "", startMs, startMs + 3))
        .build();

    Map<TopicPartition, Errors> commitErrors =
        groupMetadataManager.storeOffsets(group, memberId, offsets).get();
    assertTrue(group.hasOffsets());

    assertFalse(commitErrors.isEmpty());
    assertEquals(
        Errors.NONE,
        commitErrors.get(topicPartition1)
    );

    groupMetadataManager.cleanupGroupMetadata().get();

    Message<ByteBuffer> message = consumer.receive();
    // skip `storeOffsets` op, bypass place holder message.
    while (!message.hasKey()
        || GroupMetadataConstants.readMessageKey(ByteBuffer.wrap(message.getKeyBytes())) instanceof OffsetKey) {
        message = consumer.receive();
    }

    assertTrue(message.getEventTime() > 0L);
    assertTrue(message.hasKey());
    byte[] key = message.getKeyBytes();

    BaseKey groupKey = GroupMetadataConstants.readMessageKey(ByteBuffer.wrap(key));
    assertTrue(groupKey instanceof GroupMetadataKey);
    GroupMetadataKey gmk = (GroupMetadataKey) groupKey;
    assertEquals(groupId, gmk.key());

    ByteBuffer value = message.getValue();
    MemoryRecords memRecords = MemoryRecords.readableRecords(value);
    AtomicInteger verified = new AtomicInteger(2);
    memRecords.batches().forEach(batch -> {
        assertEquals(RecordBatch.CURRENT_MAGIC_VALUE, batch.magic());
        assertEquals(TimestampType.CREATE_TIME, batch.timestampType());
        for (Record record : batch) {
            verified.decrementAndGet();
            assertTrue(record.hasKey());
            assertFalse(record.hasValue());
            assertTrue(record.timestamp() > 0);
            BaseKey bk = GroupMetadataConstants.readMessageKey(record.key());
            assertTrue(bk instanceof OffsetKey);
            OffsetKey ok = (OffsetKey) bk;
            assertEquals(groupId, ok.key().group());
            assertEquals("foo", ok.key().topicPartition().topic());
        }
    });
    assertEquals(0, verified.get());
    assertEquals(Optional.empty(), groupMetadataManager.getGroup(groupId));
    Map<TopicPartition, PartitionData> cachedOffsets = groupMetadataManager.getOffsets(
        groupId,
        Optional.of(Lists.newArrayList(
            topicPartition1,
            topicPartition2
        ))
    );
    assertEquals(
        OffsetFetchResponse.INVALID_OFFSET,
        cachedOffsets.get(topicPartition1).offset);
    assertEquals(
        OffsetFetchResponse.INVALID_OFFSET,
        cachedOffsets.get(topicPartition2).offset);
}
 

@SuppressWarnings("unchecked")
@Override
public PulsarMessageAndMetadata<K, V> next() {

    Message<byte[]> msg = receivedMessages.poll();
    if (msg == null) {
        try {
            msg = consumer.receive();
        } catch (PulsarClientException e) {
            log.warn("Failed to receive message for {}-{}, {}", consumer.getTopic(), consumer.getSubscription(),
                    e.getMessage(), e);
            throw new RuntimeException(
                    "failed to receive message from " + consumer.getTopic() + "-" + consumer.getSubscription());
        }
    }

    int partition = TopicName.getPartitionIndex(consumer.getTopic());
    long offset = MessageIdUtils.getOffset(msg.getMessageId());
    String key = msg.getKey();
    byte[] value = msg.getValue();

    K desKey = null;
    V desValue = null;

    if (StringUtils.isNotBlank(key)) {
        if (keyDeSerializer.isPresent() && keyDeSerializer.get() instanceof StringDecoder) {
            desKey = (K) key;
        } else {
            byte[] decodedBytes = Base64.getDecoder().decode(key);
            desKey = keyDeSerializer.isPresent() ? keyDeSerializer.get().fromBytes(decodedBytes)
                    : (K) DEFAULT_DECODER.fromBytes(decodedBytes);
        }
    }

    if (value != null) {
        desValue = valueDeSerializer.isPresent() ? valueDeSerializer.get().fromBytes(msg.getData())
                : (V) DEFAULT_DECODER.fromBytes(msg.getData());
    }

    PulsarMessageAndMetadata<K, V> msgAndMetadata = new PulsarMessageAndMetadata<>(consumer.getTopic(), partition,
            null, offset, keyDeSerializer.orElse(null), valueDeSerializer.orElse(null), desKey, desValue);

    if (isAutoCommit) {
        // Commit the offset of previously dequeued messages
        consumer.acknowledgeCumulativeAsync(msg);
    }

    lastConsumedMessageId = msg.getMessageId();
    return msgAndMetadata;
}
 

@Test
public void keyValueNullInlineTest() throws PulsarClientException {
    String topic = "persistent://prop/ns-abc/kv-null-value-test";

    @Cleanup
    Producer<KeyValue<String, String>> producer = pulsarClient
            .newProducer(KeyValueSchema.of(Schema.STRING, Schema.STRING))
            .topic(topic)
            .create();

    @Cleanup
    Consumer<KeyValue<String, String>> consumer = pulsarClient
            .newConsumer(KeyValueSchema.of(Schema.STRING, Schema.STRING))
            .topic(topic)
            .subscriptionName("test")
            .subscribe();

    int numMessage = 10;
    for (int i = 0; i < numMessage; i++) {
        producer.newMessage().value(new KeyValue<>(null, "test")).send();
        producer.newMessage().value(new KeyValue<>("test", null)).send();
        producer.newMessage().value(new KeyValue<>(null, null)).send();
    }

    Message<KeyValue<String, String>> message;
    KeyValue<String, String> keyValue;
    for (int i = 0; i < numMessage; i++) {
        message = consumer.receive();
        keyValue = message.getValue();
        Assert.assertNull(keyValue.getKey());
        Assert.assertEquals("test", keyValue.getValue());

        message = consumer.receive();
        keyValue = message.getValue();
        Assert.assertEquals("test", keyValue.getKey());
        Assert.assertNull(keyValue.getValue());

        message = consumer.receive();
        keyValue = message.getValue();
        Assert.assertNull(keyValue.getKey());
        Assert.assertNull(keyValue.getValue());
    }

}
 

@Test
public void keyValueNullSeparatedTest() throws PulsarClientException {
    String topic = "persistent://prop/ns-abc/kv-null-value-test";

    @Cleanup
    Producer<KeyValue<String, String>> producer = pulsarClient
            .newProducer(KeyValueSchema.of(Schema.STRING, Schema.STRING, KeyValueEncodingType.SEPARATED))
            .topic(topic)
            .create();

    @Cleanup
    Consumer<KeyValue<String, String>> consumer = pulsarClient
            .newConsumer(KeyValueSchema.of(Schema.STRING, Schema.STRING, KeyValueEncodingType.SEPARATED))
            .topic(topic)
            .subscriptionName("test")
            .subscribe();

    int numMessage = 10;
    for (int i = 0; i < numMessage; i++) {
        producer.newMessage().value(new KeyValue<>(null, "test")).send();
        producer.newMessage().value(new KeyValue<>("test", null)).send();
        producer.newMessage().value(new KeyValue<>(null, null)).send();
    }

    Message<KeyValue<String, String>> message;
    KeyValue<String, String> keyValue;
    for (int i = 0; i < numMessage; i++) {
        message = consumer.receive();
        keyValue = message.getValue();
        Assert.assertNull(keyValue.getKey());
        Assert.assertEquals("test", keyValue.getValue());

        message = consumer.receive();
        keyValue = message.getValue();
        Assert.assertEquals("test", keyValue.getKey());
        Assert.assertNull(keyValue.getValue());

        message = consumer.receive();
        keyValue = message.getValue();
        Assert.assertNull(keyValue.getKey());
        Assert.assertNull(keyValue.getValue());
    }

}
 

@Test(dataProvider =  "CanReadLastSchemaCompatibilityStrategy")
public void testConsumerCompatibilityCheckCanReadLastTest(SchemaCompatibilityStrategy schemaCompatibilityStrategy) throws Exception {
    final String tenant = PUBLIC_TENANT;
    final String topic = "test-consumer-compatibility";

        String namespace = "test-namespace-" + randomName(16);
        String fqtn = TopicName.get(
                TopicDomain.persistent.value(),
                tenant,
                namespace,
                topic
        ).toString();

        NamespaceName namespaceName = NamespaceName.get(tenant, namespace);

        admin.namespaces().createNamespace(
                tenant + "/" + namespace,
                Sets.newHashSet(CLUSTER_NAME)
        );

        admin.namespaces().setSchemaCompatibilityStrategy(namespaceName.toString(), schemaCompatibilityStrategy);
        admin.schemas().createSchema(fqtn, Schema.AVRO(Schemas.PersonOne.class).getSchemaInfo());
        admin.schemas().createSchema(fqtn, Schema.AVRO(SchemaDefinition.builder()
                .withAlwaysAllowNull(false).withPojo(Schemas.PersonTwo.class).build()).getSchemaInfo());

        Consumer<Schemas.PersonThree> consumerThree = pulsarClient.newConsumer(Schema.AVRO(
                SchemaDefinition.<Schemas.PersonThree>builder().withAlwaysAllowNull
                        (false).withSupportSchemaVersioning(true).
                        withPojo(Schemas.PersonThree.class).build()))
                .subscriptionName("test")
                .topic(fqtn)
                .subscribe();

        Producer<Schemas.PersonOne> producerOne = pulsarClient
                .newProducer(Schema.AVRO(Schemas.PersonOne.class))
                .topic(fqtn)
                .create();


        Schemas.PersonOne personOne = new Schemas.PersonOne();
        personOne.setId(1);

        producerOne.send(personOne);
        Message<Schemas.PersonThree> message = null;

        try {
            message = consumerThree.receive();
            message.getValue();
        } catch (Exception e) {
            Assert.assertTrue(e instanceof SchemaSerializationException);
            consumerThree.acknowledge(message);
        }

        Producer<Schemas.PersonTwo> producerTwo = pulsarClient
                .newProducer(Schema.AVRO(SchemaDefinition.<Schemas.PersonTwo>builder().withAlwaysAllowNull
                        (false).withSupportSchemaVersioning(true).
                        withPojo(Schemas.PersonTwo.class).build()))
                .topic(fqtn)
                .create();

        Schemas.PersonTwo personTwo = new Schemas.PersonTwo();
        personTwo.setId(1);
        personTwo.setName("Jerry");
        producerTwo.send(personTwo);

        message = consumerThree.receive();
        Schemas.PersonThree personThree = message.getValue();
        consumerThree.acknowledge(message);

        assertEquals(personThree.getId(), 1);
        assertEquals(personThree.getName(), "Jerry");

        consumerThree.close();
        producerOne.close();
        producerTwo.close();
}
 

@Test(dataProvider =  "AllCheckSchemaCompatibilityStrategy")
public void testIsAutoUpdateSchema(SchemaCompatibilityStrategy schemaCompatibilityStrategy) throws Exception {
    final String tenant = PUBLIC_TENANT;
    final String topic = "test-consumer-compatibility";

    String namespace = "test-namespace-" + randomName(16);
    String fqtn = TopicName.get(
            TopicDomain.persistent.value(),
            tenant,
            namespace,
            topic
    ).toString();

    NamespaceName namespaceName = NamespaceName.get(tenant, namespace);

    admin.namespaces().createNamespace(
            tenant + "/" + namespace,
            Sets.newHashSet(CLUSTER_NAME)
    );

    assertEquals(admin.namespaces().getSchemaCompatibilityStrategy(namespaceName.toString()),
            SchemaCompatibilityStrategy.FULL);
    
    admin.namespaces().setSchemaCompatibilityStrategy(namespaceName.toString(), schemaCompatibilityStrategy);
    admin.schemas().createSchema(fqtn, Schema.AVRO(Schemas.PersonOne.class).getSchemaInfo());

    admin.namespaces().setIsAllowAutoUpdateSchema(namespaceName.toString(), false);
    ProducerBuilder<Schemas.PersonTwo> producerThreeBuilder = pulsarClient
            .newProducer(Schema.AVRO(SchemaDefinition.<Schemas.PersonTwo>builder().withAlwaysAllowNull
                    (false).withSupportSchemaVersioning(true).
                    withPojo(Schemas.PersonTwo.class).build()))
            .topic(fqtn);
    try {
        producerThreeBuilder.create();
    } catch (Exception e) {
        Assert.assertTrue(e.getMessage().contains("Schema not found and schema auto updating is disabled."));
    }

    admin.namespaces().setIsAllowAutoUpdateSchema(namespaceName.toString(), true);
    ConsumerBuilder<Schemas.PersonTwo> comsumerBuilder = pulsarClient.newConsumer(Schema.AVRO(
            SchemaDefinition.<Schemas.PersonTwo>builder().withAlwaysAllowNull
                    (false).withSupportSchemaVersioning(true).
                    withPojo(Schemas.PersonTwo.class).build()))
            .subscriptionName("test")
            .topic(fqtn);

    Producer<Schemas.PersonTwo> producer = producerThreeBuilder.create();
    Consumer<Schemas.PersonTwo> consumerTwo = comsumerBuilder.subscribe();

    producer.send(new Schemas.PersonTwo(2, "Lucy"));
    Message<Schemas.PersonTwo> message = consumerTwo.receive();

    Schemas.PersonTwo personTwo = message.getValue();
    consumerTwo.acknowledge(message);

    assertEquals(personTwo.getId(), 2);
    assertEquals(personTwo.getName(), "Lucy");

    producer.close();
    consumerTwo.close();

    admin.namespaces().setIsAllowAutoUpdateSchema(namespaceName.toString(), false);

    producer = producerThreeBuilder.create();
    consumerTwo = comsumerBuilder.subscribe();

    producer.send(new Schemas.PersonTwo(2, "Lucy"));
    message = consumerTwo.receive();

    personTwo = message.getValue();
    consumerTwo.acknowledge(message);

    assertEquals(personTwo.getId(), 2);
    assertEquals(personTwo.getName(), "Lucy");

    consumerTwo.close();
    producer.close();
}
 

@Test(dataProvider = "AllCheckSchemaCompatibilityStrategy")
public void testProducerSendWithOldSchemaAndConsumerCanRead(SchemaCompatibilityStrategy schemaCompatibilityStrategy) throws Exception {
    final String tenant = PUBLIC_TENANT;
    final String topic = "test-consumer-compatibility";
    String namespace = "test-namespace-" + randomName(16);
    String fqtn = TopicName.get(
            TopicDomain.persistent.value(),
            tenant,
            namespace,
            topic
    ).toString();

    NamespaceName namespaceName = NamespaceName.get(tenant, namespace);

    admin.namespaces().createNamespace(
            tenant + "/" + namespace,
            Sets.newHashSet(CLUSTER_NAME)
    );

    admin.namespaces().setSchemaCompatibilityStrategy(namespaceName.toString(), schemaCompatibilityStrategy);
    admin.schemas().createSchema(fqtn, Schema.AVRO(Schemas.PersonOne.class).getSchemaInfo());
    admin.schemas().createSchema(fqtn, Schema.AVRO(SchemaDefinition.builder()
            .withAlwaysAllowNull(false).withPojo(Schemas.PersonTwo.class).build()).getSchemaInfo());

    Producer<Schemas.PersonOne> producerOne = pulsarClient
            .newProducer(Schema.AVRO(Schemas.PersonOne.class))
            .topic(fqtn)
            .create();


    Schemas.PersonOne personOne = new Schemas.PersonOne(10);

    Consumer<Schemas.PersonOne> consumerOne = pulsarClient.newConsumer(Schema.AVRO(
            SchemaDefinition.<Schemas.PersonOne>builder().withAlwaysAllowNull
                    (false).withSupportSchemaVersioning(true).
                    withPojo(Schemas.PersonOne.class).build()))
            .subscriptionName("test")
            .topic(fqtn)
            .subscribe();

    producerOne.send(personOne);
    Message<Schemas.PersonOne> message = consumerOne.receive();
    personOne = message.getValue();

    assertEquals(10, personOne.getId());

    consumerOne.close();
    producerOne.close();

}
 

private static void publishAndConsumeMessagesBytes(String inputTopic,
                                                   String outputTopic,
                                                   int numMessages) throws Exception {
    @Cleanup PulsarClient client = PulsarClient.builder()
        .serviceUrl(pulsarCluster.getPlainTextServiceUrl())
        .build();

    @Cleanup Consumer<byte[]> consumer = client.newConsumer(Schema.BYTES)
        .topic(outputTopic)
        .subscriptionType(SubscriptionType.Exclusive)
        .subscriptionName("test-sub")
        .subscribe();

    if (inputTopic.endsWith(".*")) {
        @Cleanup Producer<byte[]> producer1 = client.newProducer(Schema.BYTES)
                .topic(inputTopic.substring(0, inputTopic.length() - 2) + "1")
                .create();

        @Cleanup Producer<byte[]> producer2 = client.newProducer(Schema.BYTES)
                .topic(inputTopic.substring(0, inputTopic.length() - 2) + "2")
                .create();

        for (int i = 0; i < numMessages / 2; i++) {
            producer1.send(("message-" + i).getBytes(UTF_8));
        }

        for (int i = numMessages / 2; i < numMessages; i++) {
            producer2.send(("message-" + i).getBytes(UTF_8));
        }
    } else {
        @Cleanup Producer<byte[]> producer = client.newProducer(Schema.BYTES)
                .topic(inputTopic)
                .create();

        for (int i = 0; i < numMessages; i++) {
            producer.send(("message-" + i).getBytes(UTF_8));
        }
    }

    Set<String> expectedMessages = new HashSet<>();
    for (int i = 0; i < numMessages; i++) {
        expectedMessages.add("message-" + i + "!");
    }

    for (int i = 0; i < numMessages; i++) {
        Message<byte[]> msg = consumer.receive(30, TimeUnit.SECONDS);
        String msgValue = new String(msg.getValue(), UTF_8);
        log.info("Received: {}", msgValue);
        assertTrue(expectedMessages.contains(msgValue));
        expectedMessages.remove(msgValue);
    }
}
 

public static void main(String[] args) throws Exception {
    String serviceUrl = "pulsar://localhost:6650";

    PulsarClientImpl client = (PulsarClientImpl) PulsarClient.builder()
        .serviceUrl(serviceUrl)
        .build();

    String inputTopic = "input-topic";
    String outputTopic1 = "output-topic-1";
    String outputTopic2 = "output-topic-2";

    ConsumerImpl<String> consumer = (ConsumerImpl<String>) client.newConsumer(Schema.STRING)
        .topic(inputTopic)
        .subscriptionType(SubscriptionType.Exclusive)
        .subscriptionName("transactional-sub")
        .subscribe();

    ProducerImpl<String> producer1 = (ProducerImpl<String>) client.newProducer(Schema.STRING)
        .topic(outputTopic1)
        .sendTimeout(0, TimeUnit.MILLISECONDS)
        .create();

    ProducerImpl<String> producer2 = (ProducerImpl<String>) client.newProducer(Schema.STRING)
        .topic(outputTopic2)
        .sendTimeout(0, TimeUnit.MILLISECONDS)
        .create();


    while (true) {
        Message<String> message = consumer.receive();

        // process the messages to generate other messages
        String outputMessage1 = message.getValue() + "-output-1";
        String outputMessage2= message.getValue() + "-output-2";

        Transaction txn = client.newTransaction()
            .withTransactionTimeout(1, TimeUnit.MINUTES)
            .build().get();

        CompletableFuture<MessageId> sendFuture1 = producer1.newMessage(txn)
            .value(outputMessage1)
            .sendAsync();

        CompletableFuture<MessageId> sendFuture2 = producer2.newMessage(txn)
            .value(outputMessage2)
            .sendAsync();

        CompletableFuture<Void> ackFuture = consumer.acknowledgeAsync(message.getMessageId(), txn);

        txn.commit().get();

        // the message ids can be returned from the sendFuture1 and sendFuture2

        MessageId msgId1 = sendFuture1.get();
        MessageId msgId2 = sendFuture2.get();
    }
}