Java Code Examples for org.apache.pulsar.client.api.Message#getKeyBytes()

The following examples show how to use org.apache.pulsar.client.api.Message#getKeyBytes() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: GroupMetadataManagerTest.java From kop with Apache License 2.0

4 votes

@Test
public void testStoreEmptyGroup() throws Exception {
    @Cleanup
    Consumer<ByteBuffer> consumer = pulsarClient.newConsumer(Schema.BYTEBUFFER)
        .topic(groupMetadataManager.getTopicPartitionName())
        .subscriptionName("test-sub")
        .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
        .subscribe();

    int generation = 27;
    String protocolType = "consumer";
    GroupMetadata group = GroupMetadata.loadGroup(
        groupId,
        Empty,
        generation,
        protocolType,
        null,
        null,
        Collections.emptyList()
    );
    groupMetadataManager.addGroup(group);

    Errors errors = groupMetadataManager.storeGroup(group, Collections.emptyMap()).get();
    assertEquals(Errors.NONE, errors);

    Message<ByteBuffer> message = consumer.receive();
    while (message.getValue().array().length == 0) {
        // bypass above place holder message.
        message = consumer.receive();
    }
    assertTrue(message.getEventTime() > 0L);
    assertTrue(message.hasKey());
    byte[] key = message.getKeyBytes();
    BaseKey groupKey = GroupMetadataConstants.readMessageKey(ByteBuffer.wrap(key));
    assertTrue(groupKey instanceof GroupMetadataKey);
    GroupMetadataKey groupMetadataKey = (GroupMetadataKey) groupKey;
    assertEquals(groupId, groupMetadataKey.key());

    ByteBuffer value = message.getValue();
    MemoryRecords memRecords = MemoryRecords.readableRecords(value);
    AtomicBoolean verified = new AtomicBoolean(false);
    memRecords.batches().forEach(batch -> {
        for (Record record : batch) {
            assertFalse(verified.get());
            BaseKey bk = GroupMetadataConstants.readMessageKey(record.key());
            assertTrue(bk instanceof GroupMetadataKey);
            GroupMetadataKey gmk = (GroupMetadataKey) bk;
            assertEquals(groupId, gmk.key());

            GroupMetadata gm = GroupMetadataConstants.readGroupMessageValue(
                groupId, record.value()
            );
            assertTrue(gm.is(Empty));
            assertEquals(generation, gm.generationId());
            assertEquals(Optional.of(protocolType), gm.protocolType());
            verified.set(true);
        }
    });
    assertTrue(verified.get());

}

Example 2

Source File: GroupMetadataManagerTest.java From kop with Apache License 2.0

4 votes

@Test
public void testStoreEmptySimpleGroup() throws Exception {
    @Cleanup
    Consumer<ByteBuffer> consumer = pulsarClient.newConsumer(Schema.BYTEBUFFER)
        .topic(groupMetadataManager.getTopicPartitionName())
        .subscriptionName("test-sub")
        .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
        .subscribe();
    GroupMetadata group = new GroupMetadata(groupId, Empty);
    groupMetadataManager.addGroup(group);

    Errors errors = groupMetadataManager.storeGroup(group, Collections.emptyMap()).get();
    assertEquals(Errors.NONE, errors);

    Message<ByteBuffer> message = consumer.receive();
    while (message.getValue().array().length == 0) {
        // bypass above place holder message.
        message = consumer.receive();
    }
    assertTrue(message.getEventTime() > 0L);
    assertTrue(message.hasKey());
    byte[] key = message.getKeyBytes();

    BaseKey groupKey = GroupMetadataConstants.readMessageKey(ByteBuffer.wrap(key));
    assertTrue(groupKey instanceof GroupMetadataKey);
    GroupMetadataKey groupMetadataKey = (GroupMetadataKey) groupKey;
    assertEquals(groupId, groupMetadataKey.key());

    ByteBuffer value = message.getValue();
    MemoryRecords memRecords = MemoryRecords.readableRecords(value);
    AtomicBoolean verified = new AtomicBoolean(false);
    memRecords.batches().forEach(batch -> {
        for (Record record : batch) {
            assertFalse(verified.get());
            BaseKey bk = GroupMetadataConstants.readMessageKey(record.key());
            assertTrue(bk instanceof GroupMetadataKey);
            GroupMetadataKey gmk = (GroupMetadataKey) bk;
            assertEquals(groupId, gmk.key());

            GroupMetadata gm = GroupMetadataConstants.readGroupMessageValue(
                groupId, record.value()
            );
            assertTrue(gm.is(Empty));
            assertEquals(0, gm.generationId());
            assertEquals(Optional.empty(), gm.protocolType());
            verified.set(true);
        }
    });
    assertTrue(verified.get());
}

Example 3

Source File: GroupMetadataManagerTest.java From kop with Apache License 2.0

4 votes

@Test
public void testStoreNoneEmptyGroup() throws Exception {
    @Cleanup
    Consumer<ByteBuffer> consumer = pulsarClient.newConsumer(Schema.BYTEBUFFER)
        .topic(groupMetadataManager.getTopicPartitionName())
        .subscriptionName("test-sub")
        .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
        .subscribe();
    String memberId = "memberId";
    String clientId = "clientId";
    String clientHost = "localhost";

    GroupMetadata group = new GroupMetadata(groupId, Empty);
    groupMetadataManager.addGroup(group);

    Map<String, byte[]> protocols = new HashMap<>();
    protocols.put("protocol", new byte[0]);
    MemberMetadata member = new MemberMetadata(
        memberId,
        groupId,
        clientId,
        clientHost,
        rebalanceTimeout,
        sessionTimeout,
        protocolType,
        protocols
    );
    CompletableFuture<JoinGroupResult> joinFuture = new CompletableFuture<>();
    member.awaitingJoinCallback(joinFuture);
    group.add(member);
    group.transitionTo(GroupState.PreparingRebalance);
    group.initNextGeneration();

    Map<String, byte[]> assignments = new HashMap<>();
    assignments.put(memberId, new byte[0]);
    Errors errors = groupMetadataManager.storeGroup(group, assignments).get();
    assertEquals(Errors.NONE, errors);

    Message<ByteBuffer> message = consumer.receive();
    while (message.getValue().array().length == 0) {
        // bypass above place holder message.
        message = consumer.receive();
    }
    assertTrue(message.getEventTime() > 0L);
    assertTrue(message.hasKey());
    byte[] key = message.getKeyBytes();
    BaseKey groupKey = GroupMetadataConstants.readMessageKey(ByteBuffer.wrap(key));
    assertTrue(groupKey instanceof GroupMetadataKey);
    GroupMetadataKey groupMetadataKey = (GroupMetadataKey) groupKey;
    assertEquals(groupId, groupMetadataKey.key());

    ByteBuffer value = message.getValue();
    MemoryRecords memRecords = MemoryRecords.readableRecords(value);
    AtomicBoolean verified = new AtomicBoolean(false);
    memRecords.batches().forEach(batch -> {
        for (Record record : batch) {
            assertFalse(verified.get());
            BaseKey bk = GroupMetadataConstants.readMessageKey(record.key());
            assertTrue(bk instanceof GroupMetadataKey);
            GroupMetadataKey gmk = (GroupMetadataKey) bk;
            assertEquals(groupId, gmk.key());

            GroupMetadata gm = GroupMetadataConstants.readGroupMessageValue(
                groupId, record.value()
            );
            assertEquals(Stable, gm.currentState());
            assertEquals(1, gm.generationId());
            assertEquals(Optional.of(protocolType), gm.protocolType());
            assertEquals("protocol", gm.protocolOrNull());
            assertTrue(gm.has(memberId));
            verified.set(true);
        }
    });
    assertTrue(verified.get());
}

Example 4

Source File: GroupMetadataManagerTest.java From kop with Apache License 2.0

4 votes

@Test
public void testCommitOffset() throws Exception {
    @Cleanup
    Consumer<ByteBuffer> consumer = pulsarClient.newConsumer(Schema.BYTEBUFFER)
        .topic(groupMetadataManager.getTopicPartitionName())
        .subscriptionName("test-sub")
        .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
        .subscribe();
    String memberId = "fakeMemberId";
    TopicPartition topicPartition = new TopicPartition("foo", 0);
    groupMetadataManager.addPartitionOwnership(groupPartitionId);
    long offset = 37L;

    GroupMetadata group = new GroupMetadata(groupId, Empty);
    groupMetadataManager.addGroup(group);

    Map<TopicPartition, OffsetAndMetadata> offsets = ImmutableMap.<TopicPartition, OffsetAndMetadata>builder()
        .put(topicPartition, OffsetAndMetadata.apply(offset))
        .build();

    Map<TopicPartition, Errors> commitErrors = groupMetadataManager.storeOffsets(
        group, memberId, offsets
    ).get();

    assertTrue(group.hasOffsets());
    assertFalse(commitErrors.isEmpty());
    Errors maybeError = commitErrors.get(topicPartition);
    assertEquals(Errors.NONE, maybeError);
    assertTrue(group.hasOffsets());

    Map<TopicPartition, PartitionData> cachedOffsets = groupMetadataManager.getOffsets(
        groupId,
        Optional.of(Lists.newArrayList(topicPartition))
    );
    PartitionData maybePartitionResponse = cachedOffsets.get(topicPartition);
    assertNotNull(maybePartitionResponse);

    assertEquals(Errors.NONE, maybePartitionResponse.error);
    assertEquals(offset, maybePartitionResponse.offset);

    Message<ByteBuffer> message = consumer.receive();
    while (message.getValue().array().length == 0) {
        // bypass above place holder message.
        message = consumer.receive();
    }
    assertTrue(message.getEventTime() > 0L);
    assertTrue(message.hasKey());
    byte[] key = message.getKeyBytes();
    BaseKey groupKey = GroupMetadataConstants.readMessageKey(ByteBuffer.wrap(key));
    assertTrue(groupKey instanceof OffsetKey);

    ByteBuffer value = message.getValue();
    MemoryRecords memRecords = MemoryRecords.readableRecords(value);
    AtomicBoolean verified = new AtomicBoolean(false);
    memRecords.batches().forEach(batch -> {
        for (Record record : batch) {
            assertFalse(verified.get());
            BaseKey bk = GroupMetadataConstants.readMessageKey(record.key());
            assertTrue(bk instanceof OffsetKey);
            OffsetKey ok = (OffsetKey) bk;
            GroupTopicPartition gtp = ok.key();
            assertEquals(groupId, gtp.group());
            assertEquals(topicPartition, gtp.topicPartition());

            OffsetAndMetadata gm = GroupMetadataConstants.readOffsetMessageValue(
                record.value()
            );
            assertEquals(offset, gm.offset());
            verified.set(true);
        }
    });
    assertTrue(verified.get());
}

Example 5

Source File: GroupMetadataManagerTest.java From kop with Apache License 2.0

4 votes

@Test
public void testGroupMetadataRemoval() throws Exception {
    @Cleanup
    Consumer<ByteBuffer> consumer = pulsarClient.newConsumer(Schema.BYTEBUFFER)
        .topic(groupMetadataManager.getTopicPartitionName())
        .subscriptionName("test-sub")
        .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
        .subscribe();
    TopicPartition topicPartition1 = new TopicPartition("foo", 0);
    TopicPartition topicPartition2 = new TopicPartition("foo", 1);

    groupMetadataManager.addPartitionOwnership(groupPartitionId);

    GroupMetadata group = new GroupMetadata(groupId, Empty);
    groupMetadataManager.addGroup(group);
    group.generationId(5);

    groupMetadataManager.cleanupGroupMetadata().get();

    Message<ByteBuffer> message = consumer.receive();
    while (message.getValue().array().length == 0) {
        // bypass above place holder message.
        message = consumer.receive();
    }
    assertTrue(message.getEventTime() > 0L);
    assertTrue(message.hasKey());
    byte[] key = message.getKeyBytes();

    BaseKey groupKey = GroupMetadataConstants.readMessageKey(ByteBuffer.wrap(key));
    assertTrue(groupKey instanceof GroupMetadataKey);
    GroupMetadataKey groupMetadataKey = (GroupMetadataKey) groupKey;
    assertEquals(groupId, groupMetadataKey.key());

    ByteBuffer value = message.getValue();
    MemoryRecords memRecords = MemoryRecords.readableRecords(value);
    AtomicBoolean verified = new AtomicBoolean(false);
    memRecords.batches().forEach(batch -> {
        assertEquals(RecordBatch.CURRENT_MAGIC_VALUE, batch.magic());
        assertEquals(TimestampType.CREATE_TIME, batch.timestampType());
        for (Record record : batch) {
            assertFalse(verified.get());
            assertTrue(record.hasKey());
            assertFalse(record.hasValue());
            assertTrue(record.timestamp() > 0);
            BaseKey bk = GroupMetadataConstants.readMessageKey(record.key());
            assertTrue(bk instanceof GroupMetadataKey);
            GroupMetadataKey gmk = (GroupMetadataKey) bk;
            assertEquals(groupId, gmk.key());
            verified.set(true);
        }
    });
    assertTrue(verified.get());
    assertEquals(Optional.empty(), groupMetadataManager.getGroup(groupId));
    Map<TopicPartition, PartitionData> cachedOffsets = groupMetadataManager.getOffsets(
        groupId,
        Optional.of(Lists.newArrayList(
            topicPartition1,
            topicPartition2
        ))
    );
    assertEquals(
        OffsetFetchResponse.INVALID_OFFSET,
        cachedOffsets.get(topicPartition1).offset);
    assertEquals(
        OffsetFetchResponse.INVALID_OFFSET,
        cachedOffsets.get(topicPartition2).offset);

}

Example 6

Source File: GroupMetadataManagerTest.java From kop with Apache License 2.0

4 votes

@Test
public void testExpireGroupWithOffsetsOnly() throws Exception {
    @Cleanup
    Consumer<ByteBuffer> consumer = pulsarClient.newConsumer(Schema.BYTEBUFFER)
        .topic(groupMetadataManager.getTopicPartitionName())
        .subscriptionName("test-sub")
        .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
        .subscribe();
    // verify that the group is removed properly, but no tombstone is written if
    // this is a group which is only using kafka for offset storage

    String memberId = "";
    TopicPartition topicPartition1 = new TopicPartition("foo", 0);
    TopicPartition topicPartition2 = new TopicPartition("foo", 1);
    long offset = 37;

    groupMetadataManager.addPartitionOwnership(groupPartitionId);

    GroupMetadata group = new GroupMetadata(groupId, Empty);
    groupMetadataManager.addGroup(group);

    long startMs = Time.SYSTEM.milliseconds();
    Map<TopicPartition, OffsetAndMetadata> offsets = ImmutableMap.<TopicPartition, OffsetAndMetadata>builder()
        .put(topicPartition1, OffsetAndMetadata.apply(offset, "", startMs, startMs + 1))
        .put(topicPartition2, OffsetAndMetadata.apply(offset, "", startMs, startMs + 3))
        .build();

    Map<TopicPartition, Errors> commitErrors =
        groupMetadataManager.storeOffsets(group, memberId, offsets).get();
    assertTrue(group.hasOffsets());

    assertFalse(commitErrors.isEmpty());
    assertEquals(
        Errors.NONE,
        commitErrors.get(topicPartition1)
    );

    groupMetadataManager.cleanupGroupMetadata().get();

    Message<ByteBuffer> message = consumer.receive();
    // skip `storeOffsets` op, bypass place holder message.
    while (!message.hasKey()
        || GroupMetadataConstants.readMessageKey(ByteBuffer.wrap(message.getKeyBytes())) instanceof OffsetKey) {
        message = consumer.receive();
    }

    assertTrue(message.getEventTime() > 0L);
    assertTrue(message.hasKey());
    byte[] key = message.getKeyBytes();

    BaseKey groupKey = GroupMetadataConstants.readMessageKey(ByteBuffer.wrap(key));
    assertTrue(groupKey instanceof GroupMetadataKey);
    GroupMetadataKey gmk = (GroupMetadataKey) groupKey;
    assertEquals(groupId, gmk.key());

    ByteBuffer value = message.getValue();
    MemoryRecords memRecords = MemoryRecords.readableRecords(value);
    AtomicInteger verified = new AtomicInteger(2);
    memRecords.batches().forEach(batch -> {
        assertEquals(RecordBatch.CURRENT_MAGIC_VALUE, batch.magic());
        assertEquals(TimestampType.CREATE_TIME, batch.timestampType());
        for (Record record : batch) {
            verified.decrementAndGet();
            assertTrue(record.hasKey());
            assertFalse(record.hasValue());
            assertTrue(record.timestamp() > 0);
            BaseKey bk = GroupMetadataConstants.readMessageKey(record.key());
            assertTrue(bk instanceof OffsetKey);
            OffsetKey ok = (OffsetKey) bk;
            assertEquals(groupId, ok.key().group());
            assertEquals("foo", ok.key().topicPartition().topic());
        }
    });
    assertEquals(0, verified.get());
    assertEquals(Optional.empty(), groupMetadataManager.getGroup(groupId));
    Map<TopicPartition, PartitionData> cachedOffsets = groupMetadataManager.getOffsets(
        groupId,
        Optional.of(Lists.newArrayList(
            topicPartition1,
            topicPartition2
        ))
    );
    assertEquals(
        OffsetFetchResponse.INVALID_OFFSET,
        cachedOffsets.get(topicPartition1).offset);
    assertEquals(
        OffsetFetchResponse.INVALID_OFFSET,
        cachedOffsets.get(topicPartition2).offset);
}