org.apache.pulsar.client.impl.MessageIdImpl Java Examples

@PUT
@Path("/{property}/{cluster}/{namespace}/{topic}/subscription/{subscriptionName}")
@ApiOperation(value = "Create a subscription on the topic.", notes = "Creates a subscription on the topic at the specified message id")
@ApiResponses(value = {
        @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"),
        @ApiResponse(code = 403, message = "Don't have admin permission"),
        @ApiResponse(code = 404, message = "Topic/Subscription does not exist"),
        @ApiResponse(code = 405, message = "Not supported for partitioned topics") })
public void createSubscription(@Suspended final AsyncResponse asyncResponse, @PathParam("property") String property,
        @PathParam("cluster") String cluster, @PathParam("namespace") String namespace,
        @PathParam("topic") @Encoded String topic, @PathParam("subscriptionName") String encodedSubName,
        @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, MessageIdImpl messageId,
        @QueryParam("replicated") boolean replicated) {
    try {
        validateTopicName(property, cluster, namespace, topic);
        internalCreateSubscription(asyncResponse, decode(encodedSubName), messageId, authoritative, replicated);
    } catch (WebApplicationException wae) {
        asyncResponse.resume(wae);
    } catch (Exception e) {
        asyncResponse.resume(new RestException(e));
    }
}
 

@Test
public void testNonPartitionedTopics() {
    pulsar.getConfiguration().setAllowAutoTopicCreation(false);
    final String nonPartitionTopic = "non-partitioned-topic";
    AsyncResponse response = mock(AsyncResponse.class);
    persistentTopics.createSubscription(response, testTenant, testNamespace, nonPartitionTopic, "test", true,
            (MessageIdImpl) MessageId.latest, false);
    ArgumentCaptor<Response> responseCaptor = ArgumentCaptor.forClass(Response.class);
    verify(response, timeout(5000).times(1)).resume(responseCaptor.capture());
    Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode());

    response = mock(AsyncResponse.class);
    persistentTopics.getSubscriptions(response, testTenant, testNamespace, nonPartitionTopic + "-partition-0",
            true);
    ArgumentCaptor<RestException> errorCaptor = ArgumentCaptor.forClass(RestException.class);
    verify(response, timeout(5000).times(1)).resume(errorCaptor.capture());
    Assert.assertTrue(errorCaptor.getValue().getMessage().contains("zero partitions"));

    final String nonPartitionTopic2 = "secondary-non-partitioned-topic";
    persistentTopics.createNonPartitionedTopic(testTenant, testNamespace, nonPartitionTopic2, true);
    Assert.assertEquals(persistentTopics
                    .getPartitionedMetadata(testTenant, testNamespace, nonPartitionTopic, true, false).partitions,
            0);
}
 

@Override
public CompletableFuture<MessageId> getLastMessageIdAsync(String topic) {
    TopicName tn = validateTopic(topic);
    WebTarget path = topicPath(tn, "lastMessageId");
    final CompletableFuture<MessageId> future = new CompletableFuture<>();
    asyncGetRequest(path,
            new InvocationCallback<BatchMessageIdImpl>() {

                @Override
                public void completed(BatchMessageIdImpl response) {
                    if (response.getBatchIndex() == -1) {
                        future.complete(new MessageIdImpl(response.getLedgerId(),
                                response.getEntryId(), response.getPartitionIndex()));
                    }
                    future.complete(response);
                }

                @Override
                public void failed(Throwable throwable) {
                    future.completeExceptionally(getApiException(throwable.getCause()));
                }
            });
    return future;
}
 

public static boolean messageIdRoughEquals(MessageId l, MessageId r) {
    if (l == null || r == null) {
        return false;
    }

    if (l instanceof BatchMessageIdImpl && r instanceof BatchMessageIdImpl) {
        return l.equals(r);
    } else if (l instanceof MessageIdImpl && r instanceof BatchMessageIdImpl) {
        BatchMessageIdImpl rb = (BatchMessageIdImpl) r;
        return l.equals(new MessageIdImpl(rb.getLedgerId(), rb.getEntryId(), rb.getPartitionIndex()));
    } else if (r instanceof MessageIdImpl && l instanceof BatchMessageIdImpl) {
        BatchMessageIdImpl lb = (BatchMessageIdImpl) l;
        return r.equals(new MessageIdImpl(lb.getLedgerId(), lb.getEntryId(), lb.getPartitionIndex()));
    } else if (l instanceof MessageIdImpl && r instanceof MessageIdImpl) {
        return l.equals(r);
    } else {
        throw new IllegalStateException(
                String.format("comparing messageIds of type %s, %s", l.getClass().toString(), r.getClass().toString()));
    }
}
 

@Override
public CompletableFuture<Void> triggerOffloadAsync(String topic, MessageId messageId) {
    TopicName tn = validateTopic(topic);
    WebTarget path = topicPath(tn, "offload");
    final CompletableFuture<Void> future = new CompletableFuture<>();
    try {
        request(path).async().put(Entity.entity(messageId, MediaType.APPLICATION_JSON)
                , new InvocationCallback<MessageIdImpl>() {
                    @Override
                    public void completed(MessageIdImpl response) {
                        future.complete(null);
                    }

                    @Override
                    public void failed(Throwable throwable) {
                        future.completeExceptionally(getApiException(throwable.getCause()));
                    }
                });
    } catch (PulsarAdminException cae) {
        future.completeExceptionally(cae);
    }
    return future;
}
 

@Test
public void testTerminatePartitionedTopic() {
    String testLocalTopicName = "topic-not-found";

    // 3) Create the partitioned topic
    AsyncResponse response  = mock(AsyncResponse.class);
    persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, testLocalTopicName, 1);
    ArgumentCaptor<Response> responseCaptor = ArgumentCaptor.forClass(Response.class);
    verify(response, timeout(5000).times(1)).resume(responseCaptor.capture());
    Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode());

    // 5) Create a subscription
    response  = mock(AsyncResponse.class);
    persistentTopics.createSubscription(response, testTenant, testNamespace, testLocalTopicName, "test", true,
            (MessageIdImpl) MessageId.earliest, false);
    responseCaptor = ArgumentCaptor.forClass(Response.class);
    verify(response, timeout(5000).times(1)).resume(responseCaptor.capture());
    Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode());

    // 9) terminate partitioned topic
    response = mock(AsyncResponse.class);
    persistentTopics.terminatePartitionedTopic(response, testTenant, testNamespace, testLocalTopicName, true);
    verify(response, timeout(5000).times(1)).resume(Arrays.asList(new MessageIdImpl(3, -1, -1)));
}
 

private CompletableFuture<PhaseOneResult> phaseOne(RawReader reader) {
    Map<String,MessageId> latestForKey = new HashMap<>();
    CompletableFuture<PhaseOneResult> loopPromise = new CompletableFuture<>();

    reader.getLastMessageIdAsync().whenComplete(
            (lastMessageId, exception) -> {
                if (exception != null) {
                    loopPromise.completeExceptionally(exception);
                } else {
                    log.info("Commencing phase one of compaction for {}, reading to {}",
                             reader.getTopic(), lastMessageId);
                    // Each entry is processed as a whole, discard the batchIndex part deliberately.
                    MessageIdImpl lastImpl = (MessageIdImpl) lastMessageId;
                    MessageIdImpl lastEntryMessageId = new MessageIdImpl(lastImpl.getLedgerId(), lastImpl.getEntryId(), lastImpl.getPartitionIndex());
                    phaseOneLoop(reader, Optional.empty(), Optional.empty(), lastEntryMessageId, latestForKey,
                            loopPromise);
                }
            });
    return loopPromise;
}
 

@PUT
@Path("/{tenant}/{cluster}/{namespace}/{topic}/offload")
@ApiOperation(value = "Offload a prefix of a topic to long term storage")
@ApiResponses(value = {
        @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"),
        @ApiResponse(code = 403, message = "Don't have admin permission"),
        @ApiResponse(code = 405, message = "Operation not allowed on persistent topic"),
        @ApiResponse(code = 404, message = "Topic does not exist"),
        @ApiResponse(code = 409, message = "Offload already running")})
public void triggerOffload(@PathParam("tenant") String tenant,
                           @PathParam("cluster") String cluster,
                           @PathParam("namespace") String namespace,
                           @PathParam("topic") @Encoded String encodedTopic,
                           @QueryParam("authoritative") @DefaultValue("false") boolean authoritative,
                           MessageIdImpl messageId) {
    validateTopicName(tenant, cluster, namespace, encodedTopic);
    internalTriggerOffload(authoritative, messageId);
}
 

@Test
public void testFindFirstLedgerWithinThreshold() throws Exception {
    List<LedgerInfo> ledgers = new ArrayList<>();
    ledgers.add(newLedger(0, 10, 1000));
    ledgers.add(newLedger(1, 10, 2000));
    ledgers.add(newLedger(2, 10, 3000));

    // test huge threshold
    Assert.assertNull(CmdTopics.findFirstLedgerWithinThreshold(ledgers, Long.MAX_VALUE));

    // test small threshold
    Assert.assertEquals(CmdTopics.findFirstLedgerWithinThreshold(ledgers, 0),
                        new MessageIdImpl(2, 0, -1));

    // test middling thresholds
    Assert.assertEquals(CmdTopics.findFirstLedgerWithinThreshold(ledgers, 1000),
                        new MessageIdImpl(2, 0, -1));
    Assert.assertEquals(CmdTopics.findFirstLedgerWithinThreshold(ledgers, 5000),
                        new MessageIdImpl(1, 0, -1));
}
 

@Test
public void testLeaderService() throws Exception {
    MessageId messageId = new MessageIdImpl(1, 2, -1);
    when(schedulerManager.getLastMessageProduced()).thenReturn(messageId);

    assertFalse(leaderService.isLeader());
    verify(mockClient, times(1)).newConsumer();

    listenerHolder.get().becameActive(mockConsumer, 0);
    assertTrue(leaderService.isLeader());

    verify(functionMetadataManager, times(1)).getIsInitialized();
    verify(metadataManagerInitFuture, times(1)).get();
    verify(functionRuntimeManager, times(1)).getIsInitialized();
    verify(runtimeManagerInitFuture, times(1)).get();

    verify(functionAssignmentTailer, times(1)).triggerReadToTheEndAndExit();
    verify(functionAssignmentTailer, times(1)).close();
    verify(schedulerManager, times((1))).initialize();

    listenerHolder.get().becameInactive(mockConsumer, 0);
    assertFalse(leaderService.isLeader());

    verify(functionAssignmentTailer, times(1)).startFromMessage(messageId);
    verify(schedulerManager, times(1)).close();
}
 

public static final long getSequenceId(MessageId messageId) {
    MessageIdImpl msgId = (MessageIdImpl) ((messageId instanceof TopicMessageIdImpl)
            ? ((TopicMessageIdImpl) messageId).getInnerMessageId()
            : messageId);
    long ledgerId = msgId.getLedgerId();
    long entryId = msgId.getEntryId();

    // Combine ledger id and entry id to form offset
    // Use less than 32 bits to represent entry id since it will get
    // rolled over way before overflowing the max int range
    long offset = (ledgerId << 28) | entryId;
    return offset;
}
 

@Test
public void testGetMessageId() {
    long lid = 12345L;
    long eid = 34566L;
    long sequenceId = (lid << 28) | eid;

    MessageIdImpl id = (MessageIdImpl) FunctionCommon.getMessageId(sequenceId);
    assertEquals(lid, id.getLedgerId());
    assertEquals(eid, id.getEntryId());
}
 

@POST
@Path("/{tenant}/{namespace}/{topic}/subscription/{subName}/resetcursor")
@ApiOperation(value = "Reset subscription to message position closest to given position.", notes = "It fence cursor and disconnects all active consumers before reseting cursor.")
@ApiResponses(value = {
        @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"),
        @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant or" +
                "subscriber is not authorized to access this operation"),
        @ApiResponse(code = 403, message = "Don't have admin permission"),
        @ApiResponse(code = 404, message = "Topic/Subscription does not exist"),
        @ApiResponse(code = 405, message = "Not supported for partitioned topics"),
        @ApiResponse(code = 412, message = "Unable to find position for position specified"),
        @ApiResponse(code = 500, message = "Internal server error"),
        @ApiResponse(code = 503, message = "Failed to validate global cluster configuration") })
public void resetCursorOnPosition(
        @ApiParam(value = "Specify the tenant", required = true)
        @PathParam("tenant") String tenant,
        @ApiParam(value = "Specify the namespace", required = true)
        @PathParam("namespace") String namespace,
        @ApiParam(value = "Specify topic name", required = true)
        @PathParam("topic") @Encoded String encodedTopic,
        @ApiParam(name = "subName", value = "Subscription to reset position on", required = true)
        @PathParam("subName") String encodedSubName,
        @ApiParam(value = "Is authentication required to perform this operation")
        @QueryParam("authoritative") @DefaultValue("false") boolean authoritative,
        @ApiParam(name = "messageId", value = "messageId to reset back to (ledgerId:entryId)")
        MessageIdImpl messageId) {
    validateTopicName(tenant, namespace, encodedTopic);
    internalResetCursorOnPosition(decode(encodedSubName), authoritative, messageId);
}
 

@PUT
@Path("/{tenant}/{namespace}/{topic}/offload")
@ApiOperation(value = "Offload a prefix of a topic to long term storage")
@ApiResponses(value = {
        @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"),
        @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant or" +
                "subscriber is not authorized to access this operation"),
        @ApiResponse(code = 403, message = "Don't have admin permission"),
        @ApiResponse(code = 404, message = "Topic does not exist"),
        @ApiResponse(code = 405, message = "Operation is not allowed on the persistent topic"),
        @ApiResponse(code = 409, message = "Offload already running"),
        @ApiResponse(code = 412, message = "Topic name is not valid"),
        @ApiResponse(code = 500, message = "Internal server error"),
        @ApiResponse(code = 503, message = "Failed to validate global cluster configuration") })
public void triggerOffload(
        @ApiParam(value = "Specify the tenant", required = true)
        @PathParam("tenant") String tenant,
        @ApiParam(value = "Specify the namespace", required = true)
        @PathParam("namespace") String namespace,
        @ApiParam(value = "Specify topic name", required = true)
        @PathParam("topic") @Encoded String encodedTopic,
        @ApiParam(value = "Is authentication required to perform this operation")
        @QueryParam("authoritative") @DefaultValue("false") boolean authoritative,
                           MessageIdImpl messageId) {
    validateTopicName(tenant, namespace, encodedTopic);
    internalTriggerOffload(authoritative, messageId);
}
 

private void internalCreateSubscriptionForNonPartitionedTopic(AsyncResponse asyncResponse, String subscriptionName,
          MessageIdImpl targetMessageId, boolean authoritative, boolean replicated) {
    try {
        validateAdminAccessForSubscriber(subscriptionName, authoritative);

        boolean isAllowAutoTopicCreation = pulsar().getConfiguration().isAllowAutoTopicCreation();
        PersistentTopic topic = (PersistentTopic) pulsar().getBrokerService()
                .getTopic(topicName.toString(), isAllowAutoTopicCreation).thenApply(Optional::get).join();
        if (topic.getSubscriptions().containsKey(subscriptionName)) {
            asyncResponse.resume(new RestException(Status.CONFLICT, "Subscription already exists for topic"));
            return;
        }
        PersistentSubscription subscription = (PersistentSubscription) topic
            .createSubscription(subscriptionName, InitialPosition.Latest, replicated).get();
        // Mark the cursor as "inactive" as it was created without a real consumer connected
        subscription.deactivateCursor();
        subscription.resetCursor(PositionImpl.get(targetMessageId.getLedgerId(), targetMessageId.getEntryId()))
            .get();
    } catch (Throwable e) {
        Throwable t = e.getCause();
        log.warn("[{}] [{}] Failed to create subscription {} at message id {}", clientAppId(), topicName,
            subscriptionName, targetMessageId, e);
        if (t instanceof SubscriptionInvalidCursorPosition) {
            asyncResponse.resume(new RestException(Status.PRECONDITION_FAILED,
                "Unable to find position for position specified: " + t.getMessage()));
        } else if (e instanceof WebApplicationException) {
            asyncResponse.resume(e);
        } else if (t instanceof SubscriptionBusyException) {
            asyncResponse.resume(new RestException(Status.PRECONDITION_FAILED,
                    "Failed for Subscription Busy: " + t.getMessage()));
        } else {
            asyncResponse.resume(new RestException(e));
        }
    }

    log.info("[{}][{}] Successfully created subscription {} at message id {}", clientAppId(), topicName,
        subscriptionName, targetMessageId);
    asyncResponse.resume(Response.noContent().build());
}
 

@Test
public void testGetSequenceId() {
    long lid = 12345L;
    long eid = 34566L;
    MessageIdImpl id = mock(MessageIdImpl.class);
    when(id.getLedgerId()).thenReturn(lid);
    when(id.getEntryId()).thenReturn(eid);

    assertEquals((lid << 28) | eid, FunctionCommon.getSequenceId(id));
}
 

public static final long getOffset(MessageId messageId) {
    MessageIdImpl msgId = (MessageIdImpl) messageId;
    long ledgerId = msgId.getLedgerId();
    long entryId = msgId.getEntryId();

    // Combine ledger id and entry id to form offset
    // Use less than 32 bits to represent entry id since it will get
    // rolled over way before overflowing the max int range
    long offset = (ledgerId << 28) | entryId;
    return offset;
}
 

static MessageId findFirstLedgerWithinThreshold(List<PersistentTopicInternalStats.LedgerInfo> ledgers,
                                                long sizeThreshold) {
    long suffixSize = 0L;

    ledgers = Lists.reverse(ledgers);
    long previousLedger = ledgers.get(0).ledgerId;
    for (PersistentTopicInternalStats.LedgerInfo l : ledgers) {
        suffixSize += l.size;
        if (suffixSize > sizeThreshold) {
            return new MessageIdImpl(previousLedger, 0L, -1);
        }
        previousLedger = l.ledgerId;
    }
    return null;
}
 

static MessageId validateMessageIdString(String resetMessageIdStr) throws PulsarAdminException {
    String[] messageId = resetMessageIdStr.split(":");
    try {
        Preconditions.checkArgument(messageId.length == 2);
        return new MessageIdImpl(Long.parseLong(messageId[0]), Long.parseLong(messageId[1]), -1);
    } catch (Exception e) {
        throw new PulsarAdminException(
                "Invalid message id (must be in format: ledgerId:entryId) value " + resetMessageIdStr);
    }
}
 

@Test()
public void testGetLastMessageId() throws Exception {
    TenantInfo tenantInfo = new TenantInfo(Sets.newHashSet("role1", "role2"), Sets.newHashSet("test"));
    admin.tenants().createTenant("prop-xyz", tenantInfo);
    admin.namespaces().createNamespace("prop-xyz/ns1", Sets.newHashSet("test"));
    final String topicName = "persistent://prop-xyz/ns1/testGetLastMessageId";

    admin.topics().createNonPartitionedTopic(topicName);
    Producer<byte[]> batchProducer = pulsarClient.newProducer().topic(topicName)
            .enableBatching(true)
            .batchingMaxMessages(100)
            .batchingMaxPublishDelay(2, TimeUnit.SECONDS)
            .create();
    admin.topics().createSubscription(topicName, "test", MessageId.earliest);
    CompletableFuture<MessageId> completableFuture = new CompletableFuture<>();
    for (int i = 0; i < 10; i++) {
        completableFuture = batchProducer.sendAsync("test".getBytes());
    }
    completableFuture.get();
    Assert.assertEquals(((BatchMessageIdImpl) admin.topics().getLastMessageId(topicName)).getBatchIndex(), 9);

    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
            .enableBatching(false)
            .create();
    producer.send("test".getBytes());

    Assert.assertTrue(admin.topics().getLastMessageId(topicName) instanceof MessageIdImpl);

}
 

private MessageId getMessageId() throws IOException {
    MessageId messageId = MessageId.latest;
    if (isNotBlank(queryParams.get("messageId"))) {
        if (queryParams.get("messageId").equals("earliest")) {
            messageId = MessageId.earliest;
        } else if (!queryParams.get("messageId").equals("latest")) {
            messageId = MessageIdImpl.fromByteArray(Base64.getDecoder().decode(queryParams.get("messageId")));
        }
    }
    return messageId;
}
 

@POST
@Path("/{property}/{cluster}/{namespace}/{topic}/subscription/{subName}/resetcursor")
@ApiOperation(hidden = true, value = "Reset subscription to message position closest to given position.", notes = "It fence cursor and disconnects all active consumers before reseting cursor.")
@ApiResponses(value = {
        @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"),
        @ApiResponse(code = 403, message = "Don't have admin permission"),
        @ApiResponse(code = 404, message = "Topic/Subscription does not exist"),
        @ApiResponse(code = 405, message = "Not supported for partitioned topics") })
public void resetCursorOnPosition(@PathParam("property") String property, @PathParam("cluster") String cluster,
        @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic,
        @PathParam("subName") String encodedSubName,
        @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, MessageIdImpl messageId) {
    validateTopicName(property, cluster, namespace, encodedTopic);
    internalResetCursorOnPosition(decode(encodedSubName), authoritative, messageId);
}
 

private RecordMetadata getRecordMetadata(String topic, MessageId messageId) {
    MessageIdImpl msgId = (MessageIdImpl) messageId;

    // Combine ledger id and entry id to form offset
    long offset = MessageIdUtils.getOffset(msgId);
    int partition = msgId.getPartitionIndex();

    TopicPartition tp = new TopicPartition(topic, partition);
    return new RecordMetadata(tp, offset, 0L);
}
 

public MessageId getPositionFromSubscription(String topic, MessageId defaultPosition) {
    try {
        TopicStats topicStats = admin.topics().getStats(topic);
        if (topicStats.subscriptions.containsKey(subscriptionName)) {
            SubscriptionStats subStats = topicStats.subscriptions.get(subscriptionName);
            if (subStats.consumers.size() != 0) {
                throw new RuntimeException("Subscription been actively used by other consumers, " +
                        "in this situation, the exactly-once semantics cannot be guaranteed.");
            } else {
                PersistentTopicInternalStats.CursorStats c =
                        admin.topics().getInternalStats(topic).cursors.get(subscriptionName);
                String[] ids = c.markDeletePosition.split(":", 2);
                long ledgerId = Long.parseLong(ids[0]);
                long entryIdInMarkDelete = Long.parseLong(ids[1]);
                // we are getting the next mid from sub position, if the entryId is -1,
                // it denotes we haven't read data from the ledger before,
                // therefore no need to skip the current entry for the next position
                long entryId = entryIdInMarkDelete == -1 ? -1 : entryIdInMarkDelete + 1;
                int partitionIdx = TopicName.getPartitionIndex(topic);
                return new MessageIdImpl(ledgerId, entryId, partitionIdx);
            }
        } else {
            // create sub on topic
            log.info("Setting up subscription {} on topic {} at position {}",
                subscriptionName, topic, defaultPosition);
            admin.topics().createSubscription(topic, subscriptionName, defaultPosition);
            log.info("Subscription {} on topic {} at position {} finished",
                subscriptionName, topic, defaultPosition);
            return defaultPosition;
        }
    } catch (PulsarAdminException e) {
        throw new RuntimeException("Failed to get stats for topic " + topic, e);
    }
}
 

@Override
public CompletableFuture<MessageId> terminateTopicAsync(String topic) {
    TopicName tn = validateTopic(topic);

    final CompletableFuture<MessageId> future = new CompletableFuture<>();
    try {
        final WebTarget path = topicPath(tn, "terminate");

        request(path).async().post(Entity.entity("", MediaType.APPLICATION_JSON),
                new InvocationCallback<MessageIdImpl>() {

                    @Override
                    public void completed(MessageIdImpl messageId) {
                        future.complete(messageId);
                    }

                    @Override
                    public void failed(Throwable throwable) {
                        log.warn("[{}] Failed to perform http post request: {}", path.getUri(),
                                throwable.getMessage());
                        future.completeExceptionally(getApiException(throwable.getCause()));
                    }
                });
    } catch (PulsarAdminException cae) {
        future.completeExceptionally(cae);
    }

    return future;
}
 

private RecordMetadata getRecordMetadata(String topic, TypedMessageBuilder<byte[]> msgBuilder, MessageId messageId,
        int size) {
    MessageIdImpl msgId = (MessageIdImpl) messageId;

    // Combine ledger id and entry id to form offset
    long offset = MessageIdUtils.getOffset(msgId);
    int partition = msgId.getPartitionIndex();

    TopicPartition tp = new TopicPartition(topic, partition);
    TypedMessageBuilderImpl<byte[]> mb = (TypedMessageBuilderImpl<byte[]>) msgBuilder;
    return new RecordMetadata(tp, offset, 0L, mb.getPublishTime(), 0L, mb.hasKey() ? mb.getKey().length() : 0, size);
}
 

@Test
void shouldPreferEventTimeOverPublishTime() throws Exception {
    var topic = getTopic();
    var eventTimestamp = Instant.now().minusSeconds(1000).truncatedTo(ChronoUnit.MILLIS);

    int partition;
    try (
            var pulsarClient = PulsarClient.builder()
                    .serviceUrl(pulsar.getPulsarBrokerUrl())
                    .build()
    ) {
        var messageId = pulsarClient.newProducer()
                .topic(topic)
                .create()
                .newMessage()
                .value("hello".getBytes())
                .eventTime(eventTimestamp.toEpochMilli())
                .send();

        partition = ((MessageIdImpl) messageId).getPartitionIndex();
    }

    var record = subscribeToPartition(partition)
            .flatMap(RecordsStorage.PartitionSource::getPublisher)
            .blockFirst(Duration.ofSeconds(10));

    assertThat(record).satisfies(it -> {
        assertThat(it.getTimestamp()).isEqualTo(eventTimestamp);
    });
}
 

public static long toOffset(MessageId messageId) {
    MessageIdImpl msgId = (MessageIdImpl) messageId;
    // Combine ledger id and entry id to form offset
    // Use less than 32 bits to represent entry id since it will get
    // rolled over way before overflowing the max int range
    return (msgId.getLedgerId() << 28) | msgId.getEntryId();
}
 

/**
 * Verify: 1. Broker should not replay already acknowledged messages 2. Dispatcher should not stuck while
 * dispatching new messages due to previous-replay of invalid/already-acked messages
 *
 * @throws Exception
 */
@Test
public void testMessageReplay() throws Exception {

    final String topicName = "persistent://prop/ns-abc/topic2";
    final String subName = "sub2";

    Message<byte[]> msg;
    int totalMessages = 10;
    int replayIndex = totalMessages / 2;

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
            .subscriptionType(SubscriptionType.Shared).receiverQueueSize(1).subscribe();
    Producer<byte[]> producer = pulsarClient.newProducer()
        .topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();

    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();
    assertNotNull(topicRef);
    PersistentSubscription subRef = topicRef.getSubscription(subName);
    PersistentDispatcherMultipleConsumers dispatcher = (PersistentDispatcherMultipleConsumers) subRef
            .getDispatcher();
    Field replayMap = PersistentDispatcherMultipleConsumers.class.getDeclaredField("messagesToRedeliver");
    replayMap.setAccessible(true);
    ConcurrentLongPairSet messagesToReplay = new ConcurrentLongPairSet(64, 1);

    assertNotNull(subRef);

    // (1) Produce messages
    for (int i = 0; i < totalMessages; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }

    MessageIdImpl firstAckedMsg = null;
    // (2) Consume and ack messages except first message
    for (int i = 0; i < totalMessages; i++) {
        msg = consumer.receive();
        consumer.acknowledge(msg);
        MessageIdImpl msgId = (MessageIdImpl) msg.getMessageId();
        if (i == 0) {
            firstAckedMsg = msgId;
        }
        if (i < replayIndex) {
            // (3) accumulate acked messages for replay
            messagesToReplay.add(msgId.getLedgerId(), msgId.getEntryId());
        }
    }

    // (4) redelivery : should redeliver only unacked messages
    Thread.sleep(1000);

    replayMap.set(dispatcher, messagesToReplay);
    // (a) redelivery with all acked-message should clear messageReply bucket
    dispatcher.redeliverUnacknowledgedMessages(dispatcher.getConsumers().get(0));
    assertEquals(messagesToReplay.size(), 0);

    // (b) fill messageReplyBucket with already acked entry again: and try to publish new msg and read it
    messagesToReplay.add(firstAckedMsg.getLedgerId(), firstAckedMsg.getEntryId());
    replayMap.set(dispatcher, messagesToReplay);
    // send new message
    final String testMsg = "testMsg";
    producer.send(testMsg.getBytes());
    // consumer should be able to receive only new message and not the
    dispatcher.consumerFlow(dispatcher.getConsumers().get(0), 1);
    msg = consumer.receive(1, TimeUnit.SECONDS);
    assertNotNull(msg);
    assertEquals(msg.getData(), testMsg.getBytes());

    consumer.close();
    producer.close();
}
 

/**
 * It verifies that redelivery-of-specific messages: that redelivers all those messages even when consumer gets
 * blocked due to unacked messsages
 *
 * Usecase: Consumer starts consuming only after all messages have been produced. So, consumer consumes total
 * receiver-queue-size number messages => ask for redelivery and receives all messages again.
 *
 * @throws Exception
 */
@Test
public void testBlockUnackedConsumerRedeliverySpecificMessagesCloseConsumerWhileProduce() throws Exception {
    log.info("-- Starting {} test --", methodName);

    int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
    try {
        final int unAckedMessagesBufferSize = 10;
        final int receiverQueueSize = 20;
        final int totalProducedMsgs = 50;

        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessagesBufferSize);
        // Only subscribe consumer
        ConsumerImpl<byte[]> consumer = (ConsumerImpl<byte[]>) pulsarClient.newConsumer()
                .topic("persistent://my-property/my-ns/unacked-topic").subscriptionName("subscriber-1")
                .receiverQueueSize(receiverQueueSize).subscriptionType(SubscriptionType.Shared).subscribe();
        consumer.close();

        Producer<byte[]> producer = pulsarClient.newProducer()
                .topic("persistent://my-property/my-ns/unacked-topic").create();

        // (1) Produced Messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
            Thread.sleep(10);
        }

        // (1.a) start consumer again
        consumer = (ConsumerImpl<byte[]>) pulsarClient.newConsumer()
                .topic("persistent://my-property/my-ns/unacked-topic").subscriptionName("subscriber-1")
                .receiverQueueSize(receiverQueueSize).subscriptionType(SubscriptionType.Shared).subscribe();

        // (2) try to consume messages: but will be able to consume number of messages = unAckedMessagesBufferSize
        Message<byte[]> msg = null;
        List<Message<byte[]>> messages1 = Lists.newArrayList();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages1.add(msg);
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }

        // client should not receive all produced messages and should be blocked due to unack-messages
        Set<MessageIdImpl> redeliveryMessages = messages1.stream().map(m -> {
            return (MessageIdImpl) m.getMessageId();
        }).collect(Collectors.toSet());

        // (3) redeliver all consumed messages
        consumer.redeliverUnacknowledgedMessages(Sets.newHashSet(redeliveryMessages));
        Thread.sleep(1000);

        Set<MessageIdImpl> messages2 = Sets.newHashSet();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages2.add((MessageIdImpl) msg.getMessageId());
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }

        assertEquals(messages1.size(), messages2.size());
        // (4) Verify: redelivered all previous unacked-consumed messages
        messages2.removeAll(redeliveryMessages);
        assertEquals(messages2.size(), 0);
        producer.close();
        consumer.close();
        log.info("-- Exiting {} test --", methodName);
    } catch (Exception e) {
        fail();
    } finally {
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
    }
}