org.apache.pulsar.client.api.MessageId Java Examples

The following examples show how to use org.apache.pulsar.client.api.MessageId. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CreateSubscriptionTest.java    From pulsar with Apache License 2.0 6 votes vote down vote up
@Test
public void createSubscriptionOnPartitionedTopicWithPartialFailure() throws Exception {
    String topic = "persistent://my-property/my-ns/my-partitioned-topic";
    admin.topics().createPartitionedTopic(topic, 10);
    
    // create subscription for one partition
    final String partitionedTopic0 = topic+"-partition-0";
    admin.topics().createSubscription(partitionedTopic0, "sub-1", MessageId.latest);

    admin.topics().createSubscription(topic, "sub-1", MessageId.latest);

    // Create should fail if the subscription already exists
    try {
        admin.topics().createSubscription(topic, "sub-1", MessageId.latest);
        fail("Should have failed");
    } catch (Exception e) {
        // Expected
    }

    for (int i = 0; i < 10; i++) {
        assertEquals(
                admin.topics().getSubscriptions(TopicName.get(topic).getPartition(i).toString()),
                Lists.newArrayList("sub-1"));
    }
}
 
Example #2
Source File: PersistentTopics.java    From pulsar with Apache License 2.0 6 votes vote down vote up
@POST
@Path("/{tenant}/{namespace}/{topic}/terminate")
@ApiOperation(value = "Terminate a topic. A topic that is terminated will not accept any more "
        + "messages to be published and will let consumer to drain existing messages in backlog")
@ApiResponses(value = {
        @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"),
        @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant or" +
                "subscriber is not authorized to access this operation"),
        @ApiResponse(code = 403, message = "Don't have admin permission"),
        @ApiResponse(code = 404, message = "Topic does not exist"),
        @ApiResponse(code = 405, message = "Termination of a partitioned topic is not allowed"),
        @ApiResponse(code = 412, message = "Topic name is not valid"),
        @ApiResponse(code = 500, message = "Internal server error"),
        @ApiResponse(code = 503, message = "Failed to validate global cluster configuration") })
public MessageId terminate(
        @ApiParam(value = "Specify the tenant", required = true)
        @PathParam("tenant") String tenant,
        @ApiParam(value = "Specify the namespace", required = true)
        @PathParam("namespace") String namespace,
        @ApiParam(value = "Specify topic name", required = true)
        @PathParam("topic") @Encoded String encodedTopic,
        @ApiParam(value = "Is authentication required to perform this operation")
        @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) {
    validateTopicName(tenant, namespace, encodedTopic);
    return internalTerminate(authoritative);
}
 
Example #3
Source File: TopicTerminationTest.java    From pulsar with Apache License 2.0 6 votes vote down vote up
@Test
public void testSimpleTermination() throws Exception {
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();

    /* MessageId msgId1 = */producer.send("test-msg-1".getBytes());
    /* MessageId msgId2 = */producer.send("test-msg-2".getBytes());
    MessageId msgId3 = producer.send("test-msg-3".getBytes());

    MessageId lastMessageId = admin.topics().terminateTopicAsync(topicName).get();
    assertEquals(lastMessageId, msgId3);

    try {
        producer.send("test-msg-4".getBytes());
        fail("Should have thrown exception");
    } catch (PulsarClientException.TopicTerminatedException e) {
        // Expected
    }
}
 
Example #4
Source File: FlinkPulsarRowSource.java    From pulsar-flink with Apache License 2.0 6 votes vote down vote up
@Override
protected PulsarFetcher<Row> createFetcher(
        SourceContext sourceContext,
        Map<String, MessageId> seedTopicsWithInitialOffsets,
        SerializedValue<AssignerWithPeriodicWatermarks<Row>> watermarksPeriodic,
        SerializedValue<AssignerWithPunctuatedWatermarks<Row>> watermarksPunctuated,
        ProcessingTimeService processingTimeProvider,
        long autoWatermarkInterval,
        ClassLoader userCodeClassLoader,
        StreamingRuntimeContext streamingRuntime) throws Exception {

    return new PulsarRowFetcher(
            sourceContext,
            seedTopicsWithInitialOffsets,
            watermarksPeriodic,
            watermarksPunctuated,
            processingTimeProvider,
            autoWatermarkInterval,
            userCodeClassLoader,
            streamingRuntime,
            clientConfigurationData,
            readerConf,
            pollTimeoutMs,
            null,
metadataReader);
}
 
Example #5
Source File: CmdPersistentTopics.java    From pulsar with Apache License 2.0 6 votes vote down vote up
@Override
void run() throws PulsarAdminException {
    String persistentTopic = validatePersistentTopic(params);
    if (isNotBlank(resetMessageIdStr)) {
        MessageId messageId = validateMessageIdString(resetMessageIdStr);
        persistentTopics.resetCursor(persistentTopic, subName, messageId);
    } else if (isNotBlank(resetTimeStr)) {
        long resetTimeInMillis = TimeUnit.SECONDS
                .toMillis(RelativeTimeUtil.parseRelativeTimeInSeconds(resetTimeStr));
        // now - go back time
        long timestamp = System.currentTimeMillis() - resetTimeInMillis;
        persistentTopics.resetCursor(persistentTopic, subName, timestamp);
    } else {
        throw new PulsarAdminException(
                "Either Timestamp (--time) or Position (--position) has to be provided to reset cursor");
    }
}
 
Example #6
Source File: MessageIdImpl.java    From pulsar with Apache License 2.0 6 votes vote down vote up
public static MessageId fromByteArray(byte[] data) throws IOException {
    checkNotNull(data);
    ByteBufCodedInputStream inputStream = ByteBufCodedInputStream.get(Unpooled.wrappedBuffer(data, 0, data.length));
    PulsarApi.MessageIdData.Builder builder = PulsarApi.MessageIdData.newBuilder();

    PulsarApi.MessageIdData idData;
    try {
        idData = builder.mergeFrom(inputStream, null).build();
    } catch (UninitializedMessageException e) {
        throw new IOException(e);
    }

    MessageIdImpl messageId;
    if (idData.hasBatchIndex()) {
        messageId = new BatchMessageIdImpl(idData.getLedgerId(), idData.getEntryId(), idData.getPartition(),
                idData.getBatchIndex());
    } else {
        messageId = new MessageIdImpl(idData.getLedgerId(), idData.getEntryId(), idData.getPartition());
    }

    inputStream.recycle();
    builder.recycle();
    idData.recycle();
    return messageId;
}
 
Example #7
Source File: PulsarKafkaSimpleConsumer.java    From pulsar with Apache License 2.0 6 votes vote down vote up
/**
 * <pre>
 * Overriden method: OffsetCommitResponse commitOffsets(OffsetCommitRequest request)
 * 
 * Note:
 * created PulsarOffsetCommitResponse as OffsetCommitRequest doesn't provide getters
 * 
 * </pre>
 */
public OffsetCommitResponse commitOffsets(PulsarOffsetCommitRequest request) {

    PulsarOffsetCommitResponse response = new PulsarOffsetCommitResponse(null);
    for (Entry<String, MessageId> topicOffset : request.getTopicOffsetMap().entrySet()) {
        final String topic = topicOffset.getKey();
        final String groupId = request.getGroupId();
        try {
            Consumer<byte[]> consumer = getConsumer(topic, groupId);
            consumer.acknowledgeCumulative(topicOffset.getValue());
        } catch (Exception e) {
            log.warn("Failed to ack message for topic {}-{}", topic, topicOffset.getValue(), e);
            response.hasError = true;
            TopicAndPartition topicPartition = new TopicAndPartition(topic, 0);
            response.errors.computeIfAbsent(topicPartition, tp -> ErrorMapping.UnknownCode());
        }
    }

    return response;
}
 
Example #8
Source File: FlinkPulsarSource.java    From pulsar-flink with Apache License 2.0 6 votes vote down vote up
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
    OperatorStateStore stateStore = context.getOperatorStateStore();

    unionOffsetStates = stateStore.getUnionListState(
            new ListStateDescriptor<>(
                    OFFSETS_STATE_NAME,
                    TypeInformation.of(new TypeHint<Tuple2<String, MessageId>>() {
                    })));

    if (context.isRestored()) {
        restoredState = new TreeMap<>();
        unionOffsetStates.get().forEach(e -> restoredState.put(e.f0, e.f1));
        log.info("Source subtask {} restored state {}",
                taskIndex,
                StringUtils.join(restoredState.entrySet()));
    } else {
        log.info("Source subtask {} has no restore state", taskIndex);
    }
}
 
Example #9
Source File: ReaderBuilderImpl.java    From pulsar with Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<Reader<T>> createAsync() {
    if (conf.getTopicName() == null) {
        return FutureUtil
                .failedFuture(new IllegalArgumentException("Topic name must be set on the reader builder"));
    }

    if (conf.getStartMessageId() != null && conf.getStartMessageFromRollbackDurationInSec() > 0 ||
            conf.getStartMessageId() == null && conf.getStartMessageFromRollbackDurationInSec() <= 0) {
        return FutureUtil
                .failedFuture(new IllegalArgumentException(
                        "Start message id or start message from roll back must be specified but they cannot be specified at the same time"));
    }

    if (conf.getStartMessageFromRollbackDurationInSec() > 0) {
        conf.setStartMessageId(MessageId.earliest);
    }

    return client.createReaderAsync(conf, schema);
}
 
Example #10
Source File: RawReaderTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Test
public void testAcknowledgeWithProperties() throws Exception {
    int numKeys = 10;

    String topic = "persistent://my-property/my-ns/my-raw-topic";

    Set<String> keys = publishMessages(topic, numKeys);

    RawReader reader = RawReader.create(pulsarClient, topic, subscription).get();
    MessageId lastMessageId = reader.getLastMessageIdAsync().get();

    while (true) {
        try (RawMessage m = reader.readNextAsync().get()) {
            Assert.assertTrue(keys.remove(extractKey(m)));

            if (lastMessageId.compareTo(m.getMessageId()) == 0) {
                break;
            }
        }
    }
    Assert.assertTrue(keys.isEmpty());

    Map<String,Long> properties = new HashMap<>();
    properties.put("foobar", 0xdeadbeefdecaL);
    reader.acknowledgeCumulativeAsync(lastMessageId, properties).get();

    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic).get();
    ManagedLedger ledger = topicRef.getManagedLedger();
    for (int i = 0; i < 30; i++) {
        if (ledger.openCursor(subscription).getProperties().get("foobar") == Long.valueOf(0xdeadbeefdecaL)) {
            break;
        }
        Thread.sleep(100);
    }
    Assert.assertEquals(ledger.openCursor(subscription).getProperties().get("foobar"),
            Long.valueOf(0xdeadbeefdecaL));
}
 
Example #11
Source File: TopicTerminationTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Test(timeOut = 20000)
public void testSimpleTerminationReader() throws Exception {
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();

    MessageId msgId1 = producer.send("test-msg-1".getBytes());
    MessageId msgId2 = producer.send("test-msg-2".getBytes());
    MessageId msgId3 = producer.send("test-msg-3".getBytes());

    MessageId lastMessageId = admin.topics().terminateTopicAsync(topicName).get();
    assertEquals(lastMessageId, msgId3);

    Reader<byte[]> reader = pulsarClient.newReader().topic(topicName).startMessageId(MessageId.earliest).create();

    Message<byte[]> msg1 = reader.readNext();
    assertEquals(msg1.getMessageId(), msgId1);

    Message<byte[]> msg2 = reader.readNext();
    assertEquals(msg2.getMessageId(), msgId2);

    Message<byte[]> msg3 = reader.readNext();
    assertEquals(msg3.getMessageId(), msgId3);

    Message<byte[]> msg4 = reader.readNext(100, TimeUnit.MILLISECONDS);
    assertNull(msg4);

    Thread.sleep(100);
    assertTrue(reader.hasReachedEndOfTopic());
}
 
Example #12
Source File: ConsumerBase.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Override
public MessageId getLastMessageId() throws PulsarClientException {
    try {
        return getLastMessageIdAsync().get();
    } catch (Exception e) {
        throw PulsarClientException.unwrap(e);
    }
}
 
Example #13
Source File: PulsarKafkaSimpleConsumer.java    From pulsar with Apache License 2.0 5 votes vote down vote up
private MessageId getMessageId(long offset) {
    if (kafka.api.OffsetRequest.EarliestTime() == offset) {
        return MessageId.earliest;
    } else if (kafka.api.OffsetRequest.LatestTime() == offset) {
        return MessageId.latest;
    } else {
        return MessageIdUtils.getMessageId(offset);
    }
}
 
Example #14
Source File: ConsumerBase.java    From pulsar with Apache License 2.0 5 votes vote down vote up
public CompletableFuture<Void> acknowledgeAsync(MessageId messageId,
                                                Transaction txn) {
    TransactionImpl txnImpl = null;
    if (null != txn) {
        checkArgument(txn instanceof TransactionImpl);
        txnImpl = (TransactionImpl) txn;
    }
    return doAcknowledgeWithTxn(messageId, AckType.Individual, Collections.emptyMap(), txnImpl);
}
 
Example #15
Source File: FlinkPulsarSource.java    From pulsar-flink with Apache License 2.0 5 votes vote down vote up
public FlinkPulsarSource<T> setStartFromSpecificOffsets(Map<String, MessageId> specificStartupOffsets) {
    this.startupMode = StartupMode.SPECIFIC_OFFSETS;
    this.specificStartupOffsets = checkNotNull(specificStartupOffsets);
    this.specificStartupOffsetsAsBytes = new HashMap<>();
    for (Map.Entry<String, MessageId> entry : specificStartupOffsets.entrySet()) {
        specificStartupOffsetsAsBytes.put(entry.getKey(), entry.getValue().toByteArray());
    }
    return this;
}
 
Example #16
Source File: ProducerImpl.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Override
public CompletableFuture<Void> flushAsync() {
    CompletableFuture<MessageId> lastSendFuture;
    synchronized (ProducerImpl.this) {
        if (isBatchMessagingEnabled()) {
            batchMessageAndSend();
        }
        lastSendFuture = this.lastSendFuture;
    }
    return lastSendFuture.thenApply(ignored -> null);
}
 
Example #17
Source File: UnAckedMessageTrackerTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Test
public void testAddAndRemove() throws Exception {
    PulsarClientImpl client = mock(PulsarClientImpl.class);
    Timer timer = new HashedWheelTimer(new DefaultThreadFactory("pulsar-timer", Thread.currentThread().isDaemon()),
            1, TimeUnit.MILLISECONDS);
    when(client.timer()).thenReturn(timer);

    ConsumerBase<byte[]> consumer = mock(ConsumerBase.class);
    doNothing().when(consumer).onAckTimeoutSend(any());
    doNothing().when(consumer).redeliverUnacknowledgedMessages(any());

    UnAckedMessageTracker tracker = new UnAckedMessageTracker(client, consumer, 1000000, 100000);
    tracker.close();

    assertTrue(tracker.isEmpty());
    assertEquals(tracker.size(), 0);

    MessageIdImpl mid = new MessageIdImpl(1L, 1L, -1);
    assertTrue(tracker.add(mid));
    assertFalse(tracker.add(mid));
    assertEquals(tracker.size(), 1);

    ConcurrentOpenHashSet<MessageId> headPartition = tracker.timePartitions.removeFirst();
    headPartition.clear();
    tracker.timePartitions.addLast(headPartition);

    assertFalse(tracker.add(mid));
    assertEquals(tracker.size(), 1);

    assertTrue(tracker.remove(mid));
    assertTrue(tracker.isEmpty());
    assertEquals(tracker.size(), 0);

    timer.stop();
}
 
Example #18
Source File: FlinkPulsarITest.java    From pulsar-flink with Apache License 2.0 5 votes vote down vote up
public static boolean roughEquals(Map<String, MessageId> a, Map<String, MessageId> b) {
    for (Map.Entry<String, MessageId> aE : a.entrySet()) {
        MessageId bmid = b.getOrDefault(aE.getKey(), MessageId.latest);
        if (!ReaderThread.messageIdRoughEquals(bmid, aE.getValue())) {
            return false;
        }
    }
    return true;
}
 
Example #19
Source File: ConsumerImpl.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Override
public void negativeAcknowledge(MessageId messageId) {
    negativeAcksTracker.add(messageId);

    // Ensure the message is not redelivered for ack-timeout, since we did receive an "ack"
    unAckedMessageTracker.remove(messageId);
}
 
Example #20
Source File: RawBatchConverter.java    From pulsar with Apache License 2.0 5 votes vote down vote up
public static List<ImmutableTriple<MessageId, String, Integer>> extractIdsAndKeysAndSize(RawMessage msg)
        throws IOException {
    checkArgument(msg.getMessageIdData().getBatchIndex() == -1);

    ByteBuf payload = msg.getHeadersAndPayload();
    MessageMetadata metadata = Commands.parseMessageMetadata(payload);
    int batchSize = metadata.getNumMessagesInBatch();

    CompressionType compressionType = metadata.getCompression();
    CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(compressionType);
    int uncompressedSize = metadata.getUncompressedSize();
    ByteBuf uncompressedPayload = codec.decode(payload, uncompressedSize);
    metadata.recycle();

    List<ImmutableTriple<MessageId, String, Integer>> idsAndKeysAndSize = new ArrayList<>();

    for (int i = 0; i < batchSize; i++) {
        SingleMessageMetadata.Builder singleMessageMetadataBuilder = SingleMessageMetadata.newBuilder();
        ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(uncompressedPayload,
                                                                                singleMessageMetadataBuilder,
                                                                                0, batchSize);
        MessageId id = new BatchMessageIdImpl(msg.getMessageIdData().getLedgerId(),
                                              msg.getMessageIdData().getEntryId(),
                                              msg.getMessageIdData().getPartition(),
                                              i);
        if (!singleMessageMetadataBuilder.getCompactedOut()) {
            idsAndKeysAndSize.add(ImmutableTriple.of(id, singleMessageMetadataBuilder.getPartitionKey(), singleMessageMetadataBuilder.getPayloadSize()));
        }
        singleMessageMetadataBuilder.recycle();
        singleMessagePayload.release();
    }
    uncompressedPayload.release();
    return idsAndKeysAndSize;
}
 
Example #21
Source File: PulsarWriter.java    From singer with Apache License 2.0 5 votes vote down vote up
@Override
public void writeLogMessages(List<LogMessage> messages) throws LogStreamWriterException {
  long bytesWritten = 0;
  List<CompletableFuture<MessageId>> messsageFutures = new ArrayList<>();
  long maxPulsarWriteLatency = System.currentTimeMillis();
  for (LogMessage m : messages) {
    TypedMessageBuilder<byte[]> message = producer.newMessage();
    if (m.isSetKey()) {
      message.keyBytes(m.getKey());
      bytesWritten += m.getKey().length;
    }
    CompletableFuture<MessageId> sendAsync = message.value(m.getMessage()).sendAsync();
    messsageFutures.add(sendAsync);
    bytesWritten += m.getMessage().length;
  }
  try {
    producer.flush();
    for (CompletableFuture<MessageId> future : messsageFutures) {
      future.get();
    }
  } catch (PulsarClientException | InterruptedException | ExecutionException e) {
    OpenTsdbMetricConverter.incr(PULSAR_WRITE_FAILURE, messages.size(), "topic=" + metricTag,
        "host=" + HOSTNAME, "logname=" + logName);
    throw new LogStreamWriterException("Message delivery failed", e);
  }

  maxPulsarWriteLatency = System.currentTimeMillis() - maxPulsarWriteLatency;
  OpenTsdbMetricConverter.gauge(PULSAR_THROUGHPUT, bytesWritten, "topic=" + metricTag,
      "host=" + HOSTNAME, "logname=" + logName);
  OpenTsdbMetricConverter.gauge(PULSAR_LATENCY, maxPulsarWriteLatency, "topic=" + metricTag,
      "host=" + HOSTNAME, "logname=" + logName);
  OpenTsdbMetricConverter.incr(NUM_PULSAR_MESSAGES, messages.size(), "topic=" + metricTag,
      "host=" + HOSTNAME, "logname=" + logName);
  LOG.info("Completed batch writes to Pulsar topic:" + metricTag + " size:" + messages.size());

}
 
Example #22
Source File: PulsarMetadataReader.java    From pulsar-flink with Apache License 2.0 5 votes vote down vote up
public void resetCursor(String topic, MessageId messageId) {
    try {
        this.admin.topics().resetCursor(topic, subscriptionName, messageId);
    } catch (PulsarAdminException e) {
        throw new RuntimeException(e);
    }
}
 
Example #23
Source File: ProducerUnitTest.java    From tutorials with MIT License 5 votes vote down vote up
public static void main(String[] args) throws IOException {
    // Create a Pulsar client instance. A single instance can be shared across many
    // producers and consumer within the same application
    PulsarClient client = PulsarClient.builder()
            .serviceUrl(SERVICE_URL)
            .build();

    // Configure producer specific settings
    Producer<byte[]> producer = client.newProducer()
            // Set the topic
            .topic(TOPIC_NAME)
            // Enable compression
            .compressionType(CompressionType.LZ4)
            .create();

    // Once the producer is created, it can be used for the entire application life-cycle
    System.out.println("Created producer for the topic "+TOPIC_NAME);

    // Send 5 test messages
    IntStream.range(1, 5).forEach(i -> {
        String content = String.format("hi-pulsar-%d", i);

        // Build a message object
        Message<byte[]> msg = MessageBuilder.create()
                .setContent(content.getBytes())
                .build();

        // Send each message and log message content and ID when successfully received
        try {
            MessageId msgId = producer.send(msg);

            System.out.println("Published message '"+content+"' with the ID "+msgId);
        } catch (PulsarClientException e) {
            System.out.println(e.getMessage());
        }
    });

    client.close();
}
 
Example #24
Source File: TopicsImpl.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Override
public void resetCursor(String topic, String subName, MessageId messageId) throws PulsarAdminException {
    try {
        TopicName tn = validateTopic(topic);
        String encodedSubName = Codec.encode(subName);
        WebTarget path = topicPath(tn, "subscription", encodedSubName, "resetcursor");
        request(path).post(Entity.entity(messageId, MediaType.APPLICATION_JSON),
                        ErrorData.class);
    } catch (Exception e) {
        throw getApiException(e);
    }
}
 
Example #25
Source File: RawReaderTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Test
public void testBatchingRebatch() throws Exception {
    String topic = "persistent://my-property/my-ns/my-raw-topic";

    try (Producer<byte[]> producer = pulsarClient.newProducer().topic(topic)
        .maxPendingMessages(3)
        .enableBatching(true)
        .batchingMaxMessages(3)
        .batchingMaxPublishDelay(1, TimeUnit.HOURS)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create()) {
        producer.newMessage().key("key1").value("my-content-1".getBytes()).sendAsync();
        producer.newMessage().key("key2").value("my-content-2".getBytes()).sendAsync();
        producer.newMessage().key("key3").value("my-content-3".getBytes()).send();
    }

    RawReader reader = RawReader.create(pulsarClient, topic, subscription).get();
    try (RawMessage m1 = reader.readNextAsync().get()) {
        RawMessage m2 = RawBatchConverter.rebatchMessage(m1, (key, id) -> key.equals("key2")).get();
        List<ImmutableTriple<MessageId, String, Integer>> idsAndKeys = RawBatchConverter.extractIdsAndKeysAndSize(m2);
        Assert.assertEquals(idsAndKeys.size(), 1);
        Assert.assertEquals(idsAndKeys.get(0).getMiddle(), "key2");
        m2.close();
        Assert.assertEquals(m1.getHeadersAndPayload().refCnt(), 1);
    } finally {
        reader.closeAsync().get();
    }
}
 
Example #26
Source File: PulsarTestBase.java    From pulsar-flink with Apache License 2.0 5 votes vote down vote up
public static <T> List<MessageId> sendTypedMessages(
        String topic,
        SchemaType type,
        List<T> messages,
        Optional<Integer> partition) throws PulsarClientException {

    return sendTypedMessages(topic, type, messages, partition, null);
}
 
Example #27
Source File: TypedMessageBuilderImpl.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Override
public CompletableFuture<MessageId> sendAsync() {
    long sequenceId = beforeSend();
    CompletableFuture<MessageId> sendFuture = producer.internalSendAsync(getMessage());
    if (txn != null) {
        // it is okay that we register produced topic after sending the messages. because
        // the transactional messages will not be visible for consumers until the transaction
        // is committed.
        txn.registerProducedTopic(producer.getTopic());
        // register the sendFuture as part of the transaction
        return txn.registerSendOp(sequenceId, sendFuture);
    } else {
        return sendFuture;
    }
}
 
Example #28
Source File: SubscriptionSeekTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Test
public void testSeekOnPartitionedTopic() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/testSeekPartitions";

    admin.topics().createPartitionedTopic(topicName, 2);
    org.apache.pulsar.client.api.Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName)
            .subscriptionName("my-subscription").subscribe();

    try {
        consumer.seek(MessageId.latest);
        fail("Should not have succeeded");
    } catch (PulsarClientException e) {
        // Expected
    }
}
 
Example #29
Source File: FunctionAssignmentTailer.java    From pulsar with Apache License 2.0 5 votes vote down vote up
public void startFromMessage(MessageId startMessageId) throws PulsarClientException {
    if (!isRunning) {
        isRunning = true;
        if (reader == null) {
            reader = createReader(startMessageId);
        }
        if (tailerThread == null || !tailerThread.isAlive()) {
            tailerThread = getTailerThread();
        }
        tailerThread.start();
    }
}
 
Example #30
Source File: FunctionRuntimeManager.java    From pulsar with Apache License 2.0 5 votes vote down vote up
/**
 * Initializes the FunctionRuntimeManager.  Does the following:
 * 1. Consume all existing assignments to establish existing/latest set of assignments
 * 2. After current assignments are read, assignments belonging to this worker will be processed
 *
 * @return the message id of the message processed during init phase
 */
public MessageId initialize() {
    try {
        Reader<byte[]> reader = WorkerUtils.createReader(
                workerService.getClient().newReader(),
                workerConfig.getWorkerId() + "-function-assignment-initialize",
                workerConfig.getFunctionAssignmentTopic(),
                MessageId.earliest);

        // start init phase
        this.isInitializePhase = true;
        // keep track of the last message read
        MessageId lastMessageRead = MessageId.earliest;
        // read all existing messages
        while (reader.hasMessageAvailable()) {
            Message<byte[]> message = reader.readNext();
            lastMessageRead = message.getMessageId();
            processAssignmentMessage(message);
        }
        // init phase is done
        this.isInitializePhase = false;
        // close reader
        reader.close();
        // realize existing assignments
        Map<String, Assignment> assignmentMap = workerIdToAssignments.get(this.workerConfig.getWorkerId());
        if (assignmentMap != null) {
            for (Assignment assignment : assignmentMap.values()) {
                if (needsStart(assignment)) {
                    startFunctionInstance(assignment);
                }
            }
        }
        // complete future to indicate initialization is complete
        isInitialized.complete(null);
        return lastMessageRead;
    } catch (Exception e) {
        log.error("Failed to initialize function runtime manager: {}", e.getMessage(), e);
        throw new RuntimeException(e);
    }
}