Java Code Examples for org.apache.pulsar.common.naming.TopicName#getNamespace()

The following examples show how to use org.apache.pulsar.common.naming.TopicName#getNamespace() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: BacklogQuotaManager.java From pulsar with Apache License 2.0

6 votes

/**
 * Handle exceeded backlog by using policies set in the zookeeper for given topic
 *
 * @param persistentTopic
 *            Topic on which backlog has been exceeded
 */
public void handleExceededBacklogQuota(PersistentTopic persistentTopic) {
    TopicName topicName = TopicName.get(persistentTopic.getName());
    String namespace = topicName.getNamespace();
    String policyPath = AdminResource.path(POLICIES, namespace);

    BacklogQuota quota = getBacklogQuota(namespace, policyPath);
    log.info("Backlog quota exceeded for topic [{}]. Applying [{}] policy", persistentTopic.getName(),
            quota.getPolicy());
    switch (quota.getPolicy()) {
    case consumer_backlog_eviction:
        dropBacklog(persistentTopic, quota);
        break;
    case producer_exception:
    case producer_request_hold:
        disconnectProducers(persistentTopic);
        break;
    default:
        break;
    }
}

Example 2

Source File: TestPulsarSplitManager.java From pulsar with Apache License 2.0

6 votes

@Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true)
public void testGetSplitNonSchema(String delimiter) throws Exception {
    updateRewriteNamespaceDelimiterIfNeeded(delimiter);
    TopicName topicName = NON_SCHEMA_TOPIC;
    setup();
    log.info("!----- topic: %s -----!", topicName);
    PulsarTableHandle pulsarTableHandle = new PulsarTableHandle(pulsarConnectorId.toString(),
        topicName.getNamespace(),
        topicName.getLocalName(),
        topicName.getLocalName());

    Map<ColumnHandle, Domain> domainMap = new HashMap<>();
    TupleDomain<ColumnHandle> tupleDomain = TupleDomain.withColumnDomains(domainMap);

    PulsarTableLayoutHandle pulsarTableLayoutHandle = new PulsarTableLayoutHandle(pulsarTableHandle, tupleDomain);
    ConnectorSplitSource connectorSplitSource = this.pulsarSplitManager.getSplits(
        mock(ConnectorTransactionHandle.class), mock(ConnectorSession.class),
        pulsarTableLayoutHandle, null);
    assertNotNull(connectorSplitSource);
}

Example 3

Source File: PersistentTopic.java From pulsar with Apache License 2.0

5 votes

/**
 *
 * @return Backlog quota for topic
 */
@Override
public BacklogQuota getBacklogQuota() {
    TopicName topicName = TopicName.get(this.getName());
    String namespace = topicName.getNamespace();
    String policyPath = AdminResource.path(POLICIES, namespace);

    BacklogQuota backlogQuota = brokerService.getBacklogQuotaManager().getBacklogQuota(namespace, policyPath);
    return backlogQuota;
}

Example 4

Source File: BookieClientStatsGenerator.java From pulsar with Apache License 2.0

5 votes

private void put(TopicName topicName, PendingBookieOpsStats bookieOpsStats) {
    String namespace = topicName.getNamespace();
    if (!nsBookieClientStatsMap.containsKey(namespace)) {
        Map<String, PendingBookieOpsStats> destBookieClientStatsMap = Maps.newTreeMap();
        destBookieClientStatsMap.put(topicName.toString(), bookieOpsStats);
        nsBookieClientStatsMap.put(namespace, destBookieClientStatsMap);
    } else {
        nsBookieClientStatsMap.get(namespace).put(topicName.toString(), bookieOpsStats);
    }

}

Example 5

Source File: ReplicatorTestBase.java From pulsar with Apache License 2.0

5 votes

MessageProducer(URL url, final TopicName dest) throws Exception {
    this.url = url;
    this.namespace = dest.getNamespace();
    this.topicName = dest.toString();
    client = PulsarClient.builder().serviceUrl(url.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    producer = client.newProducer()
        .topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();

}

Example 6

Source File: ReplicatorTestBase.java From pulsar with Apache License 2.0

5 votes

MessageProducer(URL url, final TopicName dest, boolean batch) throws Exception {
    this.url = url;
    this.namespace = dest.getNamespace();
    this.topicName = dest.toString();
    client = PulsarClient.builder().serviceUrl(url.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    ProducerBuilder<byte[]> producerBuilder = client.newProducer()
        .topic(topicName)
        .enableBatching(batch)
        .batchingMaxPublishDelay(1, TimeUnit.SECONDS)
        .batchingMaxMessages(5);
    producer = producerBuilder.create();

}

Example 7

Source File: ReplicatorTestBase.java From pulsar with Apache License 2.0

5 votes

MessageConsumer(URL url, final TopicName dest, String subId) throws Exception {
    this.url = url;
    this.namespace = dest.getNamespace();
    this.topicName = dest.toString();

    client = PulsarClient.builder().serviceUrl(url.toString()).statsInterval(0, TimeUnit.SECONDS).build();

    try {
        consumer = client.newConsumer().topic(topicName).subscriptionName(subId).subscribe();
    } catch (Exception e) {
        client.close();
        throw e;
    }
}

Example 8

Source File: ConsumerBuilderImpl.java From pulsar with Apache License 2.0

5 votes

@Override
public CompletableFuture<Consumer<T>> subscribeAsync() {
    if (conf.getTopicNames().isEmpty() && conf.getTopicsPattern() == null) {
        return FutureUtil
                .failedFuture(new InvalidConfigurationException("Topic name must be set on the consumer builder"));
    }

    if (StringUtils.isBlank(conf.getSubscriptionName())) {
        return FutureUtil.failedFuture(
                new InvalidConfigurationException("Subscription name must be set on the consumer builder"));
    }

    if (conf.getKeySharedPolicy() != null && conf.getSubscriptionType() != SubscriptionType.Key_Shared) {
        return FutureUtil.failedFuture(
                new InvalidConfigurationException("KeySharedPolicy must set with KeyShared subscription"));
    }
    if(conf.isRetryEnable() && conf.getTopicNames().size() > 0 ) {
        TopicName topicFirst = TopicName.get(conf.getTopicNames().iterator().next());
        String retryLetterTopic = topicFirst.getNamespace() + "/" + conf.getSubscriptionName() + RetryMessageUtil.RETRY_GROUP_TOPIC_SUFFIX;
        String deadLetterTopic = topicFirst.getNamespace() + "/" + conf.getSubscriptionName() + RetryMessageUtil.DLQ_GROUP_TOPIC_SUFFIX;
        if(conf.getDeadLetterPolicy() == null) {
            conf.setDeadLetterPolicy(DeadLetterPolicy.builder()
                                    .maxRedeliverCount(RetryMessageUtil.MAX_RECONSUMETIMES)
                                    .retryLetterTopic(retryLetterTopic)
                                    .deadLetterTopic(deadLetterTopic)
                                    .build());
        } else {
            if (StringUtils.isBlank(conf.getDeadLetterPolicy().getRetryLetterTopic())) {
                conf.getDeadLetterPolicy().setRetryLetterTopic(retryLetterTopic);
            }
            if (StringUtils.isBlank(conf.getDeadLetterPolicy().getDeadLetterTopic())) {
                conf.getDeadLetterPolicy().setDeadLetterTopic(deadLetterTopic);
            }
        }
        conf.getTopicNames().add(conf.getDeadLetterPolicy().getRetryLetterTopic());
    }
    return interceptorList == null || interceptorList.size() == 0 ?
            client.subscribeAsync(conf, schema, null) :
            client.subscribeAsync(conf, schema, new ConsumerInterceptors<>(interceptorList));
}

Example 9

Source File: TestPulsarSplitManager.java From pulsar with Apache License 2.0

4 votes

@Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true)
public void testPartitionedTopic(String delimiter) throws Exception {
    updateRewriteNamespaceDelimiterIfNeeded(delimiter);
    for (TopicName topicName : partitionedTopicNames) {
        setup();
        log.info("!----- topic: %s -----!", topicName);
        PulsarTableHandle pulsarTableHandle = new PulsarTableHandle(pulsarConnectorId.toString(),
                topicName.getNamespace(),
                topicName.getLocalName(),
                topicName.getLocalName());
        PulsarTableLayoutHandle pulsarTableLayoutHandle = new PulsarTableLayoutHandle(pulsarTableHandle, TupleDomain.all());

        final ResultCaptor<Collection<PulsarSplit>> resultCaptor = new ResultCaptor<>();
        doAnswer(resultCaptor).when(this.pulsarSplitManager).getSplitsPartitionedTopic(anyInt(), any(), any(), any(), any(), any());

        this.pulsarSplitManager.getSplits(mock(ConnectorTransactionHandle.class), mock(ConnectorSession.class),
                pulsarTableLayoutHandle, null);

        verify(this.pulsarSplitManager, times(1))
                .getSplitsPartitionedTopic(anyInt(), any(), any(), any(), any(), any());

        int partitions = partitionedTopicsToPartitions.get(topicName.toString());

        for (int i = 0; i < partitions; i++) {
            List<PulsarSplit> splits = getSplitsForPartition(topicName.getPartition(i), resultCaptor.getResult());
            int totalSize = 0;
            for (PulsarSplit pulsarSplit : splits) {
                assertEquals(pulsarSplit.getConnectorId(), pulsarConnectorId.toString());
                assertEquals(pulsarSplit.getSchemaName(), topicName.getNamespace());
                assertEquals(pulsarSplit.getTableName(), topicName.getPartition(i).getLocalName());
                assertEquals(pulsarSplit.getSchema(),
                        new String(topicsToSchemas.get(topicName.getSchemaName()).getSchema()));
                assertEquals(pulsarSplit.getSchemaType(), topicsToSchemas.get(topicName.getSchemaName()).getType());
                assertEquals(pulsarSplit.getStartPositionEntryId(), totalSize);
                assertEquals(pulsarSplit.getStartPositionLedgerId(), 0);
                assertEquals(pulsarSplit.getStartPosition(), PositionImpl.get(0, totalSize));
                assertEquals(pulsarSplit.getEndPositionLedgerId(), 0);
                assertEquals(pulsarSplit.getEndPositionEntryId(), totalSize + pulsarSplit.getSplitSize());
                assertEquals(pulsarSplit.getEndPosition(), PositionImpl.get(0, totalSize + pulsarSplit.getSplitSize()));

                totalSize += pulsarSplit.getSplitSize();
            }

            assertEquals(totalSize, topicsToNumEntries.get(topicName.getSchemaName()).intValue());
        }

        cleanup();
    }
}

Example 10

Source File: TestPulsarSplitManager.java From pulsar with Apache License 2.0

4 votes

@Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true)
public void testPublishTimePredicatePushdown(String delimiter) throws Exception {
    updateRewriteNamespaceDelimiterIfNeeded(delimiter);
    TopicName topicName = TOPIC_1;

    setup();
    log.info("!----- topic: %s -----!", topicName);
    PulsarTableHandle pulsarTableHandle = new PulsarTableHandle(pulsarConnectorId.toString(),
            topicName.getNamespace(),
            topicName.getLocalName(),
            topicName.getLocalName());


    Map<ColumnHandle, Domain> domainMap = new HashMap<>();
    Domain domain = Domain.create(ValueSet.ofRanges(Range.range(TIMESTAMP, currentTimeMs + 1L, true,
            currentTimeMs + 50L, true)), false);
    domainMap.put(PulsarInternalColumn.PUBLISH_TIME.getColumnHandle(pulsarConnectorId.toString(), false), domain);
    TupleDomain<ColumnHandle> tupleDomain = TupleDomain.withColumnDomains(domainMap);

    PulsarTableLayoutHandle pulsarTableLayoutHandle = new PulsarTableLayoutHandle(pulsarTableHandle, tupleDomain);

    final ResultCaptor<Collection<PulsarSplit>> resultCaptor = new ResultCaptor<>();
    doAnswer(resultCaptor).when(this.pulsarSplitManager)
            .getSplitsNonPartitionedTopic(anyInt(), any(), any(), any(), any(), any());

    ConnectorSplitSource connectorSplitSource = this.pulsarSplitManager.getSplits(
            mock(ConnectorTransactionHandle.class), mock(ConnectorSession.class),
            pulsarTableLayoutHandle, null);


    verify(this.pulsarSplitManager, times(1))
            .getSplitsNonPartitionedTopic(anyInt(), any(), any(), any(), any(), any());

    int totalSize = 0;
    int initalStart = 1;
    for (PulsarSplit pulsarSplit : resultCaptor.getResult()) {
        assertEquals(pulsarSplit.getConnectorId(), pulsarConnectorId.toString());
        assertEquals(pulsarSplit.getSchemaName(), topicName.getNamespace());
        assertEquals(pulsarSplit.getTableName(), topicName.getLocalName());
        assertEquals(pulsarSplit.getSchema(),
                new String(topicsToSchemas.get(topicName.getSchemaName()).getSchema()));
        assertEquals(pulsarSplit.getSchemaType(), topicsToSchemas.get(topicName.getSchemaName()).getType());
        assertEquals(pulsarSplit.getStartPositionEntryId(), initalStart);
        assertEquals(pulsarSplit.getStartPositionLedgerId(), 0);
        assertEquals(pulsarSplit.getStartPosition(), PositionImpl.get(0, initalStart));
        assertEquals(pulsarSplit.getEndPositionLedgerId(), 0);
        assertEquals(pulsarSplit.getEndPositionEntryId(), initalStart + pulsarSplit.getSplitSize());
        assertEquals(pulsarSplit.getEndPosition(), PositionImpl.get(0, initalStart + pulsarSplit
                .getSplitSize()));

        initalStart += pulsarSplit.getSplitSize();
        totalSize += pulsarSplit.getSplitSize();
    }
    assertEquals(totalSize, 49);

}

Example 11

Source File: TestPulsarSplitManager.java From pulsar with Apache License 2.0

4 votes

@Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true)
public void testPublishTimePredicatePushdownPartitionedTopic(String delimiter) throws Exception {
    updateRewriteNamespaceDelimiterIfNeeded(delimiter);
    TopicName topicName = PARTITIONED_TOPIC_1;

    setup();
    log.info("!----- topic: %s -----!", topicName);
    PulsarTableHandle pulsarTableHandle = new PulsarTableHandle(pulsarConnectorId.toString(),
            topicName.getNamespace(),
            topicName.getLocalName(),
            topicName.getLocalName());


    Map<ColumnHandle, Domain> domainMap = new HashMap<>();
    Domain domain = Domain.create(ValueSet.ofRanges(Range.range(TIMESTAMP, currentTimeMs + 1L, true,
            currentTimeMs + 50L, true)), false);
    domainMap.put(PulsarInternalColumn.PUBLISH_TIME.getColumnHandle(pulsarConnectorId.toString(), false), domain);
    TupleDomain<ColumnHandle> tupleDomain = TupleDomain.withColumnDomains(domainMap);

    PulsarTableLayoutHandle pulsarTableLayoutHandle = new PulsarTableLayoutHandle(pulsarTableHandle, tupleDomain);

    final ResultCaptor<Collection<PulsarSplit>> resultCaptor = new ResultCaptor<>();
    doAnswer(resultCaptor).when(this.pulsarSplitManager)
            .getSplitsPartitionedTopic(anyInt(), any(), any(), any(), any(), any());

    ConnectorSplitSource connectorSplitSource = this.pulsarSplitManager.getSplits(
            mock(ConnectorTransactionHandle.class), mock(ConnectorSession.class),
            pulsarTableLayoutHandle, null);


    verify(this.pulsarSplitManager, times(1))
            .getSplitsPartitionedTopic(anyInt(), any(), any(), any(), any(), any());


    int partitions = partitionedTopicsToPartitions.get(topicName.toString());
    for (int i = 0; i < partitions; i++) {
        List<PulsarSplit> splits = getSplitsForPartition(topicName.getPartition(i), resultCaptor.getResult());
        int totalSize = 0;
        int initialStart = 1;
        for (PulsarSplit pulsarSplit : splits) {
            assertEquals(pulsarSplit.getConnectorId(), pulsarConnectorId.toString());
            assertEquals(pulsarSplit.getSchemaName(), topicName.getNamespace());
            assertEquals(pulsarSplit.getTableName(), topicName.getPartition(i).getLocalName());
            assertEquals(pulsarSplit.getSchema(),
                    new String(topicsToSchemas.get(topicName.getSchemaName()).getSchema()));
            assertEquals(pulsarSplit.getSchemaType(), topicsToSchemas.get(topicName.getSchemaName()).getType());
            assertEquals(pulsarSplit.getStartPositionEntryId(), initialStart);
            assertEquals(pulsarSplit.getStartPositionLedgerId(), 0);
            assertEquals(pulsarSplit.getStartPosition(), PositionImpl.get(0, initialStart));
            assertEquals(pulsarSplit.getEndPositionLedgerId(), 0);
            assertEquals(pulsarSplit.getEndPositionEntryId(), initialStart + pulsarSplit.getSplitSize());
            assertEquals(pulsarSplit.getEndPosition(), PositionImpl.get(0, initialStart + pulsarSplit.getSplitSize()));

            initialStart += pulsarSplit.getSplitSize();
            totalSize += pulsarSplit.getSplitSize();
        }

        assertEquals(totalSize, 49);
    }
}

Example 12

Source File: TestPulsarMetadata.java From pulsar with Apache License 2.0

4 votes

@Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true)
public void testGetTableMetadata(String delimiter) {
    updateRewriteNamespaceDelimiterIfNeeded(delimiter);
    List<TopicName> allTopics = new LinkedList<>();
    allTopics.addAll(topicNames.stream().filter(topicName -> !topicName.equals(NON_SCHEMA_TOPIC)).collect(Collectors.toList()));
    allTopics.addAll(partitionedTopicNames);

    for (TopicName topic : allTopics) {
        PulsarTableHandle pulsarTableHandle = new PulsarTableHandle(
                topic.toString(),
                topic.getNamespace(),
                topic.getLocalName(),
                topic.getLocalName()
        );

        ConnectorTableMetadata tableMetadata = this.pulsarMetadata.getTableMetadata(mock(ConnectorSession.class),
                pulsarTableHandle);

        assertEquals(tableMetadata.getTable().getSchemaName(), topic.getNamespace());
        assertEquals(tableMetadata.getTable().getTableName(), topic.getLocalName());
        assertEquals(tableMetadata.getColumns().size(),
                fooColumnHandles.size());

        List<String> fieldNames = new LinkedList<>(fooFieldNames.keySet());

        for (PulsarInternalColumn internalField : PulsarInternalColumn.getInternalFields()) {
            fieldNames.add(internalField.getName());
        }

        for (ColumnMetadata column : tableMetadata.getColumns()) {
            if (PulsarInternalColumn.getInternalFieldsMap().containsKey(column.getName())) {
                assertEquals(column.getComment(),
                        PulsarInternalColumn.getInternalFieldsMap()
                                .get(column.getName()).getColumnMetadata(true).getComment());
            }

            fieldNames.remove(column.getName());
        }

        assertTrue(fieldNames.isEmpty());
    }
}