Java Code Examples for org.apache.pulsar.common.api.proto.PulsarApi#KeySharedMeta

The following examples show how to use org.apache.pulsar.common.api.proto.PulsarApi#KeySharedMeta . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: HashRangeExclusiveStickyKeyConsumerSelector.java    From pulsar with Apache License 2.0 5 votes vote down vote up 
private void validateKeySharedMeta(Consumer consumer) throws BrokerServiceException.ConsumerAssignException {
    if (consumer.getKeySharedMeta() == null) {
        throw new BrokerServiceException.ConsumerAssignException("Must specify key shared meta for consumer.");
    }
    List<PulsarApi.IntRange> ranges = consumer.getKeySharedMeta().getHashRangesList();
    if (ranges.isEmpty()) {
        throw new BrokerServiceException.ConsumerAssignException("Ranges for KeyShared policy must not be empty.");
    }
    for (PulsarApi.IntRange intRange : ranges) {

        if (intRange.getStart() > intRange.getEnd()) {
            throw new BrokerServiceException.ConsumerAssignException("Fixed hash range start > end");
        }

        Map.Entry<Integer, Consumer> ceilingEntry = rangeMap.ceilingEntry(intRange.getStart());
        Map.Entry<Integer, Consumer> floorEntry = rangeMap.floorEntry(intRange.getEnd());

        if (floorEntry != null && floorEntry.getKey() >= intRange.getStart()) {
            throw new BrokerServiceException.ConsumerAssignException("Range conflict with consumer " + floorEntry.getValue());
        }

        if (ceilingEntry != null && ceilingEntry.getKey() <= intRange.getEnd()) {
            throw new BrokerServiceException.ConsumerAssignException("Range conflict with consumer " + ceilingEntry.getValue());
        }

        if (ceilingEntry != null && floorEntry != null && ceilingEntry.getValue().equals(floorEntry.getValue())) {
            PulsarApi.KeySharedMeta keySharedMeta = ceilingEntry.getValue().getKeySharedMeta();
            for (PulsarApi.IntRange range : keySharedMeta.getHashRangesList()) {
                int start = Math.max(intRange.getStart(), range.getStart());
                int end = Math.min(intRange.getEnd(), range.getEnd());
                if (end >= start) {
                    throw new BrokerServiceException.ConsumerAssignException("Range conflict with consumer " + ceilingEntry.getValue());
                }
            }
        }
    }
}
Example 2
Source File: HashRangeExclusiveStickyKeyConsumerSelectorTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up 
@Test(expectedExceptions = BrokerServiceException.ConsumerAssignException.class)
public void testEmptyRanges() throws BrokerServiceException.ConsumerAssignException {
    HashRangeExclusiveStickyKeyConsumerSelector selector = new HashRangeExclusiveStickyKeyConsumerSelector(10);
    Consumer consumer = mock(Consumer.class);
    PulsarApi.KeySharedMeta keySharedMeta = PulsarApi.KeySharedMeta.newBuilder()
            .setKeySharedMode(PulsarApi.KeySharedMode.STICKY)
            .build();
    when(consumer.getKeySharedMeta()).thenReturn(keySharedMeta);
    selector.addConsumer(consumer);
}
Example 3
Source File: HashRangeExclusiveStickyKeyConsumerSelectorTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSingleRangeConflict() throws BrokerServiceException.ConsumerAssignException {
    HashRangeExclusiveStickyKeyConsumerSelector selector = new HashRangeExclusiveStickyKeyConsumerSelector(10);
    Consumer consumer1 = mock(Consumer.class);
    PulsarApi.KeySharedMeta keySharedMeta1 = PulsarApi.KeySharedMeta.newBuilder()
            .setKeySharedMode(PulsarApi.KeySharedMode.STICKY)
            .addHashRanges(PulsarApi.IntRange.newBuilder().setStart(2).setEnd(5).build())
            .build();
    when(consumer1.getKeySharedMeta()).thenReturn(keySharedMeta1);
    Assert.assertEquals(consumer1.getKeySharedMeta(), keySharedMeta1);
    selector.addConsumer(consumer1);
    Assert.assertEquals(selector.getRangeConsumer().size(),2);

    final List<PulsarApi.IntRange> testRanges = new ArrayList<>();
    testRanges.add(PulsarApi.IntRange.newBuilder().setStart(4).setEnd(6).build());
    testRanges.add(PulsarApi.IntRange.newBuilder().setStart(1).setEnd(3).build());
    testRanges.add(PulsarApi.IntRange.newBuilder().setStart(2).setEnd(2).build());
    testRanges.add(PulsarApi.IntRange.newBuilder().setStart(5).setEnd(5).build());
    testRanges.add(PulsarApi.IntRange.newBuilder().setStart(1).setEnd(5).build());
    testRanges.add(PulsarApi.IntRange.newBuilder().setStart(2).setEnd(6).build());
    testRanges.add(PulsarApi.IntRange.newBuilder().setStart(2).setEnd(5).build());
    testRanges.add(PulsarApi.IntRange.newBuilder().setStart(1).setEnd(6).build());
    testRanges.add(PulsarApi.IntRange.newBuilder().setStart(8).setEnd(6).build());

    for (PulsarApi.IntRange testRange : testRanges) {
        Consumer consumer = mock(Consumer.class);
        PulsarApi.KeySharedMeta keySharedMeta = PulsarApi.KeySharedMeta.newBuilder()
                .setKeySharedMode(PulsarApi.KeySharedMode.STICKY)
                .addHashRanges(testRange)
                .build();
        when(consumer.getKeySharedMeta()).thenReturn(keySharedMeta);
        Assert.assertEquals(consumer.getKeySharedMeta(), keySharedMeta);
        try {
            selector.addConsumer(consumer);
            Assert.fail("should be failed");
        } catch (BrokerServiceException.ConsumerAssignException ignore) {
        }
        Assert.assertEquals(selector.getRangeConsumer().size(),2);
    }
}
Example 4
Source File: HashRangeExclusiveStickyKeyConsumerSelectorTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up 
@Test
public void testMultipleRangeConflict() throws BrokerServiceException.ConsumerAssignException {
    HashRangeExclusiveStickyKeyConsumerSelector selector = new HashRangeExclusiveStickyKeyConsumerSelector(10);
    Consumer consumer1 = mock(Consumer.class);
    PulsarApi.KeySharedMeta keySharedMeta1 = PulsarApi.KeySharedMeta.newBuilder()
            .setKeySharedMode(PulsarApi.KeySharedMode.STICKY)
            .addHashRanges(PulsarApi.IntRange.newBuilder().setStart(2).setEnd(5).build())
            .build();
    when(consumer1.getKeySharedMeta()).thenReturn(keySharedMeta1);
    Assert.assertEquals(consumer1.getKeySharedMeta(), keySharedMeta1);
    selector.addConsumer(consumer1);
    Assert.assertEquals(selector.getRangeConsumer().size(),2);

    final List<List<PulsarApi.IntRange>> testRanges = new ArrayList<>();
    testRanges.add(Lists.newArrayList(
            PulsarApi.IntRange.newBuilder().setStart(2).setEnd(2).build(),
            PulsarApi.IntRange.newBuilder().setStart(3).setEnd(3).build(),
            PulsarApi.IntRange.newBuilder().setStart(4).setEnd(5).build())
    );
    testRanges.add(Lists.newArrayList(
            PulsarApi.IntRange.newBuilder().setStart(0).setEnd(0).build(),
            PulsarApi.IntRange.newBuilder().setStart(1).setEnd(2).build())
    );

    for (List<PulsarApi.IntRange> testRange : testRanges) {
        Consumer consumer = mock(Consumer.class);
        PulsarApi.KeySharedMeta keySharedMeta = PulsarApi.KeySharedMeta.newBuilder()
                .setKeySharedMode(PulsarApi.KeySharedMode.STICKY)
                .addAllHashRanges(testRange)
                .build();
        when(consumer.getKeySharedMeta()).thenReturn(keySharedMeta);
        Assert.assertEquals(consumer.getKeySharedMeta(), keySharedMeta);
        try {
            selector.addConsumer(consumer);
            Assert.fail("should be failed");
        } catch (BrokerServiceException.ConsumerAssignException ignore) {
        }
        Assert.assertEquals(selector.getRangeConsumer().size(),2);
    }
}
Example 5
Source File: Topic.java    From pulsar with Apache License 2.0 4 votes vote down vote up 
CompletableFuture<Consumer> subscribe(ServerCnx cnx, String subscriptionName, long consumerId, SubType subType,
int priorityLevel, String consumerName, boolean isDurable, MessageId startMessageId,
Map<String, String> metadata, boolean readCompacted, InitialPosition initialPosition,
long startMessageRollbackDurationSec, boolean replicateSubscriptionState, PulsarApi.KeySharedMeta keySharedMeta);
Example 6
Source File: Consumer.java    From pulsar with Apache License 2.0 4 votes vote down vote up 
public Consumer(Subscription subscription, SubType subType, String topicName, long consumerId,
                int priorityLevel, String consumerName,
                int maxUnackedMessages, ServerCnx cnx, String appId,
                Map<String, String> metadata, boolean readCompacted, InitialPosition subscriptionInitialPosition,
                PulsarApi.KeySharedMeta keySharedMeta) throws BrokerServiceException {

    this.subscription = subscription;
    this.subType = subType;
    this.topicName = topicName;
    this.partitionIdx = TopicName.getPartitionIndex(topicName);
    this.consumerId = consumerId;
    this.priorityLevel = priorityLevel;
    this.readCompacted = readCompacted;
    this.consumerName = consumerName;
    this.maxUnackedMessages = maxUnackedMessages;
    this.subscriptionInitialPosition = subscriptionInitialPosition;
    this.keySharedMeta = keySharedMeta;
    this.cnx = cnx;
    this.msgOut = new Rate();
    this.chuckedMessageRate = new Rate();
    this.msgRedeliver = new Rate();
    this.bytesOutCounter = new LongAdder();
    this.msgOutCounter = new LongAdder();
    this.appId = appId;
    this.authenticationData = cnx.authenticationData;
    this.preciseDispatcherFlowControl = cnx.isPreciseDispatcherFlowControl();

    PERMITS_RECEIVED_WHILE_CONSUMER_BLOCKED_UPDATER.set(this, 0);
    MESSAGE_PERMITS_UPDATER.set(this, 0);
    UNACKED_MESSAGES_UPDATER.set(this, 0);
    AVG_MESSAGES_PER_ENTRY.set(this, 1000);

    this.metadata = metadata != null ? metadata : Collections.emptyMap();

    stats = new ConsumerStats();
    stats.setAddress(cnx.clientAddress().toString());
    stats.consumerName = consumerName;
    stats.setConnectedSince(DateFormatter.now());
    stats.setClientVersion(cnx.getClientVersion());
    stats.metadata = this.metadata;

    if (Subscription.isIndividualAckMode(subType)) {
        this.pendingAcks = new ConcurrentLongLongPairHashMap(256, 1);
    } else {
        // We don't need to keep track of pending acks if the subscription is not shared
        this.pendingAcks = null;
    }
}
Example 7
Source File: Consumer.java    From pulsar with Apache License 2.0 4 votes vote down vote up 
public PulsarApi.KeySharedMeta getKeySharedMeta() {
    return keySharedMeta;
}
Example 8
Source File: HashRangeExclusiveStickyKeyConsumerSelectorTest.java    From pulsar with Apache License 2.0 4 votes vote down vote up 
@Test
public void testConsumerSelect() throws BrokerServiceException.ConsumerAssignException {

    HashRangeExclusiveStickyKeyConsumerSelector selector = new HashRangeExclusiveStickyKeyConsumerSelector(10);
    Consumer consumer1 = mock(Consumer.class);
    PulsarApi.KeySharedMeta keySharedMeta1 = PulsarApi.KeySharedMeta.newBuilder()
            .setKeySharedMode(PulsarApi.KeySharedMode.STICKY)
            .addHashRanges(PulsarApi.IntRange.newBuilder().setStart(0).setEnd(2).build())
            .build();
    when(consumer1.getKeySharedMeta()).thenReturn(keySharedMeta1);
    Assert.assertEquals(consumer1.getKeySharedMeta(), keySharedMeta1);
    selector.addConsumer(consumer1);
    Assert.assertEquals(selector.getRangeConsumer().size(),2);
    Consumer selectedConsumer;
    for (int i = 0; i < 3; i++) {
        selectedConsumer = selector.select(i);
        Assert.assertEquals(selectedConsumer, consumer1);
    }
    selectedConsumer = selector.select(4);
    Assert.assertNull(selectedConsumer);

    Consumer consumer2 = mock(Consumer.class);
    PulsarApi.KeySharedMeta keySharedMeta2 = PulsarApi.KeySharedMeta.newBuilder()
            .setKeySharedMode(PulsarApi.KeySharedMode.STICKY)
            .addHashRanges(PulsarApi.IntRange.newBuilder().setStart(3).setEnd(9).build())
            .build();
    when(consumer2.getKeySharedMeta()).thenReturn(keySharedMeta2);
    Assert.assertEquals(consumer2.getKeySharedMeta(), keySharedMeta2);
    selector.addConsumer(consumer2);
    Assert.assertEquals(selector.getRangeConsumer().size(),4);

    for (int i = 3; i < 10; i++) {
        selectedConsumer = selector.select(i);
        Assert.assertEquals(selectedConsumer, consumer2);
    }

    for (int i = 0; i < 3; i++) {
        selectedConsumer = selector.select(i);
        Assert.assertEquals(selectedConsumer, consumer1);
    }

    selector.removeConsumer(consumer1);
    Assert.assertEquals(selector.getRangeConsumer().size(),2);
    selectedConsumer = selector.select(1);
    Assert.assertNull(selectedConsumer);

    selector.removeConsumer(consumer2);
    Assert.assertEquals(selector.getRangeConsumer().size(),0);
    selectedConsumer = selector.select(5);
    Assert.assertNull(selectedConsumer);
}