Java Code Examples for org.apache.kafka.clients.consumer.ConsumerRebalanceListener

The following examples show how to use org.apache.kafka.clients.consumer.ConsumerRebalanceListener. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
Source Project: kbear   Source File: ConsumerProxy.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void subscribe(Collection<String> topics, ConsumerRebalanceListener callback) {
    ObjectExtension.requireNonNull(topics, "topics");
    ObjectExtension.requireNonNull(callback, "callback");

    runWithoutConcurrency(() -> {
        synchronized (_addRemoveLock) {
            removeNotUsed(topics);
            topics.forEach(t -> {
                ConsumerHolder consumerHolder = getOrAddConsumer(t);
                consumerHolder.getConsumer().subscribe(Arrays.asList(t), callback);
                consumerHolder.setConsumerRebalanceListener(callback);
                consumerHolder.setAssignments(null);
            });
        }
    });
}
 
Example 2
@Test
public void shouldUnsubscribeIfKafkaConsumerIsAlreadySubscribed() throws Exception {
    // given
    when(kafkaConsumer.subscription()).thenReturn(newSet(KAFKA_TOPIC));

    // when
    endpoint.consumeUntil(fromHorizon(), shutdown());

    // then
    verify(kafkaConsumer).unsubscribe();
    verify(kafkaConsumer).subscribe(eq(singletonList(KAFKA_TOPIC)), any(ConsumerRebalanceListener.class));
}
 
Example 3
@Override
public void subscribe(Collection<String> topics, ConsumerRebalanceListener callback) {
  try (
      @SuppressWarnings("unused") CloseableLock uLock = new CloseableLock(userLock);
      @SuppressWarnings("unused") CloseableLock srLock = new CloseableLock(delegateLock.readLock())
  ) {
    verifyOpen();
    subscribedTopics = topics;
    rebalanceListener = callback;
    assignedPartitions = null;
    subscriptionPattern = null;

    delegate.subscribe(topics, callback);
  }
}
 
Example 4
@Override
public void subscribe(Pattern pattern, ConsumerRebalanceListener callback) {
  try (
      @SuppressWarnings("unused") CloseableLock uLock = new CloseableLock(userLock);
      @SuppressWarnings("unused") CloseableLock srLock = new CloseableLock(delegateLock.readLock())
  ) {
    verifyOpen();
    subscribedTopics = null;
    rebalanceListener = callback;
    assignedPartitions = null;
    subscriptionPattern = pattern;

    delegate.subscribe(pattern, callback);
  }
}
 
Example 5
@Override
public void subscribe(Collection<String> topics, ConsumerRebalanceListener callback) {
  Set<String> newSubscription = new HashSet<>(topics);
  // TODO: This is a hot fix for KAFKA-3664 and should be removed after the issue is fixed.
  commitSync();
  for (TopicPartition tp : _kafkaConsumer.assignment()) {
    if (!newSubscription.contains(tp.topic())) {
      _consumerRecordsProcessor.clear(tp);
    }
  }
  _consumerRebalanceListener.setUserListener(callback);
  _kafkaConsumer.subscribe(new ArrayList<>(topics), _consumerRebalanceListener);
}
 
Example 6
@Override
public void subscribe(Pattern pattern, ConsumerRebalanceListener callback) {
  if (callback != null) {
    _consumerRebalanceListener.setUserListener(callback);
  }
  _kafkaConsumer.subscribe(pattern, _consumerRebalanceListener);
}
 
Example 7
Source Project: incubator-heron   Source File: KafkaSpoutTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void nextTuple() {
  when(kafkaConsumerFactory.create()).thenReturn(consumer);
  ConsumerRecords<String, byte[]> consumerRecords = new ConsumerRecords<>(
      Collections.singletonMap(new TopicPartition(DUMMY_TOPIC_NAME, 0),
          Collections.singletonList(new ConsumerRecord<>(DUMMY_TOPIC_NAME, 0, 0,
              "key", new byte[]{0xF}))));
  when(consumer.poll(any(Duration.class))).thenReturn(consumerRecords);
  doReturn(Collections.singletonMap(new MetricName("name", "group", "description",
      Collections.singletonMap("name", "value")), metric)).when(consumer).metrics();
  when(metric.metricValue()).thenReturn("sample value");

  kafkaSpout.open(Collections.singletonMap(Config.TOPOLOGY_RELIABILITY_MODE,
      ATMOST_ONCE.name()), topologyContext, collector);
  verify(consumer).subscribe(eq(Collections.singleton(DUMMY_TOPIC_NAME)),
      consumerRebalanceListenerArgumentCaptor.capture());
  ConsumerRebalanceListener consumerRebalanceListener =
      consumerRebalanceListenerArgumentCaptor.getValue();
  TopicPartition topicPartition = new TopicPartition(DUMMY_TOPIC_NAME, 0);
  consumerRebalanceListener.onPartitionsAssigned(Collections.singleton(topicPartition));

  kafkaSpout.nextTuple();
  verify(consumer).commitAsync();
  verify(topologyContext).registerMetric(eq("name-group-name-value"),
      kafkaMetricDecoratorArgumentCaptor.capture(), eq(60));
  assertEquals("sample value",
      kafkaMetricDecoratorArgumentCaptor.getValue().getValueAndReset());

  kafkaSpout.nextTuple();
  verify(collector).emit(eq("default"), listArgumentCaptor.capture());
  assertEquals("key", listArgumentCaptor.getValue().get(0));
  assertArrayEquals(new byte[]{0xF}, (byte[]) listArgumentCaptor.getValue().get(1));
}
 
Example 8
Source Project: incubator-heron   Source File: KafkaSpoutTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void ack() {
  when(kafkaConsumerFactory.create()).thenReturn(consumer);
  TopicPartition topicPartition = new TopicPartition(DUMMY_TOPIC_NAME, 0);
  List<ConsumerRecord<String, byte[]>> recordList = new ArrayList<>();
  byte[] randomBytes = new byte[1];
  for (int i = 0; i < 5; i++) {
    RANDOM.nextBytes(randomBytes);
    recordList.add(new ConsumerRecord<>(DUMMY_TOPIC_NAME, 0, i, "key",
        Arrays.copyOf(randomBytes, randomBytes.length)));
  }
  ConsumerRecords<String, byte[]> consumerRecords = new ConsumerRecords<>(
      Collections.singletonMap(topicPartition, recordList));
  when(consumer.poll(any(Duration.class))).thenReturn(consumerRecords);

  kafkaSpout.open(Collections.singletonMap(Config.TOPOLOGY_RELIABILITY_MODE,
      ATLEAST_ONCE.name()), topologyContext, collector);
  verify(consumer).subscribe(eq(Collections.singleton(DUMMY_TOPIC_NAME)),
      consumerRebalanceListenerArgumentCaptor.capture());
  ConsumerRebalanceListener consumerRebalanceListener =
      consumerRebalanceListenerArgumentCaptor.getValue();
  consumerRebalanceListener.onPartitionsAssigned(Collections.singleton(topicPartition));
  //poll the topic
  kafkaSpout.nextTuple();
  //emit all of the five records
  for (int i = 0; i < 5; i++) {
    kafkaSpout.nextTuple();
  }
  //ack came in out of order and the third record is not acknowledged
  kafkaSpout.ack(new KafkaSpout.ConsumerRecordMessageId(topicPartition, 4));
  kafkaSpout.ack(new KafkaSpout.ConsumerRecordMessageId(topicPartition, 0));
  kafkaSpout.ack(new KafkaSpout.ConsumerRecordMessageId(topicPartition, 1));
  kafkaSpout.ack(new KafkaSpout.ConsumerRecordMessageId(topicPartition, 3));
  //commit and poll
  kafkaSpout.nextTuple();
  verify(consumer).commitAsync(Collections.singletonMap(topicPartition,
      new OffsetAndMetadata(2)), null);
}
 
Example 9
Source Project: incubator-heron   Source File: KafkaSpoutTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void fail() {
  when(kafkaConsumerFactory.create()).thenReturn(consumer);
  TopicPartition topicPartition = new TopicPartition(DUMMY_TOPIC_NAME, 0);
  List<ConsumerRecord<String, byte[]>> recordList = new ArrayList<>();
  byte[] randomBytes = new byte[1];
  for (int i = 0; i < 5; i++) {
    RANDOM.nextBytes(randomBytes);
    recordList.add(new ConsumerRecord<>(DUMMY_TOPIC_NAME, 0, i, "key",
        Arrays.copyOf(randomBytes, randomBytes.length)));
  }
  ConsumerRecords<String, byte[]> consumerRecords = new ConsumerRecords<>(
      Collections.singletonMap(topicPartition, recordList));
  when(consumer.poll(any(Duration.class))).thenReturn(consumerRecords);

  kafkaSpout.open(Collections.singletonMap(Config.TOPOLOGY_RELIABILITY_MODE,
      ATLEAST_ONCE.name()), topologyContext, collector);
  verify(consumer).subscribe(eq(Collections.singleton(DUMMY_TOPIC_NAME)),
      consumerRebalanceListenerArgumentCaptor.capture());
  ConsumerRebalanceListener consumerRebalanceListener =
      consumerRebalanceListenerArgumentCaptor.getValue();
  consumerRebalanceListener.onPartitionsAssigned(Collections.singleton(topicPartition));
  //poll the topic
  kafkaSpout.nextTuple();
  //emit all of the five records
  for (int i = 0; i < 5; i++) {
    kafkaSpout.nextTuple();
  }
  //ack came in out of order, second and third record fails
  kafkaSpout.ack(new KafkaSpout.ConsumerRecordMessageId(topicPartition, 4));
  kafkaSpout.ack(new KafkaSpout.ConsumerRecordMessageId(topicPartition, 0));
  kafkaSpout.fail(new KafkaSpout.ConsumerRecordMessageId(topicPartition, 1));
  kafkaSpout.ack(new KafkaSpout.ConsumerRecordMessageId(topicPartition, 3));
  kafkaSpout.fail(new KafkaSpout.ConsumerRecordMessageId(topicPartition, 2));
  //commit and poll
  kafkaSpout.nextTuple();
  verify(consumer).seek(topicPartition, 1);
  verify(consumer).commitAsync(Collections.singletonMap(topicPartition,
      new OffsetAndMetadata(1)), null);
}
 
Example 10
Source Project: incubator-heron   Source File: KafkaSpoutTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void activate() {
  when(kafkaConsumerFactory.create()).thenReturn(consumer);
  kafkaSpout.open(Collections.singletonMap(Config.TOPOLOGY_RELIABILITY_MODE,
      ATMOST_ONCE.name()), topologyContext, collector);
  verify(consumer).subscribe(eq(Collections.singleton(DUMMY_TOPIC_NAME)),
      consumerRebalanceListenerArgumentCaptor.capture());
  ConsumerRebalanceListener consumerRebalanceListener =
      consumerRebalanceListenerArgumentCaptor.getValue();
  TopicPartition topicPartition = new TopicPartition(DUMMY_TOPIC_NAME, 0);
  consumerRebalanceListener.onPartitionsAssigned(Collections.singleton(topicPartition));
  kafkaSpout.activate();
  verify(consumer).resume(Collections.singleton(topicPartition));
}
 
Example 11
Source Project: incubator-heron   Source File: KafkaSpoutTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void deactivate() {
  when(kafkaConsumerFactory.create()).thenReturn(consumer);
  kafkaSpout.open(Collections.singletonMap(Config.TOPOLOGY_RELIABILITY_MODE,
      ATMOST_ONCE.name()), topologyContext, collector);
  verify(consumer).subscribe(eq(Collections.singleton(DUMMY_TOPIC_NAME)),
      consumerRebalanceListenerArgumentCaptor.capture());
  ConsumerRebalanceListener consumerRebalanceListener =
      consumerRebalanceListenerArgumentCaptor.getValue();
  TopicPartition topicPartition = new TopicPartition(DUMMY_TOPIC_NAME, 0);
  consumerRebalanceListener.onPartitionsAssigned(Collections.singleton(topicPartition));
  kafkaSpout.deactivate();
  verify(consumer).pause(Collections.singleton(topicPartition));
}
 
Example 12
Source Project: secor   Source File: SecorKafkaMessageIterator.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void subscribe(RebalanceHandler handler, SecorConfig config) {
    ConsumerRebalanceListener reBalanceListener = new SecorConsumerRebalanceListener(mKafkaConsumer, mZookeeperConnector, getSkipZookeeperOffsetSeek(config), config.getNewConsumerAutoOffsetReset(), handler);
    ;

    String[] subscribeList = config.getKafkaTopicList();
    if (subscribeList.length == 0 || Strings.isNullOrEmpty(subscribeList[0])) {
        mKafkaConsumer.subscribe(Pattern.compile(config.getKafkaTopicFilter()), reBalanceListener);
    } else {
        mKafkaConsumer.subscribe(Arrays.asList(subscribeList), reBalanceListener);
    }
}
 
Example 13
Source Project: kbear   Source File: ConsumerHolder.java    License: Apache License 2.0 4 votes vote down vote up
public ConsumerRebalanceListener getConsumerRebalanceListener() {
    return _consumerRebalanceListener;
}
 
Example 14
Source Project: kbear   Source File: ConsumerHolder.java    License: Apache License 2.0 4 votes vote down vote up
public void setConsumerRebalanceListener(ConsumerRebalanceListener consumerRebalanceListener) {
    _consumerRebalanceListener = consumerRebalanceListener;
}
 
Example 15
Source Project: kbear   Source File: ConsumerProxy.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void subscribe(Pattern pattern, ConsumerRebalanceListener callback) {
    throw new UnsupportedOperationException();
}
 
Example 16
Source Project: kafka-graphs   Source File: PregelConsumer.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void subscribe(Collection<String> topics, ConsumerRebalanceListener callback) {
    kafkaConsumer.subscribe(topics, callback);
}
 
Example 17
Source Project: kafka-graphs   Source File: PregelConsumer.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void subscribe(Pattern pattern, ConsumerRebalanceListener callback) {
    kafkaConsumer.subscribe(pattern, callback);
}
 
Example 18
Source Project: common-kafka   Source File: ProcessingKafkaConsumer.java    License: Apache License 2.0 4 votes vote down vote up
ConsumerRebalanceListener getRebalanceListener() {
    return new ProcessingRebalanceListener();
}
 
Example 19
Source Project: common-kafka   Source File: ProcessingKafkaConsumerITest.java    License: Apache License 2.0 4 votes vote down vote up
@Override
ConsumerRebalanceListener getRebalanceListener() {
    return new CommitTrackingProcessingRebalanceListener();
}
 
Example 20
Source Project: java-kafka-client   Source File: TracingKafkaConsumer.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void subscribe(Collection<String> topics, ConsumerRebalanceListener listener) {
  consumer.subscribe(topics, listener);
}
 
Example 21
Source Project: java-kafka-client   Source File: TracingKafkaConsumer.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) {
  consumer.subscribe(pattern, listener);
}
 
Example 22
Source Project: synapse   Source File: ConsumerRebalanceListeners.java    License: Apache License 2.0 4 votes vote down vote up
private ConsumerRebalanceListeners(final ConsumerRebalanceListener... listeners) {
    this.listeners = asList(listeners);
}
 
Example 23
Source Project: synapse   Source File: ConsumerRebalanceListeners.java    License: Apache License 2.0 4 votes vote down vote up
static ConsumerRebalanceListeners of(final ConsumerRebalanceListener... listeners) {
    return new ConsumerRebalanceListeners(listeners);
}
 
Example 24
public void setUserListener(ConsumerRebalanceListener userListener) {
  _userListener = userListener;
}
 
Example 25
@Override
public void subscribe(Collection<String> topics, ConsumerRebalanceListener callback) {
  _delegate.subscribe(topics, callback);
}
 
Example 26
@Override
public void subscribe(Pattern pattern, ConsumerRebalanceListener callback) {
  _delegate.subscribe(pattern, callback);
}
 
Example 27
Source Project: flink   Source File: KafkaConsumerThreadTest.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void subscribe(Collection<String> collection, ConsumerRebalanceListener consumerRebalanceListener) {
}
 
Example 28
Source Project: flink   Source File: KafkaConsumerThreadTest.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void subscribe(Pattern pattern, ConsumerRebalanceListener consumerRebalanceListener) {
}
 
Example 29
Source Project: brave   Source File: TracingConsumer.java    License: Apache License 2.0 4 votes vote down vote up
@Override public void subscribe(Collection<String> topics, ConsumerRebalanceListener callback) {
  delegate.subscribe(topics, callback);
}
 
Example 30
Source Project: brave   Source File: TracingConsumer.java    License: Apache License 2.0 4 votes vote down vote up
@Override public void subscribe(Pattern pattern, ConsumerRebalanceListener callback) {
  delegate.subscribe(pattern, callback);
}