org.apache.kafka.clients.consumer.ConsumerRebalanceListener Java Examples

The following examples show how to use org.apache.kafka.clients.consumer.ConsumerRebalanceListener. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ConsumerProxy.java    From kbear with Apache License 2.0 6 votes vote down vote up
@Override
public void subscribe(Collection<String> topics, ConsumerRebalanceListener callback) {
    ObjectExtension.requireNonNull(topics, "topics");
    ObjectExtension.requireNonNull(callback, "callback");

    runWithoutConcurrency(() -> {
        synchronized (_addRemoveLock) {
            removeNotUsed(topics);
            topics.forEach(t -> {
                ConsumerHolder consumerHolder = getOrAddConsumer(t);
                consumerHolder.getConsumer().subscribe(Arrays.asList(t), callback);
                consumerHolder.setConsumerRebalanceListener(callback);
                consumerHolder.setAssignments(null);
            });
        }
    });
}
 
Example #2
Source File: KafkaSpoutTest.java    From incubator-heron with Apache License 2.0 5 votes vote down vote up
@Test
public void fail() {
  when(kafkaConsumerFactory.create()).thenReturn(consumer);
  TopicPartition topicPartition = new TopicPartition(DUMMY_TOPIC_NAME, 0);
  List<ConsumerRecord<String, byte[]>> recordList = new ArrayList<>();
  byte[] randomBytes = new byte[1];
  for (int i = 0; i < 5; i++) {
    RANDOM.nextBytes(randomBytes);
    recordList.add(new ConsumerRecord<>(DUMMY_TOPIC_NAME, 0, i, "key",
        Arrays.copyOf(randomBytes, randomBytes.length)));
  }
  ConsumerRecords<String, byte[]> consumerRecords = new ConsumerRecords<>(
      Collections.singletonMap(topicPartition, recordList));
  when(consumer.poll(any(Duration.class))).thenReturn(consumerRecords);

  kafkaSpout.open(Collections.singletonMap(Config.TOPOLOGY_RELIABILITY_MODE,
      ATLEAST_ONCE.name()), topologyContext, collector);
  verify(consumer).subscribe(eq(Collections.singleton(DUMMY_TOPIC_NAME)),
      consumerRebalanceListenerArgumentCaptor.capture());
  ConsumerRebalanceListener consumerRebalanceListener =
      consumerRebalanceListenerArgumentCaptor.getValue();
  consumerRebalanceListener.onPartitionsAssigned(Collections.singleton(topicPartition));
  //poll the topic
  kafkaSpout.nextTuple();
  //emit all of the five records
  for (int i = 0; i < 5; i++) {
    kafkaSpout.nextTuple();
  }
  //ack came in out of order, second and third record fails
  kafkaSpout.ack(new KafkaSpout.ConsumerRecordMessageId(topicPartition, 4));
  kafkaSpout.ack(new KafkaSpout.ConsumerRecordMessageId(topicPartition, 0));
  kafkaSpout.fail(new KafkaSpout.ConsumerRecordMessageId(topicPartition, 1));
  kafkaSpout.ack(new KafkaSpout.ConsumerRecordMessageId(topicPartition, 3));
  kafkaSpout.fail(new KafkaSpout.ConsumerRecordMessageId(topicPartition, 2));
  //commit and poll
  kafkaSpout.nextTuple();
  verify(consumer).seek(topicPartition, 1);
  verify(consumer).commitAsync(Collections.singletonMap(topicPartition,
      new OffsetAndMetadata(1)), null);
}
 
Example #3
Source File: LiKafkaConsumerImpl.java    From li-apache-kafka-clients with BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Override
public void subscribe(Collection<String> topics, ConsumerRebalanceListener callback) {
  Set<String> newSubscription = new HashSet<>(topics);
  // TODO: This is a hot fix for KAFKA-3664 and should be removed after the issue is fixed.
  commitSync();
  for (TopicPartition tp : _kafkaConsumer.assignment()) {
    if (!newSubscription.contains(tp.topic())) {
      _consumerRecordsProcessor.clear(tp);
    }
  }
  _consumerRebalanceListener.setUserListener(callback);
  _kafkaConsumer.subscribe(new ArrayList<>(topics), _consumerRebalanceListener);
}
 
Example #4
Source File: LiKafkaConsumerImpl.java    From li-apache-kafka-clients with BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Override
public void subscribe(Pattern pattern, ConsumerRebalanceListener callback) {
  if (callback != null) {
    _consumerRebalanceListener.setUserListener(callback);
  }
  _kafkaConsumer.subscribe(pattern, _consumerRebalanceListener);
}
 
Example #5
Source File: LiKafkaInstrumentedConsumerImpl.java    From li-apache-kafka-clients with BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Override
public void subscribe(Pattern pattern, ConsumerRebalanceListener callback) {
  try (
      @SuppressWarnings("unused") CloseableLock uLock = new CloseableLock(userLock);
      @SuppressWarnings("unused") CloseableLock srLock = new CloseableLock(delegateLock.readLock())
  ) {
    verifyOpen();
    subscribedTopics = null;
    rebalanceListener = callback;
    assignedPartitions = null;
    subscriptionPattern = pattern;

    delegate.subscribe(pattern, callback);
  }
}
 
Example #6
Source File: LiKafkaInstrumentedConsumerImpl.java    From li-apache-kafka-clients with BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Override
public void subscribe(Collection<String> topics, ConsumerRebalanceListener callback) {
  try (
      @SuppressWarnings("unused") CloseableLock uLock = new CloseableLock(userLock);
      @SuppressWarnings("unused") CloseableLock srLock = new CloseableLock(delegateLock.readLock())
  ) {
    verifyOpen();
    subscribedTopics = topics;
    rebalanceListener = callback;
    assignedPartitions = null;
    subscriptionPattern = null;

    delegate.subscribe(topics, callback);
  }
}
 
Example #7
Source File: KafkaMessageLogReceiverEndpointTest.java    From synapse with Apache License 2.0 5 votes vote down vote up
@Test
public void shouldUnsubscribeIfKafkaConsumerIsAlreadySubscribed() throws Exception {
    // given
    when(kafkaConsumer.subscription()).thenReturn(newSet(KAFKA_TOPIC));

    // when
    endpoint.consumeUntil(fromHorizon(), shutdown());

    // then
    verify(kafkaConsumer).unsubscribe();
    verify(kafkaConsumer).subscribe(eq(singletonList(KAFKA_TOPIC)), any(ConsumerRebalanceListener.class));
}
 
Example #8
Source File: KafkaSpoutTest.java    From incubator-heron with Apache License 2.0 5 votes vote down vote up
@Test
public void nextTuple() {
  when(kafkaConsumerFactory.create()).thenReturn(consumer);
  ConsumerRecords<String, byte[]> consumerRecords = new ConsumerRecords<>(
      Collections.singletonMap(new TopicPartition(DUMMY_TOPIC_NAME, 0),
          Collections.singletonList(new ConsumerRecord<>(DUMMY_TOPIC_NAME, 0, 0,
              "key", new byte[]{0xF}))));
  when(consumer.poll(any(Duration.class))).thenReturn(consumerRecords);
  doReturn(Collections.singletonMap(new MetricName("name", "group", "description",
      Collections.singletonMap("name", "value")), metric)).when(consumer).metrics();
  when(metric.metricValue()).thenReturn("sample value");

  kafkaSpout.open(Collections.singletonMap(Config.TOPOLOGY_RELIABILITY_MODE,
      ATMOST_ONCE.name()), topologyContext, collector);
  verify(consumer).subscribe(eq(Collections.singleton(DUMMY_TOPIC_NAME)),
      consumerRebalanceListenerArgumentCaptor.capture());
  ConsumerRebalanceListener consumerRebalanceListener =
      consumerRebalanceListenerArgumentCaptor.getValue();
  TopicPartition topicPartition = new TopicPartition(DUMMY_TOPIC_NAME, 0);
  consumerRebalanceListener.onPartitionsAssigned(Collections.singleton(topicPartition));

  kafkaSpout.nextTuple();
  verify(consumer).commitAsync();
  verify(topologyContext).registerMetric(eq("name-group-name-value"),
      kafkaMetricDecoratorArgumentCaptor.capture(), eq(60));
  assertEquals("sample value",
      kafkaMetricDecoratorArgumentCaptor.getValue().getValueAndReset());

  kafkaSpout.nextTuple();
  verify(collector).emit(eq("default"), listArgumentCaptor.capture());
  assertEquals("key", listArgumentCaptor.getValue().get(0));
  assertArrayEquals(new byte[]{0xF}, (byte[]) listArgumentCaptor.getValue().get(1));
}
 
Example #9
Source File: KafkaSpoutTest.java    From incubator-heron with Apache License 2.0 5 votes vote down vote up
@Test
public void ack() {
  when(kafkaConsumerFactory.create()).thenReturn(consumer);
  TopicPartition topicPartition = new TopicPartition(DUMMY_TOPIC_NAME, 0);
  List<ConsumerRecord<String, byte[]>> recordList = new ArrayList<>();
  byte[] randomBytes = new byte[1];
  for (int i = 0; i < 5; i++) {
    RANDOM.nextBytes(randomBytes);
    recordList.add(new ConsumerRecord<>(DUMMY_TOPIC_NAME, 0, i, "key",
        Arrays.copyOf(randomBytes, randomBytes.length)));
  }
  ConsumerRecords<String, byte[]> consumerRecords = new ConsumerRecords<>(
      Collections.singletonMap(topicPartition, recordList));
  when(consumer.poll(any(Duration.class))).thenReturn(consumerRecords);

  kafkaSpout.open(Collections.singletonMap(Config.TOPOLOGY_RELIABILITY_MODE,
      ATLEAST_ONCE.name()), topologyContext, collector);
  verify(consumer).subscribe(eq(Collections.singleton(DUMMY_TOPIC_NAME)),
      consumerRebalanceListenerArgumentCaptor.capture());
  ConsumerRebalanceListener consumerRebalanceListener =
      consumerRebalanceListenerArgumentCaptor.getValue();
  consumerRebalanceListener.onPartitionsAssigned(Collections.singleton(topicPartition));
  //poll the topic
  kafkaSpout.nextTuple();
  //emit all of the five records
  for (int i = 0; i < 5; i++) {
    kafkaSpout.nextTuple();
  }
  //ack came in out of order and the third record is not acknowledged
  kafkaSpout.ack(new KafkaSpout.ConsumerRecordMessageId(topicPartition, 4));
  kafkaSpout.ack(new KafkaSpout.ConsumerRecordMessageId(topicPartition, 0));
  kafkaSpout.ack(new KafkaSpout.ConsumerRecordMessageId(topicPartition, 1));
  kafkaSpout.ack(new KafkaSpout.ConsumerRecordMessageId(topicPartition, 3));
  //commit and poll
  kafkaSpout.nextTuple();
  verify(consumer).commitAsync(Collections.singletonMap(topicPartition,
      new OffsetAndMetadata(2)), null);
}
 
Example #10
Source File: KafkaSpoutTest.java    From incubator-heron with Apache License 2.0 5 votes vote down vote up
@Test
public void activate() {
  when(kafkaConsumerFactory.create()).thenReturn(consumer);
  kafkaSpout.open(Collections.singletonMap(Config.TOPOLOGY_RELIABILITY_MODE,
      ATMOST_ONCE.name()), topologyContext, collector);
  verify(consumer).subscribe(eq(Collections.singleton(DUMMY_TOPIC_NAME)),
      consumerRebalanceListenerArgumentCaptor.capture());
  ConsumerRebalanceListener consumerRebalanceListener =
      consumerRebalanceListenerArgumentCaptor.getValue();
  TopicPartition topicPartition = new TopicPartition(DUMMY_TOPIC_NAME, 0);
  consumerRebalanceListener.onPartitionsAssigned(Collections.singleton(topicPartition));
  kafkaSpout.activate();
  verify(consumer).resume(Collections.singleton(topicPartition));
}
 
Example #11
Source File: KafkaSpoutTest.java    From incubator-heron with Apache License 2.0 5 votes vote down vote up
@Test
public void deactivate() {
  when(kafkaConsumerFactory.create()).thenReturn(consumer);
  kafkaSpout.open(Collections.singletonMap(Config.TOPOLOGY_RELIABILITY_MODE,
      ATMOST_ONCE.name()), topologyContext, collector);
  verify(consumer).subscribe(eq(Collections.singleton(DUMMY_TOPIC_NAME)),
      consumerRebalanceListenerArgumentCaptor.capture());
  ConsumerRebalanceListener consumerRebalanceListener =
      consumerRebalanceListenerArgumentCaptor.getValue();
  TopicPartition topicPartition = new TopicPartition(DUMMY_TOPIC_NAME, 0);
  consumerRebalanceListener.onPartitionsAssigned(Collections.singleton(topicPartition));
  kafkaSpout.deactivate();
  verify(consumer).pause(Collections.singleton(topicPartition));
}
 
Example #12
Source File: SecorKafkaMessageIterator.java    From secor with Apache License 2.0 5 votes vote down vote up
@Override
public void subscribe(RebalanceHandler handler, SecorConfig config) {
    ConsumerRebalanceListener reBalanceListener = new SecorConsumerRebalanceListener(mKafkaConsumer, mZookeeperConnector, getSkipZookeeperOffsetSeek(config), config.getNewConsumerAutoOffsetReset(), handler);
    ;

    String[] subscribeList = config.getKafkaTopicList();
    if (subscribeList.length == 0 || Strings.isNullOrEmpty(subscribeList[0])) {
        mKafkaConsumer.subscribe(Pattern.compile(config.getKafkaTopicFilter()), reBalanceListener);
    } else {
        mKafkaConsumer.subscribe(Arrays.asList(subscribeList), reBalanceListener);
    }
}
 
Example #13
Source File: ConsumerHolder.java    From kbear with Apache License 2.0 4 votes vote down vote up
public void setConsumerRebalanceListener(ConsumerRebalanceListener consumerRebalanceListener) {
    _consumerRebalanceListener = consumerRebalanceListener;
}
 
Example #14
Source File: KafkaConsumerThreadTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public void subscribe(Collection<String> collection, ConsumerRebalanceListener consumerRebalanceListener) {
}
 
Example #15
Source File: MockLiKafkaConsumer.java    From li-apache-kafka-clients with BSD 2-Clause "Simplified" License 4 votes vote down vote up
@Override
public void subscribe(Pattern pattern, ConsumerRebalanceListener callback) {
  _delegate.subscribe(pattern, callback);
}
 
Example #16
Source File: KafkaConsumerThreadTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public void subscribe(Pattern pattern, ConsumerRebalanceListener consumerRebalanceListener) {
}
 
Example #17
Source File: MockLiKafkaConsumer.java    From li-apache-kafka-clients with BSD 2-Clause "Simplified" License 4 votes vote down vote up
@Override
public void subscribe(Collection<String> topics, ConsumerRebalanceListener callback) {
  _delegate.subscribe(topics, callback);
}
 
Example #18
Source File: TracingConsumer.java    From brave with Apache License 2.0 4 votes vote down vote up
@Override public void subscribe(Collection<String> topics, ConsumerRebalanceListener callback) {
  delegate.subscribe(topics, callback);
}
 
Example #19
Source File: TracingConsumer.java    From brave with Apache License 2.0 4 votes vote down vote up
@Override public void subscribe(Pattern pattern, ConsumerRebalanceListener callback) {
  delegate.subscribe(pattern, callback);
}
 
Example #20
Source File: LiKafkaConsumerRebalanceListener.java    From li-apache-kafka-clients with BSD 2-Clause "Simplified" License 4 votes vote down vote up
public void setUserListener(ConsumerRebalanceListener userListener) {
  _userListener = userListener;
}
 
Example #21
Source File: ConsumerRebalanceListeners.java    From synapse with Apache License 2.0 4 votes vote down vote up
static ConsumerRebalanceListeners of(final ConsumerRebalanceListener... listeners) {
    return new ConsumerRebalanceListeners(listeners);
}
 
Example #22
Source File: ConsumerRebalanceListeners.java    From synapse with Apache License 2.0 4 votes vote down vote up
private ConsumerRebalanceListeners(final ConsumerRebalanceListener... listeners) {
    this.listeners = asList(listeners);
}
 
Example #23
Source File: TracingKafkaConsumer.java    From java-kafka-client with Apache License 2.0 4 votes vote down vote up
@Override
public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) {
  consumer.subscribe(pattern, listener);
}
 
Example #24
Source File: TracingKafkaConsumer.java    From java-kafka-client with Apache License 2.0 4 votes vote down vote up
@Override
public void subscribe(Collection<String> topics, ConsumerRebalanceListener listener) {
  consumer.subscribe(topics, listener);
}
 
Example #25
Source File: ProcessingKafkaConsumerITest.java    From common-kafka with Apache License 2.0 4 votes vote down vote up
@Override
ConsumerRebalanceListener getRebalanceListener() {
    return new CommitTrackingProcessingRebalanceListener();
}
 
Example #26
Source File: ProcessingKafkaConsumer.java    From common-kafka with Apache License 2.0 4 votes vote down vote up
ConsumerRebalanceListener getRebalanceListener() {
    return new ProcessingRebalanceListener();
}
 
Example #27
Source File: PregelConsumer.java    From kafka-graphs with Apache License 2.0 4 votes vote down vote up
@Override
public void subscribe(Pattern pattern, ConsumerRebalanceListener callback) {
    kafkaConsumer.subscribe(pattern, callback);
}
 
Example #28
Source File: PregelConsumer.java    From kafka-graphs with Apache License 2.0 4 votes vote down vote up
@Override
public void subscribe(Collection<String> topics, ConsumerRebalanceListener callback) {
    kafkaConsumer.subscribe(topics, callback);
}
 
Example #29
Source File: ConsumerProxy.java    From kbear with Apache License 2.0 4 votes vote down vote up
@Override
public void subscribe(Pattern pattern, ConsumerRebalanceListener callback) {
    throw new UnsupportedOperationException();
}
 
Example #30
Source File: ConsumerHolder.java    From kbear with Apache License 2.0 4 votes vote down vote up
public ConsumerRebalanceListener getConsumerRebalanceListener() {
    return _consumerRebalanceListener;
}