kafka.consumer.TopicFilter Java Examples

@Test
public void runNoData() throws Exception {
  when(iterator.hasNext()).thenReturn(false);

  final KafkaRpcPluginThread writer = Mockito.spy(
      new KafkaRpcPluginThread(group, 1, TOPICS));
  writer.run();
  verify(tsdb, never()).addPoint(anyString(), anyLong(), anyLong(), anyMap());
  verify(tsdb, never()).addHistogramPoint(anyString(), anyLong(), 
      any(byte[].class), anyMap());
  verify(tsdb, never()).addAggregatePoint(anyString(), anyLong(), anyLong(), 
      anyMap(), anyBoolean(), anyString(), anyString(), anyString());
  verify(consumer_connector, times(1))
    .createMessageStreamsByFilter(any(TopicFilter.class), anyInt());
  verify(writer, times(1)).shutdown();
  verify(consumer_connector, times(1)).shutdown();
}
 

@Test
public void runNoDataRestart() throws Exception {
  when(iterator.hasNext()).thenReturn(false);

  final KafkaRpcPluginThread writer = Mockito.spy(
      new KafkaRpcPluginThread(group, 1, TOPICS));
  writer.run();
  writer.run();
  verify(tsdb, never()).addPoint(anyString(), anyLong(), anyLong(), anyMap());
  verify(tsdb, never()).addHistogramPoint(anyString(), anyLong(), 
      any(byte[].class), anyMap());
  verify(tsdb, never()).addAggregatePoint(anyString(), anyLong(), anyLong(), 
      anyMap(), anyBoolean(), anyString(), anyString(), anyString());
  verify(consumer_connector, times(2))
    .createMessageStreamsByFilter(any(TopicFilter.class), anyInt());
  verify(writer, times(2)).shutdown();
  verify(consumer_connector, times(2)).shutdown();
}
 

@Test
public void runNoStreams() throws Exception {
  when(stream_list.get(0))
          .thenThrow(new ArrayIndexOutOfBoundsException());

  KafkaRpcPluginThread writer = Mockito.spy(
      new KafkaRpcPluginThread(group, 1, TOPICS));
  writer.run();
  verify(tsdb, never()).addPoint(anyString(), anyLong(), anyLong(), anyMap());
  verify(tsdb, never()).addHistogramPoint(anyString(), anyLong(), 
      any(byte[].class), anyMap());
  verify(tsdb, never()).addAggregatePoint(anyString(), anyLong(), anyLong(), 
      anyMap(), anyBoolean(), anyString(), anyString(), anyString());
  verify(consumer_connector, times(1))
    .createMessageStreamsByFilter(any(TopicFilter.class), anyInt());
  verify(writer, times(1)).shutdown();
  verify(consumer_connector, times(1)).shutdown();
}
 

private void verifyMessageRead(final KafkaRpcPluginThread writer,
                               final boolean requeued) {
  verify(writer, times(1)).shutdown();
  verify(consumer_connector, times(1)).shutdown();
  verify(consumer_connector, times(1))
    .createMessageStreamsByFilter(any(TopicFilter.class), anyInt());
  verify(iterator, times(2)).hasNext();
  if (requeued) {
    verify(requeue, times(1)).handleError( 
        any(IncomingDataPoint.class), any(Exception.class));
  } else {
    verify(requeue, never()).handleError(
        any(IncomingDataPoint.class), any(Exception.class));
  }
  if (data != null) {
    verify(rate_limiter, times(1)).acquire();
  }
}
 

public <K, V> Map<String, List<PulsarKafkaStream<K, V>>> createMessageStreamsByFilter(TopicFilter topicFilter,
        Map<String, Integer> topicCountMap, Decoder<K> keyDecoder, Decoder<V> valueDecoder) {

    Map<String, List<PulsarKafkaStream<K, V>>> streams = Maps.newHashMap();

    topicCountMap.forEach((topic, count) -> {
        try {
            Consumer<byte[]> consumer = consumerBuilder.topic(topic).subscribe();
            resetOffsets(consumer, strategy);
            log.info("Creating stream for {}-{} with config {}", topic, groupId, consumerBuilder.toString());
            for (int i = 0; i < count; i++) {
                PulsarKafkaStream<K, V> stream = new PulsarKafkaStream<>(keyDecoder, valueDecoder, consumer,
                        isAutoCommit, clientId);
                // if multiple thread-count present then client expects multiple streams reading from the same
                // topic. so, create multiple stream using the same consumer
                streams.computeIfAbsent(topic, key -> Lists.newArrayList()).add(stream);
                topicStreams.add(stream);
            }
        } catch (PulsarClientException e) {
            log.error("Failed to subscribe on topic {} with group-id {}, {}", topic, groupId, e.getMessage(), e);
            throw new RuntimeException("Failed to subscribe on topic " + topic, e);
        }
    });
    return streams;
}
 

@Override
public void init(SecorConfig config) throws UnknownHostException {
    this.mConfig = config;

    mConsumerConnector = Consumer.createJavaConsumerConnector(createConsumerConfig());

    if (!mConfig.getKafkaTopicBlacklist().isEmpty() && !mConfig.getKafkaTopicFilter().isEmpty()) {
        throw new RuntimeException("Topic filter and blacklist cannot be both specified.");
    }
    TopicFilter topicFilter = !mConfig.getKafkaTopicBlacklist().isEmpty() ? new Blacklist(mConfig.getKafkaTopicBlacklist()) :
            new Whitelist(mConfig.getKafkaTopicFilter());
    LOG.debug("Use TopicFilter {}({})", topicFilter.getClass(), topicFilter);
    List<KafkaStream<byte[], byte[]>> streams =
            mConsumerConnector.createMessageStreamsByFilter(topicFilter);
    KafkaStream<byte[], byte[]> stream = streams.get(0);
    mIterator = stream.iterator();
    mKafkaMessageTimestampFactory = new KafkaMessageTimestampFactory(mConfig.getKafkaMessageTimestampClass());
}
 

@Test
public void runConsumerRuntimeException() throws Exception {
  when(consumer_connector.createMessageStreamsByFilter(
      (TopicFilter) any(), anyInt())).thenThrow(
          new RuntimeException("Foobar"));
  KafkaRpcPluginThread writer = Mockito.spy(
      new KafkaRpcPluginThread(group, 1, TOPICS));
  writer.run();
  
  verify(writer, times(1)).shutdown();
  verify(consumer_connector, times(1)).shutdown();
}
 

@Test(expected = Exception.class)
public void runConsumerException() throws Exception {
  when(consumer_connector.createMessageStreamsByFilter(
      (TopicFilter) any(), anyInt())).thenThrow(
          new Exception("Foobar"));
  KafkaRpcPluginThread writer = Mockito.spy(
      new KafkaRpcPluginThread(group, 1, TOPICS));
  writer.run();
  
  verify(writer, times(1)).shutdown();
  verify(consumer_connector, times(1)).shutdown();
}
 

@SuppressWarnings("unchecked")
@Before
public void before() throws Exception {
  tsdb = PowerMockito.mock(TSDB.class);
  config = new KafkaRpcPluginConfig(new Config(false));
  group = mock(KafkaRpcPluginGroup.class);
  message = mock(MessageAndMetadata.class);
  rate_limiter = mock(RateLimiter.class);
  requeue = mock(KafkaStorageExceptionHandler.class);
  counters = new ConcurrentHashMap<String, Map<String, AtomicLong>>();
  deserializer = new JSONDeserializer();
  
  consumer_connector = mock(ConsumerConnector.class);

  mockStatic(Consumer.class);
  when(Consumer.createJavaConsumerConnector((ConsumerConfig) any()))
          .thenReturn(consumer_connector);
  
  when(tsdb.getConfig()).thenReturn(config);
  when(tsdb.getStorageExceptionHandler()).thenReturn(requeue);
  
  parent = mock(KafkaRpcPlugin.class);
  when(parent.getHost()).thenReturn(LOCALHOST);
  when(parent.getTSDB()).thenReturn(tsdb);
  when(parent.getConfig()).thenReturn(config);
  when(parent.getNamespaceCounters()).thenReturn(counters);
  when(parent.trackMetricPrefix()).thenReturn(true);
  
  when(group.getParent()).thenReturn(parent);
  when(group.getRateLimiter()).thenReturn(rate_limiter);
  when(group.getGroupID()).thenReturn(GROUPID);
  when(group.getConsumerType()).thenReturn(TsdbConsumerType.RAW);
  when(group.getDeserializer()).thenReturn(deserializer);
  
  config.overrideConfig(KafkaRpcPluginConfig.KAFKA_CONFIG_PREFIX 
      + "zookeeper.connect", ZKS);
  
  stream_list = mock(List.class);
  when(consumer_connector.createMessageStreamsByFilter(
      (TopicFilter) any(), anyInt())).thenReturn(stream_list);

  final KafkaStream<byte[], byte[]> stream = mock(KafkaStream.class);
  when(stream_list.get(0)).thenReturn(stream);

  iterator = mock(ConsumerIterator.class);
  when(stream.iterator()).thenReturn(iterator);

  when(iterator.hasNext()).thenReturn(true).thenReturn(false);
  when(iterator.next()).thenReturn(message);
  
  PowerMockito.mockStatic(ConsumerConfig.class);
  PowerMockito.whenNew(ConsumerConfig.class).withAnyArguments()
    .thenReturn(mock(ConsumerConfig.class));
  
  PowerMockito.mockStatic(Consumer.class);
  when(Consumer.createJavaConsumerConnector(any(ConsumerConfig.class)))
    .thenReturn(consumer_connector);
}
 

public List<PulsarKafkaStream<byte[], byte[]>> createMessageStreamsByFilter(TopicFilter topicFilter) {
    throw new UnsupportedOperationException("method not supported");
}
 

public List<PulsarKafkaStream<byte[], byte[]>> createMessageStreamsByFilter(TopicFilter topicFilter, int arg1) {
    throw new UnsupportedOperationException("method not supported");
}
 

public <K, V> List<PulsarKafkaStream<K, V>> createMessageStreamsByFilter(TopicFilter topicFilter, int arg1,
        Decoder<K> keyDecoder, Decoder<V> valueDecoder) {
    throw new UnsupportedOperationException("method not supported");
}