kafka.consumer.Whitelist Java Examples

The following examples show how to use kafka.consumer.Whitelist. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: KafkaReceiver.java From koper with Apache License 2.0

6 votes

/**
 * 启动 MessageReceiver，开始监听topic消息
 */
@Override
public void start() {

    if (consumer == null) {
        //sync init
        synchronized (lock) {
            init();
        }
    }

    String topicString = buildTopicsString();

    Whitelist topicFilter = new Whitelist(topicString);
    List<KafkaStream<byte[], byte[]>> streamList = consumer.createMessageStreamsByFilter(topicFilter, partitions);

    if (org.apache.commons.collections.CollectionUtils.isEmpty(streamList))
        try {
            TimeUnit.MILLISECONDS.sleep(1);
        } catch (InterruptedException e) {
            log.warn(e.getMessage(), e);
        }
    processStreamsByTopic(topicString, streamList);

}

Example #2

Source File: LegacyKafkaMessageIterator.java From secor with Apache License 2.0

6 votes

@Override
public void init(SecorConfig config) throws UnknownHostException {
    this.mConfig = config;

    mConsumerConnector = Consumer.createJavaConsumerConnector(createConsumerConfig());

    if (!mConfig.getKafkaTopicBlacklist().isEmpty() && !mConfig.getKafkaTopicFilter().isEmpty()) {
        throw new RuntimeException("Topic filter and blacklist cannot be both specified.");
    }
    TopicFilter topicFilter = !mConfig.getKafkaTopicBlacklist().isEmpty() ? new Blacklist(mConfig.getKafkaTopicBlacklist()) :
            new Whitelist(mConfig.getKafkaTopicFilter());
    LOG.debug("Use TopicFilter {}({})", topicFilter.getClass(), topicFilter);
    List<KafkaStream<byte[], byte[]>> streams =
            mConsumerConnector.createMessageStreamsByFilter(topicFilter);
    KafkaStream<byte[], byte[]> stream = streams.get(0);
    mIterator = stream.iterator();
    mKafkaMessageTimestampFactory = new KafkaMessageTimestampFactory(mConfig.getKafkaMessageTimestampClass());
}

Example #3

Source File: KafkaRpcPluginThread.java From opentsdb-rpc-kafka with Apache License 2.0

4 votes

/**
 * Default ctor
 * @param group The group object this writer belongs to
 * @param threadID The ID of the thread, an index from 0 to max int
 * @param topics The topic list to subscribe to
 */
public KafkaRpcPluginThread(final KafkaRpcPluginGroup group, 
    final int threadID, final String topics) {
  if (topics == null || topics.isEmpty()) {
    throw new IllegalArgumentException("Missing topics");
  }
  if (threadID < 0) {
    throw new IllegalArgumentException("Cannot have a negative thread ID: " 
        + threadID);
  }
  if (group.getParent().getTSDB() == null) {
    throw new IllegalArgumentException("Missing TSDB in the group");
  }
  if (group.getRateLimiter() == null) {
    throw new IllegalArgumentException("Missing rate limiter in the group");
  }
  if (group.getGroupID() == null || group.getGroupID().isEmpty()) {
    throw new IllegalArgumentException("Missing group ID");
  }
  if (group.getParent().getHost() == null || 
      group.getParent().getHost().isEmpty()) {
    throw new IllegalArgumentException("Missing host name");
  }
  
  namespace_counters = group.getParent().getNamespaceCounters();
  track_metric_prefix = group.getParent().trackMetricPrefix();
  this.thread_id = threadID;
  this.group = group;
  this.tsdb = group.getParent().getTSDB();
  this.rate_limiter = group.getRateLimiter();
  this.consumer_type = group.getConsumerType();
  thread_running.set(false);

  topic_filter = new Whitelist(topics);
  consumer_id = threadID + "_" + group.getParent().getHost();
  if (consumer_type == TsdbConsumerType.REQUEUE_RAW) {
    if (group.getParent().getConfig().hasProperty(
        KafkaRpcPluginConfig.PLUGIN_PROPERTY_BASE + "requeueDelay")) {
      requeue_delay = group.getParent().getConfig().getLong(
          KafkaRpcPluginConfig.PLUGIN_PROPERTY_BASE + "requeueDelay");
    } else {
      requeue_delay = KafkaRpcPluginConfig.DEFAULT_REQUEUE_DELAY_MS;
    }
  } else {
    requeue_delay = 0;
  }
  deserializer = group.getDeserializer();
  deserializer.initialize(tsdb);
}

Example #4

Source File: KafkaPersistReader.java From streams with Apache License 2.0

4 votes

@Override
public void startStream() {

  Properties props = new Properties();
  props.setProperty("serializer.encoding", "UTF8");

  ConsumerConfig consumerConfig = new ConsumerConfig(props);

  consumerConnector = Consumer.createJavaConsumerConnector(consumerConfig);

  Whitelist topics = new Whitelist(config.getTopic());
  VerifiableProperties vprops = new VerifiableProperties(props);

  inStreams = consumerConnector.createMessageStreamsByFilter(topics, 1, new StringDecoder(vprops), new StringDecoder(vprops));

  for (final KafkaStream stream : inStreams) {
    executor.submit(new KafkaPersistReaderTask(this, stream));
  }

}