kafka.utils.VerifiableProperties Java Examples

The following examples show how to use kafka.utils.VerifiableProperties. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KafkaConsumer.java    From incubator-iotdb with Apache License 2.0 6 votes vote down vote up
private void consume() {
  /**
   * Specify the number of consumer thread
   */
  Map<String, Integer> topicCountMap = new HashMap<>();
  topicCountMap.put(Constant.TOPIC, Constant.CONSUMER_THREAD_NUM);

  /**
   * Specify data decoder
   */
  StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties());
  StringDecoder valueDecoder = new StringDecoder(new VerifiableProperties());

  Map<String, List<KafkaStream<String, String>>> consumerMap = consumer
      .createMessageStreams(topicCountMap, keyDecoder,
          valueDecoder);

  List<KafkaStream<String, String>> streams = consumerMap.get(Constant.TOPIC);
  ExecutorService executor = Executors.newFixedThreadPool(Constant.CONSUMER_THREAD_NUM);
  for (final KafkaStream<String, String> stream : streams) {
    executor.submit(new KafkaConsumerThread(stream));
  }
}
 
Example #2
Source File: KafkaHttpMetricsReporter.java    From kafka-http-metrics-reporter with Apache License 2.0 6 votes vote down vote up
@Override
public void init(VerifiableProperties verifiableProperties) {

    if (!initialized) {
        // get configured metrics from kafka
        KafkaMetricsConfig metricsConfig = new KafkaMetricsConfig(verifiableProperties);

        // get the configured properties from kafka to set the bindAddress and port.
        bindAddress = verifiableProperties.getProperty("kafka.http.metrics.host");
        port = Integer.parseInt(verifiableProperties.getProperty("kafka.http.metrics.port"));
        enabled = Boolean.parseBoolean(verifiableProperties.getProperty("kafka.http.metrics.reporter.enabled"));

        // construct the Metrics Server
        metricsServer = new KafkaHttpMetricsServer(bindAddress, port);
        initialized = true;

        // call the method startReporter
        startReporter(metricsConfig.pollingIntervalSecs());
    } else {
        LOG.error("Kafka Http Metrics Reporter already initialized");
    }
}
 
Example #3
Source File: KafkaAdminClient.java    From common-kafka with Apache License 2.0 6 votes vote down vote up
/**
 * Returns an {@link Authorizer} to make {@link Acl} requests
 *
 * @return an {@link Authorizer} to make {@link Acl} requests
 *
 * @throws AdminOperationException
 *      if there is an issue creating the authorizer
 */
public Authorizer getAuthorizer() {
    if (authorizer == null) {
        ZKConfig zkConfig = new ZKConfig(new VerifiableProperties(properties));

        Map<String, Object> authorizerProps = new HashMap<>();
        authorizerProps.put(ZKConfig.ZkConnectProp(), zkConfig.zkConnect());
        authorizerProps.put(ZKConfig.ZkConnectionTimeoutMsProp(), zkConfig.zkConnectionTimeoutMs());
        authorizerProps.put(ZKConfig.ZkSessionTimeoutMsProp(), zkConfig.zkSessionTimeoutMs());
        authorizerProps.put(ZKConfig.ZkSyncTimeMsProp(), zkConfig.zkSyncTimeMs());

        try {
            Authorizer simpleAclAuthorizer = new SimpleAclAuthorizer();
            simpleAclAuthorizer.configure(authorizerProps);
            authorizer = simpleAclAuthorizer;
        } catch (ZkException | ZooKeeperClientException e) {
            throw new AdminOperationException("Unable to create authorizer", e);
        }
    }

    return authorizer;
}
 
Example #4
Source File: KafkaProducerServiceIntegrationTest.java    From vertx-kafka-service with Apache License 2.0 6 votes vote down vote up
private void consumeMessages() {
    final Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
    topicCountMap.put(TOPIC, 1);
    final StringDecoder decoder =
            new StringDecoder(new VerifiableProperties());
    final Map<String, List<KafkaStream<String, String>>> consumerMap =
            consumer.createMessageStreams(topicCountMap, decoder, decoder);
    final KafkaStream<String, String> stream =
            consumerMap.get(TOPIC).get(0);
    final ConsumerIterator<String, String> iterator = stream.iterator();

    Thread kafkaMessageReceiverThread = new Thread(
            () -> {
                while (iterator.hasNext()) {
                    String msg = iterator.next().message();
                    msg = msg == null ? "<null>" : msg;
                    System.out.println("got message: " + msg);
                    messagesReceived.add(msg);
                }
            },
            "kafkaMessageReceiverThread"
    );
    kafkaMessageReceiverThread.start();

}
 
Example #5
Source File: KafkaGraphiteMetricsReporter.java    From kafka-graphite with Apache License 2.0 6 votes vote down vote up
@Override
public synchronized void init(VerifiableProperties props) {
    if (!initialized) {
        KafkaMetricsConfig metricsConfig = new KafkaMetricsConfig(props);
        graphiteHost = props.getString("kafka.graphite.metrics.host", GRAPHITE_DEFAULT_HOST);
        graphitePort = props.getInt("kafka.graphite.metrics.port", GRAPHITE_DEFAULT_PORT);
        metricPrefix = props.getString("kafka.graphite.metrics.group", GRAPHITE_DEFAULT_PREFIX);
        String excludeRegex = props.getString("kafka.graphite.metrics.exclude.regex", null);
        metricDimensions = Dimension.fromProperties(props.props(), "kafka.graphite.dimension.enabled.");

        LOG.debug("Initialize GraphiteReporter [{},{},{}]", graphiteHost, graphitePort, metricPrefix);

        if (excludeRegex != null) {
            LOG.debug("Using regex [{}] for GraphiteReporter", excludeRegex);
            metricPredicate = new FilterMetricPredicate(excludeRegex);
        }
        reporter = buildGraphiteReporter();

        if (props.getBoolean("kafka.graphite.metrics.reporter.enabled", false)) {
            initialized = true;
            startReporter(metricsConfig.pollingIntervalSecs());
            LOG.debug("GraphiteReporter started.");
        }
    }
}
 
Example #6
Source File: StatsdMetricsReporter.java    From kafka-statsd-metrics2 with Apache License 2.0 6 votes vote down vote up
private void loadConfig(VerifiableProperties props) {
  enabled = props.getBoolean("external.kafka.statsd.reporter.enabled", false);
  host = props.getString("external.kafka.statsd.host", "localhost");
  port = props.getInt("external.kafka.statsd.port", 8125);
  prefix = props.getString("external.kafka.statsd.metrics.prefix", "");
  pollingPeriodInSeconds = props.getInt("kafka.metrics.polling.interval.secs", 10);
  metricDimensions = Dimension.fromProperties(props.props(), "external.kafka.statsd.dimension.enabled.");

  String excludeRegex = props.getString("external.kafka.statsd.metrics.exclude_regex", DEFAULT_EXCLUDE_REGEX);
  if (excludeRegex != null && excludeRegex.length() != 0) {
    metricPredicate = new ExcludeMetricPredicate(excludeRegex);
  } else {
    metricPredicate = MetricPredicate.ALL;
  }

  this.isTagEnabled = props.getBoolean("external.kafka.statsd.tag.enabled", true);
}
 
Example #7
Source File: KafkaMqCollect.java    From light_drtc with Apache License 2.0 6 votes vote down vote up
public void collectMq(){
	Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
       topicCountMap.put(Constants.kfTopic, new Integer(1));

       StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties());
       StringDecoder valueDecoder = new StringDecoder(new VerifiableProperties());

       Map<String, List<KafkaStream<String, String>>> consumerMap =
               consumer.createMessageStreams(topicCountMap,keyDecoder,valueDecoder);
       
       KafkaStream<String, String> stream = consumerMap.get(Constants.kfTopic).get(0);
       ConsumerIterator<String, String> it = stream.iterator();
       MessageAndMetadata<String, String> msgMeta;
       while (it.hasNext()){
       	msgMeta = it.next();
       	super.mqTimer.parseMqText(msgMeta.key(), msgMeta.message());
       	//System.out.println(msgMeta.key()+"\t"+msgMeta.message());
       }
}
 
Example #8
Source File: ConsumerGroupReporter.java    From kafka-metrics with Apache License 2.0 5 votes vote down vote up
@Override
public void init(VerifiableProperties props) {
    if (!initialized) {

        this.props = new Properties();
        if (props.containsKey(CONFIG_POLLING_INTERVAL)) {
            this.pollingIntervalSeconds = props.getInt(CONFIG_POLLING_INTERVAL);
        } else {
            this.pollingIntervalSeconds = 10;
        }

        this.brokerId = Integer.parseInt(props.getProperty("broker.id"));
        log.info("Building ConsumerGroupReporter: polling.interval=" + pollingIntervalSeconds);
        Enumeration<Object> keys = props.props().keys();
        while (keys.hasMoreElements()) {
            String key = keys.nextElement().toString();
            if (key.startsWith("kafka.metrics.")) {
                String subKey = key.substring(14);
                this.props.put(subKey, props.props().get(key));
                log.info("Building ConsumerGroupReporter: " + subKey + "=" + this.props.get(subKey));
            }
        }
        initialized = true;
        this.underlying = new X(Metrics.defaultRegistry());
        startReporter(pollingIntervalSeconds);

    }
}
 
Example #9
Source File: KafkaMessageReceiverImpl.java    From message-queue-client-framework with Apache License 2.0 5 votes vote down vote up
/**
 * Construction method.
 *
 * @param props param props
 * @param pool  receiver pool
 */
public KafkaMessageReceiverImpl(Properties props,
                                KafkaMessageReceiverPool<K, V> pool) {

    this.pool = pool;
    this.props = new VerifiableProperties(props);
    this.replicaBrokers = new LinkedHashMap<String, Integer>();
}
 
Example #10
Source File: TopicReporter.java    From kafka-metrics with Apache License 2.0 5 votes vote down vote up
public void init(VerifiableProperties kafkaConfig) {

        if (!initialized) {
            initialized = true;

            this.config = kafkaConfig.props();
            this.builder = forRegistry(Metrics.defaultRegistry());
            builder.configure(config);
            underlying = builder.build();
            startReporter(underlying.getPollingIntervaSeconds());
        }
    }
 
Example #11
Source File: JavaKafkaConsumerHighAPIHdfsImpl.java    From dk-fitting with Apache License 2.0 5 votes vote down vote up
public void run() {
    // 1. 指定Topic
    Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
    topicCountMap.put(this.topic, this.numThreads);

    // 2. 指定数据的解码器
    StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties());
    StringDecoder valueDecoder = new StringDecoder(new VerifiableProperties());

    // 3. 获取连接数据的迭代器对象集合
    /**
     * Key: Topic主题
     * Value: 对应Topic的数据流读取器,大小是topicCountMap中指定的topic大小
     */
    Map<String, List<KafkaStream<String, String>>> consumerMap = this.consumer.createMessageStreams(topicCountMap, keyDecoder, valueDecoder);

    // 4. 从返回结果中获取对应topic的数据流处理器
    List<KafkaStream<String, String>> streams = consumerMap.get(this.topic);

    // 5. 创建线程池
    this.executorPool = Executors.newFixedThreadPool(this.numThreads);


    // 6. 构建数据输出对象
    int threadNumber = 0;
    for (final KafkaStream<String, String> stream : streams) {
        this.executorPool.submit(new ConsumerKafkaStreamProcesser(stream, threadNumber));
        threadNumber++;
    }
}
 
Example #12
Source File: PulsarKafkaProducer.java    From pulsar with Apache License 2.0 5 votes vote down vote up
public <T> T newInstance(String key, Class<T> t, VerifiableProperties properties) {
    Class<?> c = null;
    try {
        c = Class.forName(key);
    } catch (ClassNotFoundException | NoClassDefFoundError e) {
        throw new IllegalArgumentException("class not found for :" + key);
    }
    if (c == null)
        return null;
    Object o = newInstance(c, properties);
    if (!t.isInstance(o)) {
        throw new IllegalArgumentException(c.getName() + " is not an instance of " + t.getName());
    }
    return t.cast(o);
}
 
Example #13
Source File: KafkaStatsdMetricsReporter.java    From kafka-statsd-reporter with MIT License 5 votes vote down vote up
@Override
public synchronized void init(VerifiableProperties props) {
	if (!initialized) {
		KafkaMetricsConfig metricsConfig = new KafkaMetricsConfig(props);
		statsdHost = props.getString("kafka.statsd.metrics.host", STATSD_DEFAULT_HOST).trim();
		statsdPort = props.getInt("kafka.statsd.metrics.port", STATSD_DEFAULT_PORT);
		statsdGroupPrefix = props.getString("kafka.statsd.metrics.group", STATSD_DEFAULT_PREFIX).trim();
           String regex = props.getString("kafka.statsd.metrics.exclude.regex", null);

           LOG.debug("Initialize StatsdReporter ["+statsdHost+","+statsdPort+","+statsdGroupPrefix+"]");

           if (regex != null) {
           	predicate = new RegexMetricPredicate(regex);
           }
           try {
           	reporter = new StatsdReporter(
           			Metrics.defaultRegistry(),
           			statsdGroupPrefix,
           			predicate,
           			statsdHost,
           			statsdPort,
           			Clock.defaultClock()
           			);
           } catch (IOException e) {
           	LOG.error("Unable to initialize StatsdReporter", e);
           }
           if (props.getBoolean("kafka.statsd.metrics.reporter.enabled", false)) {
           	initialized = true;
           	startReporter(metricsConfig.pollingIntervalSecs());
               LOG.debug("StatsdReporter started.");
           }
       }
   }
 
Example #14
Source File: AbstractKafkaAvroDeserializer.java    From MongoDb-Sink-Connector with Apache License 2.0 5 votes vote down vote up
protected KafkaAvroDeserializerConfig deserializerConfig(VerifiableProperties props) {
    try {
        return new KafkaAvroDeserializerConfig(props.props());
    } catch (io.confluent.common.config.ConfigException e) {
        throw new ConfigException(e.getMessage());
    }
}
 
Example #15
Source File: KafkaGraphiteMetricsReporterTest.java    From kafka-graphite with Apache License 2.0 5 votes vote down vote up
@Test
public void initStartStopWithPropertiesSet() {
    KafkaGraphiteMetricsReporter reporter = new KafkaGraphiteMetricsReporter();
    Properties properties = new Properties();
    properties.setProperty("kafka.graphite.metrics.exclude.regex", "xyz.*");
    properties.setProperty("kafka.graphite.metrics.reporter.enabled", "true");

    reporter.init(new VerifiableProperties(properties));

    reporter.startReporter(1L);
    reporter.stopReporter();
}
 
Example #16
Source File: KafkaBrokerReporter.java    From metrics-kafka with Apache License 2.0 5 votes vote down vote up
synchronized public void init(VerifiableProperties props) {
    if (!initialized) {
        this.props = props;
        props.props().put("metadata.broker.list", String.format("%s:%d", "localhost", props.getInt("port")));

        final KafkaMetricsConfig metricsConfig = new KafkaMetricsConfig(props);

        this.underlying = new TopicReporter(Metrics.defaultRegistry(),
                new ProducerConfig(props.props()),
                "broker%s".format(props.getString("broker.id")));
        initialized = true;
        startReporter(metricsConfig.pollingIntervalSecs());
    }
}
 
Example #17
Source File: JavaKafkaConsumerHighAPIESImpl.java    From dk-fitting with Apache License 2.0 5 votes vote down vote up
public void run() {
    // 1. 指定Topic
    Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
    topicCountMap.put(this.topic, this.numThreads);

    // 2. 指定数据的解码器
    StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties());
    StringDecoder valueDecoder = new StringDecoder(new VerifiableProperties());

    // 3. 获取连接数据的迭代器对象集合
    /**
     * Key: Topic主题
     * Value: 对应Topic的数据流读取器,大小是topicCountMap中指定的topic大小
     */
    Map<String, List<KafkaStream<String, String>>> consumerMap = this.consumer.createMessageStreams(topicCountMap,
            keyDecoder, valueDecoder);

    // 4. 从返回结果中获取对应topic的数据流处理器
    List<KafkaStream<String, String>> streams = consumerMap.get(this.topic);

    // 5. 创建线程池
    this.executorPool = Executors.newFixedThreadPool(this.numThreads);

    // 6. 构建数据输出对象
    int threadNumber = 0;
    for (final KafkaStream<String, String> stream : streams) {
        this.executorPool.submit(new ConsumerKafkaStreamProcesser(stream, threadNumber));
        threadNumber++;
    }
}
 
Example #18
Source File: JavaKafkaConsumerHighAPIHbaseImpl.java    From dk-fitting with Apache License 2.0 5 votes vote down vote up
public void run() {
    // 1. 指定Topic
    Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
    topicCountMap.put(this.topic, this.numThreads);

    // 2. 指定数据的解码器
    StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties());
    StringDecoder valueDecoder = new StringDecoder(new VerifiableProperties());

    // 3. 获取连接数据的迭代器对象集合
    /**
     * Key: Topic主题
     * Value: 对应Topic的数据流读取器,大小是topicCountMap中指定的topic大小
     */
    Map<String, List<KafkaStream<String, String>>> consumerMap = this.consumer.createMessageStreams(topicCountMap,
            keyDecoder, valueDecoder);

    // 4. 从返回结果中获取对应topic的数据流处理器
    List<KafkaStream<String, String>> streams = consumerMap.get(this.topic);

    // 5. 创建线程池
    this.executorPool = Executors.newFixedThreadPool(this.numThreads);

    // 6. 构建数据输出对象
    int threadNumber = 0;
    for (final KafkaStream<String, String> stream : streams) {
        this.executorPool.submit(new ConsumerKafkaStreamProcesser(stream, threadNumber));
        threadNumber++;
    }
}
 
Example #19
Source File: KafkaMessageProcessorIT.java    From mod-kafka with Apache License 2.0 5 votes vote down vote up
private void consumeMessages() {
    final Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
    topicCountMap.put(KafkaProperties.DEFAULT_TOPIC, 1);
    final StringDecoder decoder =
            new StringDecoder(new VerifiableProperties());
    final Map<String, List<KafkaStream<String, String>>> consumerMap =
            consumer.createMessageStreams(topicCountMap, decoder, decoder);
    final KafkaStream<String, String> stream =
            consumerMap.get(KafkaProperties.DEFAULT_TOPIC).get(0);
    final ConsumerIterator<String, String> iterator = stream.iterator();

    Thread kafkaMessageReceiverThread = new Thread(
            new Runnable() {
                @Override
                public void run() {
                    while (iterator.hasNext()) {
                        String msg = iterator.next().message();
                        msg = msg == null ? "<null>" : msg;
                        System.out.println("got message: " + msg);
                        messagesReceived.add(msg);
                    }
                }
            },
            "kafkaMessageReceiverThread"
    );
    kafkaMessageReceiverThread.start();

}
 
Example #20
Source File: KafkaTimelineMetricsReporterTest.java    From ambari-metrics with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
  @SuppressWarnings({ "rawtypes", "unchecked" })
  Gauge g = registry.newGauge(System.class, "gauge", gauge);
  Counter counter = registry.newCounter(System.class, "counter");
  Histogram histogram = registry.newHistogram(System.class, "histogram");
  Meter meter = registry.newMeter(System.class, "meter", "empty", TimeUnit.MILLISECONDS);
  Timer timer = registry.newTimer(System.class, "timer");
  list.add(g);
  list.add(counter);
  list.add(histogram);
  list.add(meter);
  list.add(timer);
  Properties properties = new Properties();
  properties.setProperty("zookeeper.connect", "localhost:2181");
  properties.setProperty("kafka.timeline.metrics.sendInterval", "5900");
  properties.setProperty("kafka.timeline.metrics.maxRowCacheSize", "10000");
  properties.setProperty("kafka.timeline.metrics.hosts", "localhost:6188");
  properties.setProperty("kafka.timeline.metrics.port", "6188");
  properties.setProperty("kafka.timeline.metrics.reporter.enabled", "true");
  properties.setProperty("external.kafka.metrics.exclude.prefix", "a.b.c");
  properties.setProperty("external.kafka.metrics.include.prefix", "a.b.c.d");
  properties.setProperty("external.kafka.metrics.include.regex", "a.b.c.*.f");
  properties.setProperty("kafka.timeline.metrics.instanceId", "cluster");
  properties.setProperty("kafka.timeline.metrics.set.instanceId", "false");
  props = new VerifiableProperties(properties);
}
 
Example #21
Source File: KafkaTimelineMetricsReporterTest.java    From ambari-metrics with Apache License 2.0 5 votes vote down vote up
@Test
public void testReporterStartStopHttps() {
  mockStatic(Metrics.class);
  EasyMock.expect(Metrics.defaultRegistry()).andReturn(registry).times(2);
  TimelineMetricsCache timelineMetricsCache = getTimelineMetricsCache(kafkaTimelineMetricsReporter);
  kafkaTimelineMetricsReporter.setMetricsCache(timelineMetricsCache);
  replay(Metrics.class, timelineMetricsCache);

  Properties properties = new Properties();
  properties.setProperty("zookeeper.connect", "localhost:2181");
  properties.setProperty("kafka.timeline.metrics.sendInterval", "5900");
  properties.setProperty("kafka.timeline.metrics.maxRowCacheSize", "10000");
  properties.setProperty("kafka.timeline.metrics.hosts", "localhost:6188");
  properties.setProperty("kafka.timeline.metrics.port", "6188");
  properties.setProperty("kafka.timeline.metrics.reporter.enabled", "true");
  properties.setProperty("external.kafka.metrics.exclude.prefix", "a.b.c");
  properties.setProperty("external.kafka.metrics.include.prefix", "a.b.c.d");
  properties.setProperty("external.kafka.metrics.include.regex", "a.b.c.*.f");
  properties.setProperty("kafka.timeline.metrics.protocol", "https");
  properties.setProperty("kafka.timeline.metrics.truststore.path", "");
  properties.setProperty("kafka.timeline.metrics.truststore.type", "");
  properties.setProperty("kafka.timeline.metrics.truststore.password", "");
  properties.setProperty("kafka.timeline.metrics.instanceId", "cluster");
  properties.setProperty("kafka.timeline.metrics.set.instanceId", "false");
  kafkaTimelineMetricsReporter.init(new VerifiableProperties(properties));
  kafkaTimelineMetricsReporter.stopReporter();
  verifyAll();
}
 
Example #22
Source File: StatsdMetricsReporterTest.java    From kafka-statsd-metrics2 with Apache License 2.0 5 votes vote down vote up
@Before
public void init() {
  properties = createMock(VerifiableProperties.class);
  expect(properties.props()).andReturn(new Properties());
  expect(properties.getInt("kafka.metrics.polling.interval.secs", 10)).andReturn(11);
  expect(properties.getString("external.kafka.statsd.host", "localhost")).andReturn("127.0.0.1");
  expect(properties.getInt("external.kafka.statsd.port", 8125)).andReturn(1234);
  expect(properties.getString("external.kafka.statsd.metrics.prefix", "")).andReturn("foo");
  expect(properties.getString("external.kafka.statsd.metrics.exclude_regex",
      StatsdMetricsReporter.DEFAULT_EXCLUDE_REGEX)).andReturn("foo");
  expect(properties.getBoolean("external.kafka.statsd.tag.enabled", true)).andReturn(false);
}
 
Example #23
Source File: StatsdMetricsReporter.java    From kafka-statsd-metrics2 with Apache License 2.0 5 votes vote down vote up
@Override
public synchronized void init(VerifiableProperties props) {
  loadConfig(props);
  if (enabled) {
    log.info("Reporter is enabled and starting...");
    startReporter(pollingPeriodInSeconds);
  } else {
    log.warn("Reporter is disabled");
  }
}
 
Example #24
Source File: KafkaPersistReader.java    From streams with Apache License 2.0 4 votes vote down vote up
@Override
public void startStream() {

  Properties props = new Properties();
  props.setProperty("serializer.encoding", "UTF8");

  ConsumerConfig consumerConfig = new ConsumerConfig(props);

  consumerConnector = Consumer.createJavaConsumerConnector(consumerConfig);

  Whitelist topics = new Whitelist(config.getTopic());
  VerifiableProperties vprops = new VerifiableProperties(props);

  inStreams = consumerConnector.createMessageStreamsByFilter(topics, 1, new StringDecoder(vprops), new StringDecoder(vprops));

  for (final KafkaStream stream : inStreams) {
    executor.submit(new KafkaPersistReaderTask(this, stream));
  }

}
 
Example #25
Source File: JSONKafkaSerializer.java    From opensoc-streaming with Apache License 2.0 4 votes vote down vote up
public JSONKafkaSerializer(VerifiableProperties props) {
	// Do Nothing. constructor needed by Storm
}
 
Example #26
Source File: KafkaTestPartitioner.java    From attic-apex-malhar with Apache License 2.0 4 votes vote down vote up
public KafkaTestPartitioner(VerifiableProperties props)
{
}
 
Example #27
Source File: UserPartitioner.java    From eagle with Apache License 2.0 4 votes vote down vote up
public UserPartitioner(VerifiableProperties prop) {
    this.prop = prop;
}
 
Example #28
Source File: KafkaAvroEncoder.java    From geowave with Apache License 2.0 4 votes vote down vote up
public KafkaAvroEncoder(final VerifiableProperties verifiableProperties) {
  // This constructor must be present to avoid runtime errors
}
 
Example #29
Source File: UserPartitioner.java    From Eagle with Apache License 2.0 4 votes vote down vote up
public UserPartitioner(VerifiableProperties prop){
	this.prop = prop;
}
 
Example #30
Source File: KafkaGraphiteMetricsReporterTest.java    From kafka-graphite with Apache License 2.0 4 votes vote down vote up
@Test
public void initWithoutPropertiesSet() {
    KafkaGraphiteMetricsReporter reporter = new KafkaGraphiteMetricsReporter();
    reporter.init(new VerifiableProperties());
}