kafka.utils.VerifiableProperties Java Examples
The following examples show how to use
kafka.utils.VerifiableProperties.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KafkaConsumer.java From incubator-iotdb with Apache License 2.0 | 6 votes |
private void consume() { /** * Specify the number of consumer thread */ Map<String, Integer> topicCountMap = new HashMap<>(); topicCountMap.put(Constant.TOPIC, Constant.CONSUMER_THREAD_NUM); /** * Specify data decoder */ StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties()); StringDecoder valueDecoder = new StringDecoder(new VerifiableProperties()); Map<String, List<KafkaStream<String, String>>> consumerMap = consumer .createMessageStreams(topicCountMap, keyDecoder, valueDecoder); List<KafkaStream<String, String>> streams = consumerMap.get(Constant.TOPIC); ExecutorService executor = Executors.newFixedThreadPool(Constant.CONSUMER_THREAD_NUM); for (final KafkaStream<String, String> stream : streams) { executor.submit(new KafkaConsumerThread(stream)); } }
Example #2
Source File: KafkaHttpMetricsReporter.java From kafka-http-metrics-reporter with Apache License 2.0 | 6 votes |
@Override public void init(VerifiableProperties verifiableProperties) { if (!initialized) { // get configured metrics from kafka KafkaMetricsConfig metricsConfig = new KafkaMetricsConfig(verifiableProperties); // get the configured properties from kafka to set the bindAddress and port. bindAddress = verifiableProperties.getProperty("kafka.http.metrics.host"); port = Integer.parseInt(verifiableProperties.getProperty("kafka.http.metrics.port")); enabled = Boolean.parseBoolean(verifiableProperties.getProperty("kafka.http.metrics.reporter.enabled")); // construct the Metrics Server metricsServer = new KafkaHttpMetricsServer(bindAddress, port); initialized = true; // call the method startReporter startReporter(metricsConfig.pollingIntervalSecs()); } else { LOG.error("Kafka Http Metrics Reporter already initialized"); } }
Example #3
Source File: KafkaAdminClient.java From common-kafka with Apache License 2.0 | 6 votes |
/** * Returns an {@link Authorizer} to make {@link Acl} requests * * @return an {@link Authorizer} to make {@link Acl} requests * * @throws AdminOperationException * if there is an issue creating the authorizer */ public Authorizer getAuthorizer() { if (authorizer == null) { ZKConfig zkConfig = new ZKConfig(new VerifiableProperties(properties)); Map<String, Object> authorizerProps = new HashMap<>(); authorizerProps.put(ZKConfig.ZkConnectProp(), zkConfig.zkConnect()); authorizerProps.put(ZKConfig.ZkConnectionTimeoutMsProp(), zkConfig.zkConnectionTimeoutMs()); authorizerProps.put(ZKConfig.ZkSessionTimeoutMsProp(), zkConfig.zkSessionTimeoutMs()); authorizerProps.put(ZKConfig.ZkSyncTimeMsProp(), zkConfig.zkSyncTimeMs()); try { Authorizer simpleAclAuthorizer = new SimpleAclAuthorizer(); simpleAclAuthorizer.configure(authorizerProps); authorizer = simpleAclAuthorizer; } catch (ZkException | ZooKeeperClientException e) { throw new AdminOperationException("Unable to create authorizer", e); } } return authorizer; }
Example #4
Source File: KafkaProducerServiceIntegrationTest.java From vertx-kafka-service with Apache License 2.0 | 6 votes |
private void consumeMessages() { final Map<String, Integer> topicCountMap = new HashMap<String, Integer>(); topicCountMap.put(TOPIC, 1); final StringDecoder decoder = new StringDecoder(new VerifiableProperties()); final Map<String, List<KafkaStream<String, String>>> consumerMap = consumer.createMessageStreams(topicCountMap, decoder, decoder); final KafkaStream<String, String> stream = consumerMap.get(TOPIC).get(0); final ConsumerIterator<String, String> iterator = stream.iterator(); Thread kafkaMessageReceiverThread = new Thread( () -> { while (iterator.hasNext()) { String msg = iterator.next().message(); msg = msg == null ? "<null>" : msg; System.out.println("got message: " + msg); messagesReceived.add(msg); } }, "kafkaMessageReceiverThread" ); kafkaMessageReceiverThread.start(); }
Example #5
Source File: KafkaGraphiteMetricsReporter.java From kafka-graphite with Apache License 2.0 | 6 votes |
@Override public synchronized void init(VerifiableProperties props) { if (!initialized) { KafkaMetricsConfig metricsConfig = new KafkaMetricsConfig(props); graphiteHost = props.getString("kafka.graphite.metrics.host", GRAPHITE_DEFAULT_HOST); graphitePort = props.getInt("kafka.graphite.metrics.port", GRAPHITE_DEFAULT_PORT); metricPrefix = props.getString("kafka.graphite.metrics.group", GRAPHITE_DEFAULT_PREFIX); String excludeRegex = props.getString("kafka.graphite.metrics.exclude.regex", null); metricDimensions = Dimension.fromProperties(props.props(), "kafka.graphite.dimension.enabled."); LOG.debug("Initialize GraphiteReporter [{},{},{}]", graphiteHost, graphitePort, metricPrefix); if (excludeRegex != null) { LOG.debug("Using regex [{}] for GraphiteReporter", excludeRegex); metricPredicate = new FilterMetricPredicate(excludeRegex); } reporter = buildGraphiteReporter(); if (props.getBoolean("kafka.graphite.metrics.reporter.enabled", false)) { initialized = true; startReporter(metricsConfig.pollingIntervalSecs()); LOG.debug("GraphiteReporter started."); } } }
Example #6
Source File: StatsdMetricsReporter.java From kafka-statsd-metrics2 with Apache License 2.0 | 6 votes |
private void loadConfig(VerifiableProperties props) { enabled = props.getBoolean("external.kafka.statsd.reporter.enabled", false); host = props.getString("external.kafka.statsd.host", "localhost"); port = props.getInt("external.kafka.statsd.port", 8125); prefix = props.getString("external.kafka.statsd.metrics.prefix", ""); pollingPeriodInSeconds = props.getInt("kafka.metrics.polling.interval.secs", 10); metricDimensions = Dimension.fromProperties(props.props(), "external.kafka.statsd.dimension.enabled."); String excludeRegex = props.getString("external.kafka.statsd.metrics.exclude_regex", DEFAULT_EXCLUDE_REGEX); if (excludeRegex != null && excludeRegex.length() != 0) { metricPredicate = new ExcludeMetricPredicate(excludeRegex); } else { metricPredicate = MetricPredicate.ALL; } this.isTagEnabled = props.getBoolean("external.kafka.statsd.tag.enabled", true); }
Example #7
Source File: KafkaMqCollect.java From light_drtc with Apache License 2.0 | 6 votes |
public void collectMq(){ Map<String, Integer> topicCountMap = new HashMap<String, Integer>(); topicCountMap.put(Constants.kfTopic, new Integer(1)); StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties()); StringDecoder valueDecoder = new StringDecoder(new VerifiableProperties()); Map<String, List<KafkaStream<String, String>>> consumerMap = consumer.createMessageStreams(topicCountMap,keyDecoder,valueDecoder); KafkaStream<String, String> stream = consumerMap.get(Constants.kfTopic).get(0); ConsumerIterator<String, String> it = stream.iterator(); MessageAndMetadata<String, String> msgMeta; while (it.hasNext()){ msgMeta = it.next(); super.mqTimer.parseMqText(msgMeta.key(), msgMeta.message()); //System.out.println(msgMeta.key()+"\t"+msgMeta.message()); } }
Example #8
Source File: ConsumerGroupReporter.java From kafka-metrics with Apache License 2.0 | 5 votes |
@Override public void init(VerifiableProperties props) { if (!initialized) { this.props = new Properties(); if (props.containsKey(CONFIG_POLLING_INTERVAL)) { this.pollingIntervalSeconds = props.getInt(CONFIG_POLLING_INTERVAL); } else { this.pollingIntervalSeconds = 10; } this.brokerId = Integer.parseInt(props.getProperty("broker.id")); log.info("Building ConsumerGroupReporter: polling.interval=" + pollingIntervalSeconds); Enumeration<Object> keys = props.props().keys(); while (keys.hasMoreElements()) { String key = keys.nextElement().toString(); if (key.startsWith("kafka.metrics.")) { String subKey = key.substring(14); this.props.put(subKey, props.props().get(key)); log.info("Building ConsumerGroupReporter: " + subKey + "=" + this.props.get(subKey)); } } initialized = true; this.underlying = new X(Metrics.defaultRegistry()); startReporter(pollingIntervalSeconds); } }
Example #9
Source File: KafkaMessageReceiverImpl.java From message-queue-client-framework with Apache License 2.0 | 5 votes |
/** * Construction method. * * @param props param props * @param pool receiver pool */ public KafkaMessageReceiverImpl(Properties props, KafkaMessageReceiverPool<K, V> pool) { this.pool = pool; this.props = new VerifiableProperties(props); this.replicaBrokers = new LinkedHashMap<String, Integer>(); }
Example #10
Source File: TopicReporter.java From kafka-metrics with Apache License 2.0 | 5 votes |
public void init(VerifiableProperties kafkaConfig) { if (!initialized) { initialized = true; this.config = kafkaConfig.props(); this.builder = forRegistry(Metrics.defaultRegistry()); builder.configure(config); underlying = builder.build(); startReporter(underlying.getPollingIntervaSeconds()); } }
Example #11
Source File: JavaKafkaConsumerHighAPIHdfsImpl.java From dk-fitting with Apache License 2.0 | 5 votes |
public void run() { // 1. 指定Topic Map<String, Integer> topicCountMap = new HashMap<String, Integer>(); topicCountMap.put(this.topic, this.numThreads); // 2. 指定数据的解码器 StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties()); StringDecoder valueDecoder = new StringDecoder(new VerifiableProperties()); // 3. 获取连接数据的迭代器对象集合 /** * Key: Topic主题 * Value: 对应Topic的数据流读取器,大小是topicCountMap中指定的topic大小 */ Map<String, List<KafkaStream<String, String>>> consumerMap = this.consumer.createMessageStreams(topicCountMap, keyDecoder, valueDecoder); // 4. 从返回结果中获取对应topic的数据流处理器 List<KafkaStream<String, String>> streams = consumerMap.get(this.topic); // 5. 创建线程池 this.executorPool = Executors.newFixedThreadPool(this.numThreads); // 6. 构建数据输出对象 int threadNumber = 0; for (final KafkaStream<String, String> stream : streams) { this.executorPool.submit(new ConsumerKafkaStreamProcesser(stream, threadNumber)); threadNumber++; } }
Example #12
Source File: PulsarKafkaProducer.java From pulsar with Apache License 2.0 | 5 votes |
public <T> T newInstance(String key, Class<T> t, VerifiableProperties properties) { Class<?> c = null; try { c = Class.forName(key); } catch (ClassNotFoundException | NoClassDefFoundError e) { throw new IllegalArgumentException("class not found for :" + key); } if (c == null) return null; Object o = newInstance(c, properties); if (!t.isInstance(o)) { throw new IllegalArgumentException(c.getName() + " is not an instance of " + t.getName()); } return t.cast(o); }
Example #13
Source File: KafkaStatsdMetricsReporter.java From kafka-statsd-reporter with MIT License | 5 votes |
@Override public synchronized void init(VerifiableProperties props) { if (!initialized) { KafkaMetricsConfig metricsConfig = new KafkaMetricsConfig(props); statsdHost = props.getString("kafka.statsd.metrics.host", STATSD_DEFAULT_HOST).trim(); statsdPort = props.getInt("kafka.statsd.metrics.port", STATSD_DEFAULT_PORT); statsdGroupPrefix = props.getString("kafka.statsd.metrics.group", STATSD_DEFAULT_PREFIX).trim(); String regex = props.getString("kafka.statsd.metrics.exclude.regex", null); LOG.debug("Initialize StatsdReporter ["+statsdHost+","+statsdPort+","+statsdGroupPrefix+"]"); if (regex != null) { predicate = new RegexMetricPredicate(regex); } try { reporter = new StatsdReporter( Metrics.defaultRegistry(), statsdGroupPrefix, predicate, statsdHost, statsdPort, Clock.defaultClock() ); } catch (IOException e) { LOG.error("Unable to initialize StatsdReporter", e); } if (props.getBoolean("kafka.statsd.metrics.reporter.enabled", false)) { initialized = true; startReporter(metricsConfig.pollingIntervalSecs()); LOG.debug("StatsdReporter started."); } } }
Example #14
Source File: AbstractKafkaAvroDeserializer.java From MongoDb-Sink-Connector with Apache License 2.0 | 5 votes |
protected KafkaAvroDeserializerConfig deserializerConfig(VerifiableProperties props) { try { return new KafkaAvroDeserializerConfig(props.props()); } catch (io.confluent.common.config.ConfigException e) { throw new ConfigException(e.getMessage()); } }
Example #15
Source File: KafkaGraphiteMetricsReporterTest.java From kafka-graphite with Apache License 2.0 | 5 votes |
@Test public void initStartStopWithPropertiesSet() { KafkaGraphiteMetricsReporter reporter = new KafkaGraphiteMetricsReporter(); Properties properties = new Properties(); properties.setProperty("kafka.graphite.metrics.exclude.regex", "xyz.*"); properties.setProperty("kafka.graphite.metrics.reporter.enabled", "true"); reporter.init(new VerifiableProperties(properties)); reporter.startReporter(1L); reporter.stopReporter(); }
Example #16
Source File: KafkaBrokerReporter.java From metrics-kafka with Apache License 2.0 | 5 votes |
synchronized public void init(VerifiableProperties props) { if (!initialized) { this.props = props; props.props().put("metadata.broker.list", String.format("%s:%d", "localhost", props.getInt("port"))); final KafkaMetricsConfig metricsConfig = new KafkaMetricsConfig(props); this.underlying = new TopicReporter(Metrics.defaultRegistry(), new ProducerConfig(props.props()), "broker%s".format(props.getString("broker.id"))); initialized = true; startReporter(metricsConfig.pollingIntervalSecs()); } }
Example #17
Source File: JavaKafkaConsumerHighAPIESImpl.java From dk-fitting with Apache License 2.0 | 5 votes |
public void run() { // 1. 指定Topic Map<String, Integer> topicCountMap = new HashMap<String, Integer>(); topicCountMap.put(this.topic, this.numThreads); // 2. 指定数据的解码器 StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties()); StringDecoder valueDecoder = new StringDecoder(new VerifiableProperties()); // 3. 获取连接数据的迭代器对象集合 /** * Key: Topic主题 * Value: 对应Topic的数据流读取器,大小是topicCountMap中指定的topic大小 */ Map<String, List<KafkaStream<String, String>>> consumerMap = this.consumer.createMessageStreams(topicCountMap, keyDecoder, valueDecoder); // 4. 从返回结果中获取对应topic的数据流处理器 List<KafkaStream<String, String>> streams = consumerMap.get(this.topic); // 5. 创建线程池 this.executorPool = Executors.newFixedThreadPool(this.numThreads); // 6. 构建数据输出对象 int threadNumber = 0; for (final KafkaStream<String, String> stream : streams) { this.executorPool.submit(new ConsumerKafkaStreamProcesser(stream, threadNumber)); threadNumber++; } }
Example #18
Source File: JavaKafkaConsumerHighAPIHbaseImpl.java From dk-fitting with Apache License 2.0 | 5 votes |
public void run() { // 1. 指定Topic Map<String, Integer> topicCountMap = new HashMap<String, Integer>(); topicCountMap.put(this.topic, this.numThreads); // 2. 指定数据的解码器 StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties()); StringDecoder valueDecoder = new StringDecoder(new VerifiableProperties()); // 3. 获取连接数据的迭代器对象集合 /** * Key: Topic主题 * Value: 对应Topic的数据流读取器,大小是topicCountMap中指定的topic大小 */ Map<String, List<KafkaStream<String, String>>> consumerMap = this.consumer.createMessageStreams(topicCountMap, keyDecoder, valueDecoder); // 4. 从返回结果中获取对应topic的数据流处理器 List<KafkaStream<String, String>> streams = consumerMap.get(this.topic); // 5. 创建线程池 this.executorPool = Executors.newFixedThreadPool(this.numThreads); // 6. 构建数据输出对象 int threadNumber = 0; for (final KafkaStream<String, String> stream : streams) { this.executorPool.submit(new ConsumerKafkaStreamProcesser(stream, threadNumber)); threadNumber++; } }
Example #19
Source File: KafkaMessageProcessorIT.java From mod-kafka with Apache License 2.0 | 5 votes |
private void consumeMessages() { final Map<String, Integer> topicCountMap = new HashMap<String, Integer>(); topicCountMap.put(KafkaProperties.DEFAULT_TOPIC, 1); final StringDecoder decoder = new StringDecoder(new VerifiableProperties()); final Map<String, List<KafkaStream<String, String>>> consumerMap = consumer.createMessageStreams(topicCountMap, decoder, decoder); final KafkaStream<String, String> stream = consumerMap.get(KafkaProperties.DEFAULT_TOPIC).get(0); final ConsumerIterator<String, String> iterator = stream.iterator(); Thread kafkaMessageReceiverThread = new Thread( new Runnable() { @Override public void run() { while (iterator.hasNext()) { String msg = iterator.next().message(); msg = msg == null ? "<null>" : msg; System.out.println("got message: " + msg); messagesReceived.add(msg); } } }, "kafkaMessageReceiverThread" ); kafkaMessageReceiverThread.start(); }
Example #20
Source File: KafkaTimelineMetricsReporterTest.java From ambari-metrics with Apache License 2.0 | 5 votes |
@Before public void setUp() throws Exception { @SuppressWarnings({ "rawtypes", "unchecked" }) Gauge g = registry.newGauge(System.class, "gauge", gauge); Counter counter = registry.newCounter(System.class, "counter"); Histogram histogram = registry.newHistogram(System.class, "histogram"); Meter meter = registry.newMeter(System.class, "meter", "empty", TimeUnit.MILLISECONDS); Timer timer = registry.newTimer(System.class, "timer"); list.add(g); list.add(counter); list.add(histogram); list.add(meter); list.add(timer); Properties properties = new Properties(); properties.setProperty("zookeeper.connect", "localhost:2181"); properties.setProperty("kafka.timeline.metrics.sendInterval", "5900"); properties.setProperty("kafka.timeline.metrics.maxRowCacheSize", "10000"); properties.setProperty("kafka.timeline.metrics.hosts", "localhost:6188"); properties.setProperty("kafka.timeline.metrics.port", "6188"); properties.setProperty("kafka.timeline.metrics.reporter.enabled", "true"); properties.setProperty("external.kafka.metrics.exclude.prefix", "a.b.c"); properties.setProperty("external.kafka.metrics.include.prefix", "a.b.c.d"); properties.setProperty("external.kafka.metrics.include.regex", "a.b.c.*.f"); properties.setProperty("kafka.timeline.metrics.instanceId", "cluster"); properties.setProperty("kafka.timeline.metrics.set.instanceId", "false"); props = new VerifiableProperties(properties); }
Example #21
Source File: KafkaTimelineMetricsReporterTest.java From ambari-metrics with Apache License 2.0 | 5 votes |
@Test public void testReporterStartStopHttps() { mockStatic(Metrics.class); EasyMock.expect(Metrics.defaultRegistry()).andReturn(registry).times(2); TimelineMetricsCache timelineMetricsCache = getTimelineMetricsCache(kafkaTimelineMetricsReporter); kafkaTimelineMetricsReporter.setMetricsCache(timelineMetricsCache); replay(Metrics.class, timelineMetricsCache); Properties properties = new Properties(); properties.setProperty("zookeeper.connect", "localhost:2181"); properties.setProperty("kafka.timeline.metrics.sendInterval", "5900"); properties.setProperty("kafka.timeline.metrics.maxRowCacheSize", "10000"); properties.setProperty("kafka.timeline.metrics.hosts", "localhost:6188"); properties.setProperty("kafka.timeline.metrics.port", "6188"); properties.setProperty("kafka.timeline.metrics.reporter.enabled", "true"); properties.setProperty("external.kafka.metrics.exclude.prefix", "a.b.c"); properties.setProperty("external.kafka.metrics.include.prefix", "a.b.c.d"); properties.setProperty("external.kafka.metrics.include.regex", "a.b.c.*.f"); properties.setProperty("kafka.timeline.metrics.protocol", "https"); properties.setProperty("kafka.timeline.metrics.truststore.path", ""); properties.setProperty("kafka.timeline.metrics.truststore.type", ""); properties.setProperty("kafka.timeline.metrics.truststore.password", ""); properties.setProperty("kafka.timeline.metrics.instanceId", "cluster"); properties.setProperty("kafka.timeline.metrics.set.instanceId", "false"); kafkaTimelineMetricsReporter.init(new VerifiableProperties(properties)); kafkaTimelineMetricsReporter.stopReporter(); verifyAll(); }
Example #22
Source File: StatsdMetricsReporterTest.java From kafka-statsd-metrics2 with Apache License 2.0 | 5 votes |
@Before public void init() { properties = createMock(VerifiableProperties.class); expect(properties.props()).andReturn(new Properties()); expect(properties.getInt("kafka.metrics.polling.interval.secs", 10)).andReturn(11); expect(properties.getString("external.kafka.statsd.host", "localhost")).andReturn("127.0.0.1"); expect(properties.getInt("external.kafka.statsd.port", 8125)).andReturn(1234); expect(properties.getString("external.kafka.statsd.metrics.prefix", "")).andReturn("foo"); expect(properties.getString("external.kafka.statsd.metrics.exclude_regex", StatsdMetricsReporter.DEFAULT_EXCLUDE_REGEX)).andReturn("foo"); expect(properties.getBoolean("external.kafka.statsd.tag.enabled", true)).andReturn(false); }
Example #23
Source File: StatsdMetricsReporter.java From kafka-statsd-metrics2 with Apache License 2.0 | 5 votes |
@Override public synchronized void init(VerifiableProperties props) { loadConfig(props); if (enabled) { log.info("Reporter is enabled and starting..."); startReporter(pollingPeriodInSeconds); } else { log.warn("Reporter is disabled"); } }
Example #24
Source File: KafkaPersistReader.java From streams with Apache License 2.0 | 4 votes |
@Override public void startStream() { Properties props = new Properties(); props.setProperty("serializer.encoding", "UTF8"); ConsumerConfig consumerConfig = new ConsumerConfig(props); consumerConnector = Consumer.createJavaConsumerConnector(consumerConfig); Whitelist topics = new Whitelist(config.getTopic()); VerifiableProperties vprops = new VerifiableProperties(props); inStreams = consumerConnector.createMessageStreamsByFilter(topics, 1, new StringDecoder(vprops), new StringDecoder(vprops)); for (final KafkaStream stream : inStreams) { executor.submit(new KafkaPersistReaderTask(this, stream)); } }
Example #25
Source File: JSONKafkaSerializer.java From opensoc-streaming with Apache License 2.0 | 4 votes |
public JSONKafkaSerializer(VerifiableProperties props) { // Do Nothing. constructor needed by Storm }
Example #26
Source File: KafkaTestPartitioner.java From attic-apex-malhar with Apache License 2.0 | 4 votes |
public KafkaTestPartitioner(VerifiableProperties props) { }
Example #27
Source File: UserPartitioner.java From eagle with Apache License 2.0 | 4 votes |
public UserPartitioner(VerifiableProperties prop) { this.prop = prop; }
Example #28
Source File: KafkaAvroEncoder.java From geowave with Apache License 2.0 | 4 votes |
public KafkaAvroEncoder(final VerifiableProperties verifiableProperties) { // This constructor must be present to avoid runtime errors }
Example #29
Source File: UserPartitioner.java From Eagle with Apache License 2.0 | 4 votes |
public UserPartitioner(VerifiableProperties prop){ this.prop = prop; }
Example #30
Source File: KafkaGraphiteMetricsReporterTest.java From kafka-graphite with Apache License 2.0 | 4 votes |
@Test public void initWithoutPropertiesSet() { KafkaGraphiteMetricsReporter reporter = new KafkaGraphiteMetricsReporter(); reporter.init(new VerifiableProperties()); }