/*
* (C) Copyright 2015-2016 Hewlett Packard Enterprise Development Company LP.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package monasca.thresh.utils;
import java.io.IOException;
import java.io.StringWriter;
import java.lang.Boolean;
import java.lang.String;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.storm.Config;
import org.apache.storm.metric.api.IMetricsConsumer;
import org.apache.storm.task.IErrorReporter;
import org.apache.storm.task.TopologyContext;
import com.fasterxml.jackson.core.JsonGenerationException;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.timgroup.statsd.NonBlockingUdpSender;
import com.timgroup.statsd.StatsDClientErrorHandler;
public class StatsdMetricConsumer implements IMetricsConsumer {
public static final String STATSD_HOST = "metrics.statsd.host";
public static final String STATSD_PORT = "metrics.statsd.port";
public static final String STATSD_METRICMAP = "metrics.statsd.metricmap";
public static final String STATSD_WHITELIST = "metrics.statsd.whitelist";
public static final String STATSD_DIMENSIONS = "metrics.statsd.dimensions";
public static final String STATSD_DEBUGMETRICS = "metrics.statsd.debugmetrics";
private String topologyName;
private String statsdHost = "localhost";
private int statsdPort = 8125;
private String monascaStatsdDimPrefix = "|#";
private List<String> whiteList = new ArrayList<String>();
private Map<String, String> metricMap = new HashMap<String, String>();
private Boolean debugMetrics = false;
String defaultDimensions = new StringBuilder().append(monascaStatsdDimPrefix)
.append("{\"service\":\"monitoring\",\"component\":\"storm\"}")
.toString();
String statsdDimensions = defaultDimensions;
/*
* https://github.com/openstack/monasca-agent#statsd
*
* Example metric produced from this code from Monasca statsd
* filtering-bolt.sendqueue.read_pos:69
* |c|#{"hostname":"localhost","service":"monitoring","component":"storm"}
*
* This is the Monasca specific string that adds the dimension element to
* StatsD
* |#{"hostname":"localhost","service":"monitoring","component":"storm"}
*
* To debug this code:
* vi /usr/local/lib/python2.7/dist-packages/monasca_agent/statsd/udp.py
* start():186 log.info('%s' % str(message))
* sudo service monasca-agent restart
* tail -f /var/log/monasca/agent/statsd.log
* /vagrant/tests/smoke.py
*
* Note: You only know that "|#" is a delimeter by looking at the Monasca
* Python Agent code since the Monasca StatsD server is a derivative of what
* the general purpose StatsD implements and it is executed in the Monasca
* Agent which was forked from DataDog. It extends UDP data by postfixing a
* json struct describing the dimensions.
*/
transient NonBlockingUdpSender udpclient;
private transient StatsDClientErrorHandler handler;
private transient Logger logger;
@Override
public void prepare(Map stormConf, Object registrationArgument,
TopologyContext context, IErrorReporter errorReporter) {
logger = LoggerFactory.getLogger(Logging.categoryFor(getClass(), context));
/* Sets up locals from the config STATSD_WHITELIST, STATSD_HOST ... */
parseConfig(stormConf);
/* Sets up local vars from config vars if present */
if (registrationArgument instanceof Map) {
parseConfig((Map<?, ?>) registrationArgument);
}
initClient();
logger.info(
"topologyName ({}), "
+ "clean(topologyName) ({})", new Object[] { topologyName,
clean(topologyName) });
}
private void initClient() {
try {
handler = statsdErrorHandler;
udpclient = new NonBlockingUdpSender(statsdHost, statsdPort,
Charset.defaultCharset(), handler);
}
catch (IOException e) {
/* NonBlockingUdpSender only throws an IOException */
logger.error("{}", e);
}
catch (Exception e) {
/* General purpose exception */
logger.error("{}", e);
}
}
StatsDClientErrorHandler statsdErrorHandler = new StatsDClientErrorHandler() {
@Override
public void handle(Exception e) {
logger.error("Error with StatsD UDP client! {}", e);
}
};
@SuppressWarnings("unchecked")
void parseConfig(Map<?, ?> conf) {
if (conf.containsKey(Config.TOPOLOGY_NAME)) {
topologyName = (String) conf.get(Config.TOPOLOGY_NAME);
}
if (conf.containsKey(STATSD_HOST)) {
statsdHost = (String) conf.get(STATSD_HOST);
}
if (conf.containsKey(STATSD_PORT)) {
statsdPort = ((Number) conf.get(STATSD_PORT)).intValue();
}
if (conf.containsKey(STATSD_DIMENSIONS)) {
statsdDimensions = mapToJsonStr((Map<String, String>) conf
.get(STATSD_DIMENSIONS));
if (!isValidJSON(statsdDimensions)) {
logger.error("Ignoring dimensions element invalid JSON ({})",
new Object[] { statsdDimensions });
// You get default dimensions
statsdDimensions = monascaStatsdDimPrefix + defaultDimensions;
}
else {
statsdDimensions = monascaStatsdDimPrefix + statsdDimensions;
}
}
if (conf.containsKey(STATSD_WHITELIST)) {
whiteList = (List<String>) conf.get(STATSD_WHITELIST);
}
if (conf.containsKey(STATSD_METRICMAP)) {
metricMap = (Map<String, String>) conf.get(STATSD_METRICMAP);
}
if (conf.containsKey(STATSD_DEBUGMETRICS)) {
debugMetrics = (Boolean) conf.get(STATSD_DEBUGMETRICS);
}
}
private String mapToJsonStr(Map<String, String> inputMap) {
String results = new String();
ObjectMapper mapper = new ObjectMapper();
StringWriter sw = new StringWriter();
try {
mapper.writeValue(sw, inputMap);
results = sw.toString();
}
catch (JsonGenerationException e) {
logger.error("{}", e);
}
catch (JsonMappingException e) {
logger.error("{}", e);
}
catch (IOException e) {
logger.error("{}", e);
}
return results;
}
private boolean isValidJSON(final String json) {
boolean valid = false;
try {
final JsonParser parser = new ObjectMapper().getFactory().createParser(
json);
while (parser.nextToken() != null) {
}
valid = true;
}
catch (JsonParseException jpe) {
valid = false;
}
catch (IOException ioe) {
valid = false;
}
return valid;
}
String clean(String s) {
/* storm metrics look pretty bad so cleanup is needed */
return s.replace('.', '_').replace('/', '_').replace(':', '_')
.replaceAll("__", "");
}
@Override
public void handleDataPoints(TaskInfo taskInfo,
Collection<DataPoint> dataPoints) {
for (Metric metric : dataPointsToMetrics(taskInfo, dataPoints)) {
reportUOM(metric.name, metric.value);
}
}
public static class Metric {
String name;
Double value;
public Metric(String name, Double value) {
this.name = name;
this.value = value;
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
Metric other = (Metric) obj;
if (name == null) {
if (other.name != null)
return false;
}
else if (!name.equals(other.name))
return false;
if (value != other.value)
return false;
return true;
}
@Override
public String toString() {
return "Metric [name=" + name + ", value=" + value + "]";
}
}
private List<Metric> dataPointsToMetrics(TaskInfo taskInfo,
Collection<DataPoint> dataPoints) {
List<Metric> res = new LinkedList<>();
StringBuilder sb = new StringBuilder().append(
clean(taskInfo.srcComponentId)).append(".");
int hdrLength = sb.length();
for (DataPoint p : dataPoints) {
sb.delete(hdrLength, sb.length());
sb.append(clean(p.name));
logger.debug("Storm StatsD metric p.name ({}) p.value ({})",
new Object[] { p.name, p.value });
if (p.value instanceof Number) {
res.add(new Metric(sb.toString(), ((Number) p.value).doubleValue()));
}
// There is a map of data points and it's not empty
else if (p.value instanceof Map &&
!(((Map<?, ?>) (p.value)).isEmpty())) {
int hdrAndNameLength = sb.length();
@SuppressWarnings("rawtypes")
Map map = (Map) p.value;
for (Object subName : map.keySet()) {
Object subValue = map.get(subName);
if (subValue instanceof Number) {
sb.delete(hdrAndNameLength, sb.length());
sb.append(".").append(clean(subName.toString()));
res.add(new Metric(sb.toString(),
((Number) subValue).doubleValue()));
}
}
}
}
return res;
}
/*
* Since the Java client doesn't support the Monasca metric type we need to
* build it with a raw UDP request
*/
public void report(String s) {
if (udpclient != null) {
logger.debug("reporting: {}", s);
udpclient.send(s);
}
else {
/* Try to setup the UDP client since it was null */
initClient();
}
}
private void reportUOM(String s, Double number) {
String metricName = null;
StringBuilder results = new StringBuilder();
Boolean published = false;
if (whiteList.contains(s)) {
if (!metricMap.isEmpty() && metricMap.containsKey(s)) {
metricName = metricMap.get(s);
}
/* Send the unmapped uom as the same name storm calls it */
else {
metricName = s;
}
/* Make sure we don't send metric names that may be null or empty */
if (metricName != null && !metricName.isEmpty()) {
published = true;
}
}
/*
* To enable debug message, you also need to add an entry like this:
*
* <logger name="monasca.thresh" additivity="false">
* <level value="INFO" />
* <appender-ref ref="A1" />
* </logger>
*
* Storm/Thresh logger config file:
* /opt/storm/apache-storm-0.9.5/logback/cluster.xml
*
*/
if (debugMetrics) {
String mappedName = new String();
if (!metricMap.isEmpty() && metricMap.containsKey(s)) {
mappedName = metricMap.get(s);
}
else {
mappedName = s;
}
logger.info(", RawMetricName, {}, MappedMetricName, {}, "
+ "val, {}, {}", new Object[]
{ s, mappedName, number,
published == true ? "PUBLISHED" : "UNPUBLISHED"});
}
if (published) {
results = results.append(metricName).append(":")
.append(String.valueOf(number)).append("|c")
.append(statsdDimensions);
report(results.toString());
}
}
@Override
public void cleanup() {
udpclient.stop();
}
}
