Java tutorial
/* * (C) Copyright 2015-2016 Hewlett Packard Enterprise Development Company LP. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. * See the License for the specific language governing permissions and * limitations under the License. */ package monasca.thresh.utils; import java.io.IOException; import java.io.StringWriter; import java.lang.Boolean; import java.lang.String; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.storm.Config; import org.apache.storm.metric.api.IMetricsConsumer; import org.apache.storm.task.IErrorReporter; import org.apache.storm.task.TopologyContext; import com.fasterxml.jackson.core.JsonGenerationException; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.timgroup.statsd.NonBlockingUdpSender; import com.timgroup.statsd.StatsDClientErrorHandler; public class StatsdMetricConsumer implements IMetricsConsumer { public static final String STATSD_HOST = "metrics.statsd.host"; public static final String STATSD_PORT = "metrics.statsd.port"; public static final String STATSD_METRICMAP = "metrics.statsd.metricmap"; public static final String STATSD_WHITELIST = "metrics.statsd.whitelist"; public static final String STATSD_DIMENSIONS = "metrics.statsd.dimensions"; public static final String STATSD_DEBUGMETRICS = "metrics.statsd.debugmetrics"; private String topologyName; private String statsdHost = "localhost"; private int statsdPort = 8125; private String monascaStatsdDimPrefix = "|#"; private List<String> whiteList = new ArrayList<String>(); private Map<String, String> metricMap = new HashMap<String, String>(); private Boolean debugMetrics = false; String defaultDimensions = new StringBuilder().append(monascaStatsdDimPrefix) .append("{\"service\":\"monitoring\",\"component\":\"storm\"}").toString(); String statsdDimensions = defaultDimensions; /* * https://github.com/openstack/monasca-agent#statsd * * Example metric produced from this code from Monasca statsd * filtering-bolt.sendqueue.read_pos:69 * |c|#{"hostname":"localhost","service":"monitoring","component":"storm"} * * This is the Monasca specific string that adds the dimension element to * StatsD * |#{"hostname":"localhost","service":"monitoring","component":"storm"} * * To debug this code: * vi /usr/local/lib/python2.7/dist-packages/monasca_agent/statsd/udp.py * start():186 log.info('%s' % str(message)) * sudo service monasca-agent restart * tail -f /var/log/monasca/agent/statsd.log * /vagrant/tests/smoke.py * * Note: You only know that "|#" is a delimeter by looking at the Monasca * Python Agent code since the Monasca StatsD server is a derivative of what * the general purpose StatsD implements and it is executed in the Monasca * Agent which was forked from DataDog. It extends UDP data by postfixing a * json struct describing the dimensions. */ transient NonBlockingUdpSender udpclient; private transient StatsDClientErrorHandler handler; private transient Logger logger; @Override public void prepare(Map stormConf, Object registrationArgument, TopologyContext context, IErrorReporter errorReporter) { logger = LoggerFactory.getLogger(Logging.categoryFor(getClass(), context)); /* Sets up locals from the config STATSD_WHITELIST, STATSD_HOST ... */ parseConfig(stormConf); /* Sets up local vars from config vars if present */ if (registrationArgument instanceof Map) { parseConfig((Map<?, ?>) registrationArgument); } initClient(); logger.info("topologyName ({}), " + "clean(topologyName) ({})", new Object[] { topologyName, clean(topologyName) }); } private void initClient() { try { handler = statsdErrorHandler; udpclient = new NonBlockingUdpSender(statsdHost, statsdPort, Charset.defaultCharset(), handler); } catch (IOException e) { /* NonBlockingUdpSender only throws an IOException */ logger.error("{}", e); } catch (Exception e) { /* General purpose exception */ logger.error("{}", e); } } StatsDClientErrorHandler statsdErrorHandler = new StatsDClientErrorHandler() { @Override public void handle(Exception e) { logger.error("Error with StatsD UDP client! {}", e); } }; @SuppressWarnings("unchecked") void parseConfig(Map<?, ?> conf) { if (conf.containsKey(Config.TOPOLOGY_NAME)) { topologyName = (String) conf.get(Config.TOPOLOGY_NAME); } if (conf.containsKey(STATSD_HOST)) { statsdHost = (String) conf.get(STATSD_HOST); } if (conf.containsKey(STATSD_PORT)) { statsdPort = ((Number) conf.get(STATSD_PORT)).intValue(); } if (conf.containsKey(STATSD_DIMENSIONS)) { statsdDimensions = mapToJsonStr((Map<String, String>) conf.get(STATSD_DIMENSIONS)); if (!isValidJSON(statsdDimensions)) { logger.error("Ignoring dimensions element invalid JSON ({})", new Object[] { statsdDimensions }); // You get default dimensions statsdDimensions = monascaStatsdDimPrefix + defaultDimensions; } else { statsdDimensions = monascaStatsdDimPrefix + statsdDimensions; } } if (conf.containsKey(STATSD_WHITELIST)) { whiteList = (List<String>) conf.get(STATSD_WHITELIST); } if (conf.containsKey(STATSD_METRICMAP)) { metricMap = (Map<String, String>) conf.get(STATSD_METRICMAP); } if (conf.containsKey(STATSD_DEBUGMETRICS)) { debugMetrics = (Boolean) conf.get(STATSD_DEBUGMETRICS); } } private String mapToJsonStr(Map<String, String> inputMap) { String results = new String(); ObjectMapper mapper = new ObjectMapper(); StringWriter sw = new StringWriter(); try { mapper.writeValue(sw, inputMap); results = sw.toString(); } catch (JsonGenerationException e) { logger.error("{}", e); } catch (JsonMappingException e) { logger.error("{}", e); } catch (IOException e) { logger.error("{}", e); } return results; } private boolean isValidJSON(final String json) { boolean valid = false; try { final JsonParser parser = new ObjectMapper().getFactory().createParser(json); while (parser.nextToken() != null) { } valid = true; } catch (JsonParseException jpe) { valid = false; } catch (IOException ioe) { valid = false; } return valid; } String clean(String s) { /* storm metrics look pretty bad so cleanup is needed */ return s.replace('.', '_').replace('/', '_').replace(':', '_').replaceAll("__", ""); } @Override public void handleDataPoints(TaskInfo taskInfo, Collection<DataPoint> dataPoints) { for (Metric metric : dataPointsToMetrics(taskInfo, dataPoints)) { reportUOM(metric.name, metric.value); } } public static class Metric { String name; Double value; public Metric(String name, Double value) { this.name = name; this.value = value; } @Override public boolean equals(Object obj) { if (this == obj) return true; if (obj == null) return false; if (getClass() != obj.getClass()) return false; Metric other = (Metric) obj; if (name == null) { if (other.name != null) return false; } else if (!name.equals(other.name)) return false; if (value != other.value) return false; return true; } @Override public String toString() { return "Metric [name=" + name + ", value=" + value + "]"; } } private List<Metric> dataPointsToMetrics(TaskInfo taskInfo, Collection<DataPoint> dataPoints) { List<Metric> res = new LinkedList<>(); StringBuilder sb = new StringBuilder().append(clean(taskInfo.srcComponentId)).append("."); int hdrLength = sb.length(); for (DataPoint p : dataPoints) { sb.delete(hdrLength, sb.length()); sb.append(clean(p.name)); logger.debug("Storm StatsD metric p.name ({}) p.value ({})", new Object[] { p.name, p.value }); if (p.value instanceof Number) { res.add(new Metric(sb.toString(), ((Number) p.value).doubleValue())); } // There is a map of data points and it's not empty else if (p.value instanceof Map && !(((Map<?, ?>) (p.value)).isEmpty())) { int hdrAndNameLength = sb.length(); @SuppressWarnings("rawtypes") Map map = (Map) p.value; for (Object subName : map.keySet()) { Object subValue = map.get(subName); if (subValue instanceof Number) { sb.delete(hdrAndNameLength, sb.length()); sb.append(".").append(clean(subName.toString())); res.add(new Metric(sb.toString(), ((Number) subValue).doubleValue())); } } } } return res; } /* * Since the Java client doesn't support the Monasca metric type we need to * build it with a raw UDP request */ public void report(String s) { if (udpclient != null) { logger.debug("reporting: {}", s); udpclient.send(s); } else { /* Try to setup the UDP client since it was null */ initClient(); } } private void reportUOM(String s, Double number) { String metricName = null; StringBuilder results = new StringBuilder(); Boolean published = false; if (whiteList.contains(s)) { if (!metricMap.isEmpty() && metricMap.containsKey(s)) { metricName = metricMap.get(s); } /* Send the unmapped uom as the same name storm calls it */ else { metricName = s; } /* Make sure we don't send metric names that may be null or empty */ if (metricName != null && !metricName.isEmpty()) { published = true; } } /* * To enable debug message, you also need to add an entry like this: * * <logger name="monasca.thresh" additivity="false"> * <level value="INFO" /> * <appender-ref ref="A1" /> * </logger> * * Storm/Thresh logger config file: * /opt/storm/apache-storm-0.9.5/logback/cluster.xml * */ if (debugMetrics) { String mappedName = new String(); if (!metricMap.isEmpty() && metricMap.containsKey(s)) { mappedName = metricMap.get(s); } else { mappedName = s; } logger.info(", RawMetricName, {}, MappedMetricName, {}, " + "val, {}, {}", new Object[] { s, mappedName, number, published == true ? "PUBLISHED" : "UNPUBLISHED" }); } if (published) { results = results.append(metricName).append(":").append(String.valueOf(number)).append("|c") .append(statsdDimensions); report(results.toString()); } } @Override public void cleanup() { udpclient.stop(); } }