Java tutorial
/** Copyright [2013] [Mushroom] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ /** Notice : this source is extracted from Hadoop metric2 package and some source code may changed by zavakid */ package com.zavakid.mushroom.impl; import java.io.StringWriter; import java.lang.reflect.InvocationHandler; import java.lang.reflect.Method; import java.lang.reflect.Proxy; import java.net.InetAddress; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Timer; import java.util.TimerTask; import javax.management.ObjectName; import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.commons.math.util.MathUtils; import com.zavakid.mushroom.MetricsBuilder; import com.zavakid.mushroom.MetricsException; import com.zavakid.mushroom.MetricsFilter; import com.zavakid.mushroom.MetricsRecordBuilder; import com.zavakid.mushroom.MetricsSink; import com.zavakid.mushroom.MetricsSource; import com.zavakid.mushroom.MetricsSystem; import com.zavakid.mushroom.MetricsTag; import com.zavakid.mushroom.lib.MetricMutableCounterLong; import com.zavakid.mushroom.lib.MetricMutableStat; import com.zavakid.mushroom.util.Contracts; import com.zavakid.mushroom.util.MBeans; /** * A base class for metrics system singletons * * @author Hadoop metric2 package's authors * @since 0.1 */ public class MetricsSystemImpl implements MetricsSystem { private static final Log LOG = LogFactory.getLog(MetricsSystemImpl.class); static final String MS_CONTEXT = "metricssystem"; static final String NUM_SOURCES_KEY = "num_sources"; static final String NUM_SOURCES_DESC = "Number of metrics sources"; static final String NUM_SINKS_KEY = "num_sinks"; static final String NUM_SINKS_DESC = "Number of metrics sinks"; static final String MS_NAME = "MetricsSystem"; static final String MS_STATS_NAME = MS_NAME + ",sub=Stats"; static final String MS_STATS_DESC = "Metrics system metrics"; static final String MS_CONTROL_NAME = MS_NAME + ",sub=Control"; private final Map<String, MetricsSourceAdapter> sources; private final Map<String, MetricsSinkAdapter> sinks; private final List<Callback> callbacks; private final MetricsBuilderImpl metricsBuilder; private final MetricMutableStat snapshotStat = new MetricMutableStat("snapshot", "snapshot stats", "ops", "time", true); private final MetricMutableStat publishStat = new MetricMutableStat("publish", "publishing stats", "ops", "time", true); private final MetricMutableCounterLong dropStat = new MetricMutableCounterLong("dropped_pub_all", "number of dropped updates by all sinks", 0L); private final List<MetricsTag> injectedTags; // Things that are changed by init()/start()/stop() private String prefix; private MetricsFilter sourceFilter; private MetricsConfig config; private Map<String, MetricsConfig> sourceConfigs, sinkConfigs; private boolean monitoring = false; private Timer timer; private int period; // seconds private long logicalTime; // number of timer invocations * period private ObjectName mbeanName; private boolean publishSelfMetrics = true; private MetricsSourceAdapter sysSource; /** * Construct the metrics system * * @param prefix for the system */ public MetricsSystemImpl(String prefix) { this.prefix = prefix; sources = new LinkedHashMap<String, MetricsSourceAdapter>(); sinks = new LinkedHashMap<String, MetricsSinkAdapter>(); sourceConfigs = new HashMap<String, MetricsConfig>(); sinkConfigs = new HashMap<String, MetricsConfig>(); callbacks = new ArrayList<Callback>(); injectedTags = new ArrayList<MetricsTag>(); metricsBuilder = new MetricsBuilderImpl(); if (prefix != null) { // prefix could be null for default ctor, which requires init later initSystemMBean(); } } /** * Construct the system but not initializing (read config etc.) it. */ public MetricsSystemImpl() { this(null); } /** * Initialized the metrics system with a prefix. * * @param prefix the system will look for configs with the prefix */ public synchronized void init(String prefix) { if (monitoring) { LOG.warn(this.prefix + " metrics system already initialized!"); return; } this.prefix = Contracts.checkNotNull(prefix, "prefix"); try { start(); } catch (MetricsConfigException e) { // Usually because hadoop-metrics2.properties is missing // We can always start the metrics system later via JMX. LOG.warn("Metrics system not started: " + e.getMessage()); LOG.debug("Stacktrace: ", e); } initSystemMBean(); } @Override public synchronized void start() { Contracts.checkNotNull(prefix, "prefix"); if (monitoring) { LOG.warn(prefix + " metrics system already started!", new MetricsException("Illegal start")); return; } for (Callback cb : callbacks) cb.preStart(); configure(prefix); startTimer(); monitoring = true; LOG.info(prefix + " metrics system started"); for (Callback cb : callbacks) cb.postStart(); } @Override public synchronized void stop() { if (!monitoring) { LOG.warn(prefix + " metrics system not yet started!", new MetricsException("Illegal stop")); return; } for (Callback cb : callbacks) cb.preStop(); LOG.info("Stopping " + prefix + " metrics system..."); stopTimer(); stopSources(); stopSinks(); clearConfigs(); monitoring = false; LOG.info(prefix + " metrics system stopped."); for (Callback cb : callbacks) cb.postStop(); } @Override public synchronized <T extends MetricsSource> T register(final String name, final String desc, final T source) { if (monitoring) { registerSource(name, desc, source); } // We want to re-register the source to pick up new config when the // metrics system restarts. register(new AbstractCallback() { @Override public void postStart() { registerSource(name, desc, source); } }); return source; } @Override public synchronized <T extends MetricsSource> T registerIfAbsent(final String name, final String desc, final T source) { MetricsSource oldSource = null; if (monitoring) { oldSource = registerSource(name, desc, source); } // the source is the oldSource, means that the source register success if (source == oldSource) { register(new AbstractCallback() { @Override public void postStart() { registerSource(name, desc, source); } }); } return (T) oldSource; } synchronized MetricsSource registerSource(String name, String desc, MetricsSource source) { Contracts.checkNotNull(config, "config"); MetricsSourceAdapter sa = sources.get(name); if (sa != null) { LOG.warn("Source name " + name + " already exists!"); return sa.source(); } MetricsConfig conf = sourceConfigs.get(name); sa = conf != null ? new MetricsSourceAdapter(prefix, name, desc, source, injectedTags, period, conf) : new MetricsSourceAdapter(prefix, name, desc, source, injectedTags, period, config.subset(MetricsConfig.SOURCE_KEY)); sources.put(name, sa); sa.start(); LOG.debug("Registered source " + name); return sa.source(); } @Override public synchronized <T extends MetricsSink> T register(final String name, final String description, final T sink) { if (config != null) { registerSink(name, description, sink); } // We want to re-register the sink to pick up new config // when the metrics system restarts. register(new AbstractCallback() { @Override public void postStart() { registerSink(name, description, sink); } }); return sink; } @Override public synchronized <T extends MetricsSink> T registerIfAbsent(final String name, final String desc, final T sink) { MetricsSink oldSink = null; if (monitoring) { oldSink = registerSink(name, desc, sink); } // the sink is the oldSource, means that the source register success if (sink == oldSink) { register(new AbstractCallback() { @Override public void postStart() { registerSink(name, desc, sink); } }); } return (T) oldSink; } synchronized MetricsSink registerSink(String name, String desc, MetricsSink sink) { Contracts.checkNotNull(config, "config"); MetricsSinkAdapter sa = sinks.get(name); if (sa != null) { LOG.warn("Sink name " + name + " already exists!"); return sa.sink(); } MetricsConfig conf = sinkConfigs.get(name); sa = conf != null ? newSink(name, desc, sink, conf) : newSink(name, desc, sink, config.subset(MetricsConfig.SINK_KEY)); sinks.put(name, sa); sa.start(); LOG.debug("Registered sink " + name); return sa.sink(); } @Override public synchronized void register(final Callback callback) { callbacks.add((Callback) Proxy.newProxyInstance(callback.getClass().getClassLoader(), new Class<?>[] { Callback.class }, new InvocationHandler() { public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { try { return method.invoke(callback, args); } catch (Exception e) { LOG.warn("Caught exception in callback " + method.getName(), e); } return null; } })); } @Override public synchronized void refreshMBeans() { for (MetricsSourceAdapter sa : sources.values()) { sa.refreshMBean(); } } @Override public synchronized String currentConfig() { PropertiesConfiguration saver = new PropertiesConfiguration(); StringWriter writer = new StringWriter(); saver.copy(config); try { saver.save(writer); } catch (Exception e) { throw new MetricsConfigException("Error stringify config", e); } return writer.toString(); } private synchronized void startTimer() { if (timer != null) { LOG.warn(prefix + " metrics system timer already started!"); return; } logicalTime = 0; long millis = period * 1000; timer = new Timer("Timer for '" + prefix + "' metrics system", true); timer.scheduleAtFixedRate(new TimerTask() { public void run() { try { onTimerEvent(); } catch (Exception e) { LOG.warn(e); } } }, millis, millis); LOG.info("Scheduled snapshot period at " + period + " second(s)."); } synchronized void onTimerEvent() { logicalTime += period; if (sinks.size() > 0) { publishMetrics(snapshotMetrics()); } } /** * snapshot all the sources for a snapshot of metrics/tags * * @return the metrics buffer containing the snapshot */ synchronized MetricsBuffer snapshotMetrics() { metricsBuilder.clear(); MetricsBufferBuilder bufferBuilder = new MetricsBufferBuilder(); for (Entry<String, MetricsSourceAdapter> entry : sources.entrySet()) { if (sourceFilter == null || sourceFilter.accepts(entry.getKey())) { snapshotMetrics(entry.getValue(), bufferBuilder); } } if (publishSelfMetrics) { snapshotMetrics(sysSource, bufferBuilder); } MetricsBuffer buffer = bufferBuilder.get(); return buffer; } private void snapshotMetrics(MetricsSourceAdapter sa, MetricsBufferBuilder bufferBuilder) { long startTime = System.currentTimeMillis(); bufferBuilder.add(sa.name(), sa.getMetrics(metricsBuilder, false)); metricsBuilder.clear(); snapshotStat.add(System.currentTimeMillis() - startTime); LOG.debug("Snapshotted source " + sa.name()); } /** * Publish a metrics snapshot to all the sinks * * @param buffer the metrics snapshot to publish */ synchronized void publishMetrics(MetricsBuffer buffer) { int dropped = 0; for (MetricsSinkAdapter sa : sinks.values()) { long startTime = System.currentTimeMillis(); dropped += sa.putMetrics(buffer, logicalTime) ? 0 : 1; publishStat.add(System.currentTimeMillis() - startTime); } dropStat.incr(dropped); } private synchronized void stopTimer() { if (timer == null) { LOG.warn(prefix + " metrics system timer already stopped!"); return; } timer.cancel(); timer = null; } private synchronized void stopSources() { for (Entry<String, MetricsSourceAdapter> entry : sources.entrySet()) { MetricsSourceAdapter sa = entry.getValue(); LOG.info("Stopping metrics source " + entry.getKey() + "(" + sa.source().getClass().getName() + ")"); sa.stop(); } sysSource.stop(); sources.clear(); } private synchronized void stopSinks() { for (Entry<String, MetricsSinkAdapter> entry : sinks.entrySet()) { MetricsSinkAdapter sa = entry.getValue(); LOG.info("Stopping metrics sink " + entry.getKey() + "(" + sa.sink().getClass().getName() + ")"); sa.stop(); } sinks.clear(); } private synchronized void configure(String prefix) { config = MetricsConfig.create(prefix); configureSinks(); configureSources(); configureSystem(); } private synchronized void configureSystem() { injectedTags.add(new MetricsTag("hostName", "Local hostname", getHostname())); } private synchronized void configureSinks() { sinkConfigs = config.getInstanceConfigs(MetricsConfig.SINK_KEY); int confPeriod = 0; for (Entry<String, MetricsConfig> entry : sinkConfigs.entrySet()) { MetricsConfig conf = entry.getValue(); int sinkPeriod = conf.getInt(MetricsConfig.PERIOD_KEY, MetricsConfig.PERIOD_DEFAULT); confPeriod = confPeriod == 0 ? sinkPeriod : MathUtils.gcd(confPeriod, sinkPeriod); String sinkName = entry.getKey(); LOG.debug("sink " + sinkName + " config:\n" + conf); try { MetricsSinkAdapter sa = newSink(sinkName, conf.getString(MetricsConfig.DESC_KEY, sinkName), conf); // we allow config of later registered sinks if (sa != null) { sa.start(); sinks.put(sinkName, sa); } } catch (Exception e) { LOG.warn("Error creating " + sinkName, e); } } period = confPeriod > 0 ? confPeriod : config.getInt(MetricsConfig.PERIOD_KEY, MetricsConfig.PERIOD_DEFAULT); } static MetricsSinkAdapter newSink(String name, String desc, MetricsSink sink, MetricsConfig conf) { return new MetricsSinkAdapter(name, desc, sink, conf.getString(MetricsConfig.CONTEXT_KEY), conf.getFilter(MetricsConfig.SOURCE_FILTER_KEY), conf.getFilter(MetricsConfig.RECORD_FILTER_KEY), conf.getFilter(MetricsConfig.METRIC_FILTER_KEY), conf.getInt(MetricsConfig.PERIOD_KEY, MetricsConfig.PERIOD_DEFAULT), conf.getInt(MetricsConfig.QUEUE_CAPACITY_KEY, MetricsConfig.QUEUE_CAPACITY_DEFAULT), conf.getInt(MetricsConfig.RETRY_DELAY_KEY, MetricsConfig.RETRY_DELAY_DEFAULT), conf.getFloat(MetricsConfig.RETRY_BACKOFF_KEY, MetricsConfig.RETRY_BACKOFF_DEFAULT), conf.getInt(MetricsConfig.RETRY_COUNT_KEY, MetricsConfig.RETRY_COUNT_DEFAULT)); } static MetricsSinkAdapter newSink(String name, String desc, MetricsConfig conf) { MetricsSink sink = conf.getPlugin(""); if (sink == null) return null; return newSink(name, desc, sink, conf); } private void configureSources() { sourceFilter = config.getFilter(MetricsConfig.PREFIX_DEFAULT + MetricsConfig.SOURCE_FILTER_KEY); Map<String, MetricsConfig> confs = config.getInstanceConfigs(MetricsConfig.SOURCE_KEY); for (Entry<String, MetricsConfig> entry : confs.entrySet()) { sourceConfigs.put(entry.getKey(), entry.getValue()); } registerSystemSource(); } private void clearConfigs() { sinkConfigs.clear(); sourceConfigs.clear(); injectedTags.clear(); config = null; } static String getHostname() { try { return InetAddress.getLocalHost().getHostName(); } catch (Exception e) { LOG.error("Error getting localhost name. Using 'localhost'...", e); } return "localhost"; } private void registerSystemSource() { sysSource = new MetricsSourceAdapter(prefix, MS_STATS_NAME, MS_STATS_DESC, new MetricsSource() { @Override public void getMetrics(MetricsBuilder builder, boolean all) { int numSources, numSinks; synchronized (MetricsSystemImpl.this) { numSources = sources.size(); numSinks = sinks.size(); } MetricsRecordBuilder rb = builder.addRecord(MS_NAME).setContext(MS_CONTEXT) .addGauge(NUM_SOURCES_KEY, NUM_SOURCES_DESC, numSources) .addGauge(NUM_SINKS_KEY, NUM_SINKS_DESC, numSinks); synchronized (MetricsSystemImpl.this) { for (MetricsSinkAdapter sa : sinks.values()) { sa.snapshot(rb, all); } } snapshotStat.snapshot(rb, all); publishStat.snapshot(rb, all); dropStat.snapshot(rb, all); } }, injectedTags, null, null, period); sysSource.start(); } private void initSystemMBean() { Contracts.checkNotNull(prefix, "prefix should not be null here!"); mbeanName = MBeans.register(prefix, MS_CONTROL_NAME, this); } @Override public synchronized void shutdown() { if (monitoring) { try { stop(); } catch (Exception e) { LOG.warn("Error stopping the metrics system", e); } } MBeans.unregister(mbeanName); } }