Java tutorial
/* * Copyright 2013 LinkedIn Corp. All rights reserved * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package com.linkedin.databus.client.pub.mbean; import java.io.IOException; import java.io.OutputStream; import java.util.Hashtable; import java.util.List; import java.util.concurrent.locks.Lock; import javax.management.MBeanServer; import javax.management.MalformedObjectNameException; import javax.management.ObjectName; import org.apache.avro.io.JsonEncoder; import org.apache.avro.specific.SpecificDatumWriter; import org.apache.commons.math3.stat.StatUtils; import com.codahale.metrics.MergeableExponentiallyDecayingReservoir; // auto-generated by Avro from UnifiedClientStatsEvent.*.avsc: import com.linkedin.databus.client.pub.monitoring.events.UnifiedClientStatsEvent; import com.linkedin.databus.core.DbusClientMode; import com.linkedin.databus.core.DbusConstants; import com.linkedin.databus.core.DbusEvent; import com.linkedin.databus.core.monitoring.mbean.AbstractMonitoringMBean; import com.linkedin.databus.core.monitoring.mbean.DatabusMonitoringMBean; import com.linkedin.databus.core.monitoring.mbean.StatsCollectorMergeable; /** * Joint relay/bootstrap client-library metrics for Databus v2 and v3 consumers. At the lowest * level, these stats may represent a single table (in a v2 multi-tenant setup) or a single * partition for a single table (in a client load-balancing [CLB] setup); in the latter case, * they may not even be exposed. The same class is also used to aggregate stats over multiple * partitions and/or tables. Thus in the CLB case, for example, the lowest level is one partition * of a table; the second level is aggregated across all partitions for a single table; and the * third level is aggregated across all tables [for a single DB or for all subscriptions? TODO/FIXME]. * * This class is intended to supersede ConsumerCallbackStats for consumers. */ public class UnifiedClientStats extends AbstractMonitoringMBean<UnifiedClientStatsEvent> implements UnifiedClientStatsMBean, StatsCollectorMergeable<UnifiedClientStats> { // default threshold for lack of data events to be considered "idle": public static final int DEFAULT_DEADNESS_THRESHOLD_MS = 300000; private final String _name; private final String _dimension; // used in MBean/JMX naming to distinguish similar collectors; see mbeanProps below private final MBeanServer _mbeanServer; private final MergeableExponentiallyDecayingReservoir _reservoirTimeLagConsumerCallbacksMs; private MergeableExponentiallyDecayingReservoir _reservoirTimeLagSourceToReceiptMs; private long _deadnessThresholdMs; private boolean _isBootstrapping = false; // used only by tests: public UnifiedClientStats(int ownerId, String name, String dimension) { this(ownerId, name, dimension, true, false, DEFAULT_DEADNESS_THRESHOLD_MS, null, null); } public UnifiedClientStats(int ownerId, String name, String dimension, boolean enabled, boolean threadSafe, long deadnessThresholdMs, UnifiedClientStatsEvent initData) { this(ownerId, name, dimension, enabled, threadSafe, deadnessThresholdMs, initData, null); } public UnifiedClientStats(int ownerId, String name, String dimension, boolean enabled, boolean threadSafe, long deadnessThresholdMs, UnifiedClientStatsEvent initData, MBeanServer server) { super(enabled, threadSafe, initData); _event.ownerId = ownerId; _name = name; _dimension = dimension; _deadnessThresholdMs = deadnessThresholdMs; _mbeanServer = server; _reservoirTimeLagSourceToReceiptMs = new MergeableExponentiallyDecayingReservoir(); _reservoirTimeLagConsumerCallbacksMs = new MergeableExponentiallyDecayingReservoir(); resetData(); registerAsMbean(); } public void setBootstrappingState(boolean isBootstrapping) { if (!_enabled.get()) return; Lock writeLock = acquireWriteLock(); try { // timeLagSourceToReceiptMs is defined to be -1 when bootstrapping, which means there's no need even to // collect the data. If codahale's Reservoir interface supported a clear() method, we could just use that... if (!_isBootstrapping && isBootstrapping) { // online-consumption -> bootstrap transition: nuke _reservoirTimeLagSourceToReceiptMs _reservoirTimeLagSourceToReceiptMs = null; } else if (_isBootstrapping && !isBootstrapping) { // bootstrap -> online-consumption transition: resurrect _reservoirTimeLagSourceToReceiptMs _reservoirTimeLagSourceToReceiptMs = new MergeableExponentiallyDecayingReservoir(); } _isBootstrapping = isBootstrapping; _event.curBootstrappingPartitions = isBootstrapping ? 1 : 0; } finally { releaseLock(writeLock); } } public void setHeartbeatTimestamp(long heartbeatTimestamp) { if (!_enabled.get()) return; Lock writeLock = acquireWriteLock(); try { _event.timestampOfLastHeartbeatMs = heartbeatTimestamp; } finally { releaseLock(writeLock); } } // used only for tests (=> no need for corresponding getter) public void setDeadnessThresholdMs(long deadnessThresholdMs) { _deadnessThresholdMs = deadnessThresholdMs; } public String getDimension() { return _dimension; } public String getName() { return _name; } public MergeableExponentiallyDecayingReservoir getReservoirTimeLagSourceToReceiptMs() { return _reservoirTimeLagSourceToReceiptMs; } public MergeableExponentiallyDecayingReservoir getReservoirTimeLagConsumerCallbacksMs() { return _reservoirTimeLagConsumerCallbacksMs; } public void registerAsMbean() { super.registerAsMbean(_mbeanServer); } public void unregisterAsMbean() { super.unregisterMbean(_mbeanServer); } @Override public JsonEncoder createJsonEncoder(OutputStream out) throws IOException { return new JsonEncoder(_event.getSchema(), out); } @Override public ObjectName generateObjectName() throws MalformedObjectNameException { Hashtable<String, String> mbeanProps = generateBaseMBeanProps(); mbeanProps.put("ownerId", Integer.toString(_event.ownerId)); mbeanProps.put("dimension", _dimension); return new ObjectName(AbstractMonitoringMBean.JMX_DOMAIN, mbeanProps); } // called by ctor above (no lock required) and by reset() in superclass (which acquires write lock) @Override protected void resetData() { long now = System.currentTimeMillis(); _event.timestampLastResetMs = now; _event.aggregated = false; _event.curBootstrappingPartitions = 0; _event.curDeadConnections = 0; _event.numConsumerErrors = 0; _event.numDataEvents = 0; _event.timestampOfLastHeartbeatMs = now; _event.timestampLastDataEventWasReceivedMs = 0; // timestamp values are always stored in ms } // called only by getStatistics() in superclass (which acquires read lock) @Override protected void cloneData(UnifiedClientStatsEvent event) { // ConsumerCallbackStats doesn't do this... // event.timestampLastResetMs = _event.timestampLastResetMs; event.aggregated = _event.aggregated; event.curBootstrappingPartitions = _event.curBootstrappingPartitions; event.curDeadConnections = _event.curDeadConnections; event.numConsumerErrors = _event.numConsumerErrors; event.numDataEvents = _event.numDataEvents; event.timestampOfLastHeartbeatMs = _event.timestampOfLastHeartbeatMs; event.timestampLastDataEventWasReceivedMs = _event.timestampLastDataEventWasReceivedMs; } @Override protected UnifiedClientStatsEvent newDataEvent() { return new UnifiedClientStatsEvent(); } @Override protected SpecificDatumWriter<UnifiedClientStatsEvent> getAvroWriter() { return new SpecificDatumWriter<UnifiedClientStatsEvent>(UnifiedClientStatsEvent.class); } // called by superclass's (AbstractMonitoringMBean's) mergeStats() and by // StatsCollectors.mergeStatsCollectors() -> resetAndMerge() -> merge(); callers // handle locking @Override protected void doMergeStats(Object eventData) { if (!(eventData instanceof UnifiedClientStats)) { LOG.warn("Attempt to merge stats from unknown event class: " + eventData.getClass().getName()); return; } UnifiedClientStats otherEvent = (UnifiedClientStats) eventData; UnifiedClientStatsEvent e = otherEvent._event; _event.aggregated = true; // standalone metrics; aggregation = simple sum: _event.curBootstrappingPartitions += e.curBootstrappingPartitions; _event.numConsumerErrors += e.numConsumerErrors; _event.numDataEvents += e.numDataEvents; // special-case, half-standalone/half-derived metric: aggregation is slightly complicated... if (e.aggregated) { // we're the third (or higher) level up, so it's safe to trust the lower level's count; just add it in: _event.curDeadConnections += e.curDeadConnections; } else if (System.currentTimeMillis() - e.timestampOfLastHeartbeatMs > _deadnessThresholdMs) { // we're the second level (first level of aggregation), so we need to check the first level's timestamp // to see if its connection is dead ++_event.curDeadConnections; } // support metrics for timeLagLastReceivedToNowMs; since want worst case across aggregated time lags // (i.e., maximum interval), want _minimum_ (oldest) non-zero timestamp: if (_event.timestampLastDataEventWasReceivedMs == 0) { // other one is same or better, so assign unconditionally _event.timestampLastDataEventWasReceivedMs = e.timestampLastDataEventWasReceivedMs; } else if (e.timestampLastDataEventWasReceivedMs > 0) { // both are non-zero, so assign minimum _event.timestampLastDataEventWasReceivedMs = Math.min(_event.timestampLastDataEventWasReceivedMs, e.timestampLastDataEventWasReceivedMs); } // else e.timestampLastDataEventWasReceivedMs == 0, so ignore it // Support for timeLagSourceToReceiptMs histogram metrics, which uses exponentially weighted statistical // sampling; aggregation is tricky (see MergeableExponentiallyDecayingReservoir for details). Special // cases: (1) if otherEvent is bootstrapping, its getReservoirTimeLagSourceToReceiptMs() will be null, so // merge() will return immediately; (2) if no data events yet received, e.timestampLastDataEventWasReceivedMs // will be zero, so skip merge in that case. if (e.timestampLastDataEventWasReceivedMs > 0) { // we're an aggregate, so _reservoirTimeLagSourceToReceiptMs should never be null (aggregates can't bootstrap) _reservoirTimeLagSourceToReceiptMs.merge(otherEvent.getReservoirTimeLagSourceToReceiptMs()); } // Support for timeLagConsumerCallbacksMs histogram metrics, which uses exponentially weighted statistical // sampling; aggregation is tricky (see MergeableExponentiallyDecayingReservoir for details). No special // handling for bootstrap mode is needed. If no callbacks have occurred, reservoirs will be empty, and all // percentile values will be zero (see getTimeLagConsumerCallbacksMs_HistPct() below). _reservoirTimeLagConsumerCallbacksMs.merge(otherEvent.getReservoirTimeLagConsumerCallbacksMs()); } @Override public void merge(UnifiedClientStats obj) { if (!_enabled.get()) return; Lock writeLock = acquireWriteLock(); try { doMergeStats(obj); } finally { releaseLock(writeLock); } } // called by StatsCollectors.mergeStatsCollectors() @Override public void resetAndMerge(List<UnifiedClientStats> objList) { Lock writeLock = acquireWriteLock(); try { reset(); for (UnifiedClientStats t : objList) { merge(t); } } finally { releaseLock(writeLock); } } // "Rich" copy of default impl in AbstractMonitoringMBean: never pass UnifiedClientStatsEvent, // only UnifiedClientStats (which contains UnifiedClientStatsEvent _event). We need this so // we can merge the internal reservoir (array) objects efficiently; we _really_ don't want to // convert and copy them to pass around inside _event. (Histograms are so fun...) @Override public void mergeStats(DatabusMonitoringMBean<UnifiedClientStatsEvent> other) { if (!(other instanceof UnifiedClientStats)) return; UnifiedClientStats otherObj = (UnifiedClientStats) other; Lock otherReadLock = otherObj.acquireReadLock(); Lock thisWriteLock = null; try { thisWriteLock = acquireWriteLock(otherReadLock); doMergeStats(otherObj); } finally { releaseLock(thisWriteLock); releaseLock(otherReadLock); } } // We use "cur" (a.k.a. "current") instead of "num" to prevent MBeanSensorHelper from treating this // metric as an RRD counter. (We want gauge instead.) @Override public int getCurBootstrappingPartitions() { int result = 0; Lock readLock = acquireReadLock(); try { result = _event.curBootstrappingPartitions; } finally { releaseLock(readLock); } return result; } // We use "cur" (a.k.a. "current") instead of "num" to prevent MBeanSensorHelper from treating this // metric as an RRD counter. (We want gauge instead.) @Override public int getCurDeadConnections() { int result = 0; Lock readLock = acquireReadLock(); try { // For the lowest-level (non-aggregated) stats, which we detect via register*() calls, this getter is the // most reasonable place to check whether we're dead. But it may never be called (or only rarely), so we // can't rely on its calculation of curDeadConnections to compute the aggregated versions. (And we // certainly can't depend on further register* calls if we're dead.) Ergo, for aggregated versions, do a // separate calculation in doMergeStats(). if (!_event.aggregated) { long timeIntervalSinceHeartbeatMs = System.currentTimeMillis() - _event.timestampOfLastHeartbeatMs; _event.curDeadConnections = (timeIntervalSinceHeartbeatMs > _deadnessThresholdMs) ? 1 : 0; } result = _event.curDeadConnections; } finally { releaseLock(readLock); } return result; } @Override public long getNumConsumerErrors() { long result = 0; Lock readLock = acquireReadLock(); try { result = _event.numConsumerErrors; } finally { releaseLock(readLock); } return result; } @Override public long getNumDataEvents() { long result = 0; Lock readLock = acquireReadLock(); try { result = _event.numDataEvents; } finally { releaseLock(readLock); } return result; } // timeLagSourceToReceiptMs // - this is for data events only => update in registerDataEventReceived() // - value is _event.timestampLastDataEventWasReceivedMs - sourceTimestampOfLastEventReceivedMs // - for getters: if no events processed OR if bootstrap mode => all values are -1 // timeLagSourceToReceiptMs_HistPct_50 // timeLagSourceToReceiptMs_HistPct_90 // these are official "reserved suffixes" per // timeLagSourceToReceiptMs_HistPct_95 // https://iwww.corp.linkedin.com/wiki/cf/display/SOP/Getting+Started+with+Autometrics // timeLagSourceToReceiptMs_HistPct_99 // https://iwww.corp.linkedin.com/wiki/cf/display/SOP/Metrics+Naming+Policy @Override public double getTimeLagSourceToReceiptMs_HistPct_50() { return getTimeLagSourceToReceiptMs_HistPct(50.0); } @Override public double getTimeLagSourceToReceiptMs_HistPct_90() { return getTimeLagSourceToReceiptMs_HistPct(90.0); } @Override public double getTimeLagSourceToReceiptMs_HistPct_95() { return getTimeLagSourceToReceiptMs_HistPct(95.0); } @Override public double getTimeLagSourceToReceiptMs_HistPct_99() { return getTimeLagSourceToReceiptMs_HistPct(99.0); } private double getTimeLagSourceToReceiptMs_HistPct(double percentile) { double result = (double) AbstractMonitoringMBean.DEFAULT_MIN_LONG_VALUE; // -1.0 Lock readLock = acquireReadLock(); try { // If we're an aggregate, _isBootstrapping should never be true, but all of our constituents // could be bootstrapping, in which case their reservoirs will all be null and ours will be // empty. Alternatively, some constituents might be in online consumption, but if they haven't // yet received any data events, their reservoirs will be empty, and so will ours. In either // case, we return -1.0 as a special value. if (!_isBootstrapping && _event.timestampLastDataEventWasReceivedMs > 0 && _reservoirTimeLagSourceToReceiptMs != null && _reservoirTimeLagSourceToReceiptMs.size() > 0) { double[] dataValues = _reservoirTimeLagSourceToReceiptMs.getUnsortedValues(); if (dataValues.length > 0) // percentile() returns Double.NaN for empty arrays, but we want -1.0 { result = StatUtils.percentile(dataValues, percentile); } } } finally { releaseLock(readLock); } return result; } // timeLagLastReceivedToNowMs // [want max time lag over whatever is being aggregated, i.e., track the _minimum_ (oldest) timestamp] @Override public long getTimeLagLastReceivedToNowMs() { long result = 0; Lock readLock = acquireReadLock(); try { result = (_event.timestampLastDataEventWasReceivedMs != 0) ? System.currentTimeMillis() - _event.timestampLastDataEventWasReceivedMs : -1; } finally { releaseLock(readLock); } return result; } // timeLagConsumerCallbacksMs // - this is for ALL callbacks => update in registerCallbacksProcessed() // - value is timeElapsedNs / DbusConstants.NUM_NSECS_IN_MSEC == timeElapsedMs // - for getters: if no events processed => all values are zero; no special handling for bootstraps // timeLagConsumerCallbacksMs_HistPct_50 // timeLagConsumerCallbacksMs_HistPct_90 // these are official "reserved suffixes" per // timeLagConsumerCallbacksMs_HistPct_95 // https://iwww.corp.linkedin.com/wiki/cf/display/SOP/Getting+Started+with+Autometrics // timeLagConsumerCallbacksMs_HistPct_99 // https://iwww.corp.linkedin.com/wiki/cf/display/SOP/Metrics+Naming+Policy // timeLagConsumerCallbacksMs_Max @Override public double getTimeLagConsumerCallbacksMs_HistPct_50() { return getTimeLagConsumerCallbacksMs_HistPct(50.0); } @Override public double getTimeLagConsumerCallbacksMs_HistPct_90() { return getTimeLagConsumerCallbacksMs_HistPct(90.0); } @Override public double getTimeLagConsumerCallbacksMs_HistPct_95() { return getTimeLagConsumerCallbacksMs_HistPct(95.0); } @Override public double getTimeLagConsumerCallbacksMs_HistPct_99() { return getTimeLagConsumerCallbacksMs_HistPct(99.0); } // integer milliseconds would suffice for this metric, but Apache Commons Math's stats methods require doubles private double getTimeLagConsumerCallbacksMs_HistPct(double percentile) { double result = (double) AbstractMonitoringMBean.DEFAULT_MIN_LONG_VALUE; // -1.0 Lock readLock = acquireReadLock(); try { if (_reservoirTimeLagConsumerCallbacksMs.size() > 0) { double[] dataValues = _reservoirTimeLagConsumerCallbacksMs.getUnsortedValues(); result = StatUtils.percentile(dataValues, percentile); } } finally { releaseLock(readLock); } return result; } @Override public double getTimeLagConsumerCallbacksMs_Max() { double result = (double) AbstractMonitoringMBean.DEFAULT_MIN_LONG_VALUE; // -1.0 Lock readLock = acquireReadLock(); try { if (_reservoirTimeLagConsumerCallbacksMs.size() > 0) { double[] dataValues = _reservoirTimeLagConsumerCallbacksMs.getUnsortedValues(); result = StatUtils.max(dataValues); } } finally { releaseLock(readLock); } return result; } @Override public long getTimeSinceLastResetMs() { long result = 0; Lock readLock = acquireReadLock(); try { result = System.currentTimeMillis() - _event.timestampLastResetMs; } finally { releaseLock(readLock); } return result; } @Override public long getTimestampLastResetMs() { long result = 0; Lock readLock = acquireReadLock(); try { result = _event.timestampLastResetMs; } finally { releaseLock(readLock); } return result; } //----------------------------- "EVENTS RECEIVED" CALLS ----------------------------- // ("received" by client lib from relay or bootstrap but not yet passed to consumer) // called only by MultiConsumerCallback => only for lowest-level (non-aggregated) stats public void registerDataEventReceived(DbusEvent e) { if (!_enabled.get()) return; Lock writeLock = acquireWriteLock(); try { ++_event.numDataEvents; _event.timestampLastDataEventWasReceivedMs = System.currentTimeMillis(); // if we're bootstrapping, we'll return -1 regardless, so no need to waste time storing data if (!_isBootstrapping) { // not bootstrapping, so _reservoirTimeLagSourceToReceiptMs shouldn't be null final long sourceTimestampOfLastEventReceivedMs = e.timestampInNanos() / DbusConstants.NUM_NSECS_IN_MSEC; _reservoirTimeLagSourceToReceiptMs.update( _event.timestampLastDataEventWasReceivedMs - sourceTimestampOfLastEventReceivedMs, _event.timestampLastDataEventWasReceivedMs / DbusConstants.NUM_MSECS_IN_SEC); } } finally { releaseLock(writeLock); } } //----------------------------- "EVENTS PROCESSED" CALLS ----------------------------- // ("processed" by consumer, i.e., output side of client lib) // called by *ConsumerCallbackFactory public void registerCallbacksProcessed(long timeElapsedNs) // note that ConsumerCallbackStats version uses ms { if (!_enabled.get()) return; Lock writeLock = acquireWriteLock(); try { _reservoirTimeLagConsumerCallbacksMs.update((double) timeElapsedNs / DbusConstants.NUM_NSECS_IN_MSEC); } finally { releaseLock(writeLock); } } //----------------------------- "ERRORS PROCESSED" CALLS ----------------------------- // was registerErrorEventsProcessed() public void registerCallbackError() { if (!_enabled.get()) return; Lock writeLock = acquireWriteLock(); try { ++_event.numConsumerErrors; } finally { releaseLock(writeLock); } } }