List of usage examples for org.apache.commons.math3.stat.descriptive DescriptiveStatistics getStandardDeviation
public double getStandardDeviation()
From source file:org.apache.metron.common.math.stats.OnlineStatisticsProviderTest.java
public static void validateStatisticsProvider(StatisticsProvider statsProvider, SummaryStatistics summaryStats, DescriptiveStatistics stats) { //N//from w ww . j av a 2 s.c om Assert.assertEquals(statsProvider.getCount(), stats.getN()); //sum Assert.assertEquals(statsProvider.getSum(), stats.getSum(), 1e-3); //sum of squares Assert.assertEquals(statsProvider.getSumSquares(), stats.getSumsq(), 1e-3); //sum of squares Assert.assertEquals(statsProvider.getSumLogs(), summaryStats.getSumOfLogs(), 1e-3); //Mean Assert.assertEquals(statsProvider.getMean(), stats.getMean(), 1e-3); //Quadratic Mean Assert.assertEquals(statsProvider.getQuadraticMean(), summaryStats.getQuadraticMean(), 1e-3); //SD Assert.assertEquals(statsProvider.getStandardDeviation(), stats.getStandardDeviation(), 1e-3); //Variance Assert.assertEquals(statsProvider.getVariance(), stats.getVariance(), 1e-3); //Min Assert.assertEquals(statsProvider.getMin(), stats.getMin(), 1e-3); //Max Assert.assertEquals(statsProvider.getMax(), stats.getMax(), 1e-3); //Kurtosis Assert.assertEquals(stats.getKurtosis(), statsProvider.getKurtosis(), 1e-3); //Skewness Assert.assertEquals(stats.getSkewness(), statsProvider.getSkewness(), 1e-3); for (double d = 10.0; d < 100.0; d += 10) { //This is a sketch, so we're a bit more forgiving here in our choice of \epsilon. Assert.assertEquals("Percentile mismatch for " + d + "th %ile", statsProvider.getPercentile(d), stats.getPercentile(d), 1e-2); } }
From source file:org.apache.metron.common.stellar.benchmark.Microbenchmark.java
public static String describe(DescriptiveStatistics stats, Double[] percentiles) { StringBuilder sb = new StringBuilder(); sb.append(String.format("round: mean of %dms [+-%d], measured %d rounds;\n", (long) stats.getMean(), (long) stats.getStandardDeviation(), stats.getN())); sb.append("\tMin - " + (long) stats.getMin() + "\n"); for (double pctile : percentiles) { sb.append("\t" + pctile + " - " + stats.getPercentile(pctile) + "\n"); }//from www .ja va2 s.co m sb.append("\tMax - " + (long) stats.getMax()); return sb.toString(); }
From source file:org.apache.metron.statistics.outlier.MedianAbsoluteDeviationTest.java
@Test public void test() { GaussianRandomGenerator gaussian = new GaussianRandomGenerator(new MersenneTwister(0L)); DescriptiveStatistics stats = new DescriptiveStatistics(); List<MedianAbsoluteDeviationFunctions.State> states = new ArrayList<>(); MedianAbsoluteDeviationFunctions.State currentState = null; //initialize the state currentState = (MedianAbsoluteDeviationFunctions.State) run("OUTLIER_MAD_STATE_MERGE(states, NULL)", ImmutableMap.of("states", states)); for (int i = 0, j = 0; i < 10000; ++i, ++j) { Double d = gaussian.nextNormalizedDouble(); stats.addValue(d);/*from www.java2s . co m*/ run("OUTLIER_MAD_ADD(currentState, data)", ImmutableMap.of("currentState", currentState, "data", d)); if (j >= 1000) { j = 0; List<MedianAbsoluteDeviationFunctions.State> stateWindow = new ArrayList<>(); for (int stateIndex = Math.max(0, states.size() - 5); stateIndex < states.size(); ++stateIndex) { stateWindow.add(states.get(stateIndex)); } currentState = (MedianAbsoluteDeviationFunctions.State) run( "OUTLIER_MAD_STATE_MERGE(states, currentState)", ImmutableMap.of("states", stateWindow, "currentState", currentState)); } } { Double score = (Double) run("OUTLIER_MAD_SCORE(currentState, value)", ImmutableMap.of("currentState", currentState, "value", stats.getMin())); Assert.assertTrue("Score: " + score + " is not an outlier despite being a minimum.", score > 3.5); } { Double score = (Double) run("OUTLIER_MAD_SCORE(currentState, value)", ImmutableMap.of("currentState", currentState, "value", stats.getMax())); Assert.assertTrue("Score: " + score + " is not an outlier despite being a maximum", score > 3.5); } { Double score = (Double) run("OUTLIER_MAD_SCORE(currentState, value)", ImmutableMap.of("currentState", currentState, "value", stats.getMean() + 4 * stats.getStandardDeviation())); Assert.assertTrue( "Score: " + score + " is not an outlier despite being 4 std deviations away from the mean", score > 3.5); } { Double score = (Double) run("OUTLIER_MAD_SCORE(currentState, value)", ImmutableMap.of("currentState", currentState, "value", stats.getMean() - 4 * stats.getStandardDeviation())); Assert.assertTrue( "Score: " + score + " is not an outlier despite being 4 std deviations away from the mean", score > 3.5); } { Double score = (Double) run("OUTLIER_MAD_SCORE(currentState, value)", ImmutableMap.of("currentState", currentState, "value", stats.getMean())); Assert.assertFalse("Score: " + score + " is an outlier despite being the mean", score > 3.5); } }
From source file:org.apache.metron.statistics.outlier.MedianAbsoluteDeviationTest.java
@Test public void testLongTailed() { TDistribution generator = new TDistribution(new MersenneTwister(0L), 100); DescriptiveStatistics stats = new DescriptiveStatistics(); List<MedianAbsoluteDeviationFunctions.State> states = new ArrayList<>(); MedianAbsoluteDeviationFunctions.State currentState = null; //initialize the state currentState = (MedianAbsoluteDeviationFunctions.State) run("OUTLIER_MAD_STATE_MERGE(states, NULL)", ImmutableMap.of("states", states)); for (int i = 0, j = 0; i < 10000; ++i, ++j) { Double d = generator.sample(); stats.addValue(d);//from ww w . jav a 2s.c om run("OUTLIER_MAD_ADD(currentState, data)", ImmutableMap.of("currentState", currentState, "data", d)); if (j >= 1000) { j = 0; List<MedianAbsoluteDeviationFunctions.State> stateWindow = new ArrayList<>(); for (int stateIndex = Math.max(0, states.size() - 5); stateIndex < states.size(); ++stateIndex) { stateWindow.add(states.get(stateIndex)); } currentState = (MedianAbsoluteDeviationFunctions.State) run( "OUTLIER_MAD_STATE_MERGE(states, currentState)", ImmutableMap.of("states", stateWindow, "currentState", currentState)); } } { Double score = (Double) run("OUTLIER_MAD_SCORE(currentState, value)", ImmutableMap.of("currentState", currentState, "value", stats.getMin())); Assert.assertTrue("Score: " + score + " is not an outlier despite being a minimum.", score > 3.5); } { Double score = (Double) run("OUTLIER_MAD_SCORE(currentState, value)", ImmutableMap.of("currentState", currentState, "value", stats.getMax())); Assert.assertTrue("Score: " + score + " is not an outlier despite being a maximum", score > 3.5); } { Double score = (Double) run("OUTLIER_MAD_SCORE(currentState, value)", ImmutableMap.of("currentState", currentState, "value", stats.getMean() + 4 * stats.getStandardDeviation())); Assert.assertTrue( "Score: " + score + " is not an outlier despite being 4 std deviations away from the mean", score > 3.5); } { Double score = (Double) run("OUTLIER_MAD_SCORE(currentState, value)", ImmutableMap.of("currentState", currentState, "value", stats.getMean() - 4 * stats.getStandardDeviation())); Assert.assertTrue( "Score: " + score + " is not an outlier despite being 4 std deviations away from the mean", score > 3.5); } { Double score = (Double) run("OUTLIER_MAD_SCORE(currentState, value)", ImmutableMap.of("currentState", currentState, "value", stats.getMean())); Assert.assertFalse("Score: " + score + " is an outlier despite being the mean", score > 3.5); } }
From source file:org.apache.metron.statistics.sampling.UniformSamplerTest.java
public void validateDistribution(Sampler sample, DescriptiveStatistics distribution) { DescriptiveStatistics s = new DescriptiveStatistics(); for (Object d : sample.get()) { s.addValue((Double) d);/*from w ww.j a v a2 s .co m*/ } Assert.assertEquals(s.getMean(), distribution.getMean(), .1); Assert.assertEquals(s.getStandardDeviation(), distribution.getStandardDeviation(), .1); }
From source file:org.apache.solr.client.solrj.io.eval.DescribeEvaluator.java
@Override public Object doWork(Object value) throws IOException { if (!(value instanceof List<?>)) { throw new IOException( String.format(Locale.ROOT, "Invalid expression %s - expecting a numeric list but found %s", toExpression(constructingFactory), value.getClass().getSimpleName())); }// w w w . j a v a2 s .c om // we know each value is a BigDecimal or a list of BigDecimals DescriptiveStatistics descriptiveStatistics = new DescriptiveStatistics(); ((List<?>) value).stream().mapToDouble(innerValue -> ((BigDecimal) innerValue).doubleValue()) .forEach(innerValue -> descriptiveStatistics.addValue(innerValue)); Map<String, Number> map = new HashMap<>(); map.put("max", descriptiveStatistics.getMax()); map.put("mean", descriptiveStatistics.getMean()); map.put("min", descriptiveStatistics.getMin()); map.put("stdev", descriptiveStatistics.getStandardDeviation()); map.put("sum", descriptiveStatistics.getSum()); map.put("N", descriptiveStatistics.getN()); map.put("var", descriptiveStatistics.getVariance()); map.put("kurtosis", descriptiveStatistics.getKurtosis()); map.put("skewness", descriptiveStatistics.getSkewness()); map.put("popVar", descriptiveStatistics.getPopulationVariance()); map.put("geometricMean", descriptiveStatistics.getGeometricMean()); map.put("sumsq", descriptiveStatistics.getSumsq()); return new Tuple(map); }
From source file:org.cirdles.ludwig.isoplot3.Means.java
/** * Ludwig's WeightedAverage and assumes ConstExtErr = true since all * possible values are returned and caller can decide * * @param inValues as double[] with length nPts * @param inErrors as double[] with length nPts * @param canReject/* ww w. j av a 2s . c o m*/ * @param canTukeys * @return double[7][]{mean, sigmaMean, err68, err95, MSWD, probability, externalFlag}, {values * with rejected as 0.0}. externalFlag = 1.0 for external uncertainty, 0.0 for internal */ public static double[][] weightedAverage(double[] inValues, double[] inErrors, boolean canReject, boolean canTukeys) { double[] values = inValues.clone(); double[] errors = inErrors.clone(); double[][] retVal = new double[][] { { 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 }, {} }; // check precondition of same size values and errors and at least 3 points int nPts = values.length; int nN = nPts; int count = 0; // where does this come from?? boolean hardRej = false; if ((nPts == errors.length) && nPts > 2) { // proceed double[] inverseVar = new double[nPts]; double[] wtdResid = new double[nPts]; double[] yy; double[] iVarY; double[] tbX = new double[nPts]; double[][] wRejected = new double[nPts][2]; for (int i = 0; i < nPts; i++) { inverseVar[i] = 1.0 / Math.pow(errors[i], 2); } double intMean = 0.0; double MSWD = 0.0; double intSigmaMean = 0.0; double probability = 0.0; double intMeanErr95 = 0.0; double intErr68 = 0.0; double extMean = 0.0; double extMeanErr95 = 0.0; double extMeanErr68 = 0.0; double extSigma = 0.0; double biWtMean = 0.0; double biWtSigma = 0.0; boolean reCalc; // entry point for RECALC goto - consider another private method? do { reCalc = false; extSigma = 0.0; double weight = 0.0; double sumWtdRatios = 0.0; double q = 0.0; count++; for (int i = 0; i < nPts; i++) { if (values[i] * errors[i] != 0.0) { weight += inverseVar[i]; sumWtdRatios += inverseVar[i] * values[i]; q += inverseVar[i] * Math.pow(values[i], 2); } } int nU = nN - 1;// ' Deg. freedom TDistribution studentsT = new TDistribution(nU); // see https://stackoverflow.com/questions/21730285/calculating-t-inverse // for explanation of cutting the tail mass in two to get agreement with Excel two-tail double t68 = Math.abs(studentsT.inverseCumulativeProbability((1.0 - 0.6826) / 2.0)); double t95 = Math.abs(studentsT.inverseCumulativeProbability((1.0 - 0.95) / 2)); intMean = sumWtdRatios / weight;// ' "Internal" error of wtd average double sums = 0.0; for (int i = 0; i < nPts; i++) { if (values[i] * errors[i] != 0.0) { double resid = values[i] - intMean;// ' Simple residual wtdResid[i] = resid / errors[i];// ' Wtd residual double wtdR2 = Math.pow(wtdResid[i], 2);//' Square of wtd residual sums += wtdR2; } } sums = Math.max(sums, 0.0); MSWD = sums / nU;// ' Mean square of weighted deviates intSigmaMean = Math.sqrt(1.0 / weight); // http://commons.apache.org/proper/commons-math/apidocs/org/apache/commons/math3/distribution/FDistribution.html FDistribution fdist = new FDistribution(nU, 1E9); probability = 1.0 - fdist.cumulativeProbability(MSWD);// ChiSquare(.MSWD, (nU)) intMeanErr95 = intSigmaMean * (double) (probability >= 0.3 ? 1.96 : t95 * Math.sqrt(MSWD)); intErr68 = intSigmaMean * (double) (probability >= 0.3 ? 0.9998 : t68 * Math.sqrt(MSWD)); extMean = 0.0; extMeanErr95 = 0.0; extMeanErr68 = 0.0; // need to find external uncertainty List<Double> yyList = new ArrayList<>(); List<Double> iVarYList = new ArrayList<>(); if ((probability < SQUID_MINIMUM_PROBABILITY) && (MSWD > 1.0)) { // Find the MLE constant external variance nN = 0; for (int i = 0; i < nPts; i++) { if (values[i] != 0.0) { yyList.add(values[i]); iVarYList.add(errors[i] * errors[i]); nN++; } } // resize arrays yy = yyList.stream().mapToDouble(Double::doubleValue).toArray(); iVarY = iVarYList.stream().mapToDouble(Double::doubleValue).toArray(); // call secant method double[] wtdExtRtsec = wtdExtRTSEC(0, 10.0 * intSigmaMean * intSigmaMean, yy, iVarY); // check for failure if (wtdExtRtsec[3] == 0.0) { extMean = wtdExtRtsec[1]; extSigma = Math.sqrt(wtdExtRtsec[0]); studentsT = new TDistribution(2 * nN - 2); extMeanErr95 = Math.abs(studentsT.inverseCumulativeProbability((1.0 - 0.95) / 2.0)) * wtdExtRtsec[2]; } else if (MSWD > 4.0) { //Failure of RTSEC algorithm because of extremely high MSWD DescriptiveStatistics stats = new DescriptiveStatistics(yy); extSigma = stats.getStandardDeviation(); extMean = stats.getMean(); extMeanErr95 = t95 * extSigma / Math.sqrt(nN); } else { extSigma = 0.0; extMean = 0.0; extMeanErr95 = 0.0; } extMeanErr68 = t68 / t95 * extMeanErr95; } if (canReject && (probability < SQUID_MINIMUM_PROBABILITY)) { // GOSUB REJECT double wtdAvg = 0.0; if (extSigma != 0.0) { wtdAvg = extMean; } else { wtdAvg = intMean; } // reject outliers int n0 = nN; for (int i = 0; i < nPts; i++) { if ((values[i] != 0.0) && (nN > 0.85 * nPts)) { // Reject no more than 30% of ratios // Start rej. tolerance at 2-sigma, increase slightly each pass. double pointError = 2.0 * Math.sqrt(errors[i] * errors[i] + extSigma * extSigma); // 2-sigma error of point being tested double totalError = Math .sqrt(pointError * pointError + (4.0 * extMeanErr68 * extMeanErr68)); // 1st-pass tolerance is 2-sigma; 2nd is 2.25-sigma; 3rd is 2.5-sigma. double tolerance = (1.0 + (double) (count - 1.0) / 4.0) * totalError; if (hardRej) { tolerance = tolerance * 1.25; } // 1st-pass tolerance is 2-sigma; 2nd is 2.5-sigma; 3rd is 3-sigma... q = values[i] - wtdAvg; if ((Math.abs(q) > tolerance) && nN > 2) { nN--; wRejected[i][0] = values[i]; values[i] = 0.0; wRejected[i][1] = errors[i]; errors[i] = 0.0; } // check tolerance } // Reject no more than 30% of ratios } // nPts loop reCalc = (nN < n0); } // canReject test } while (reCalc); if (canTukeys) { // March 2018 not finished as not sure where used System.arraycopy(values, 0, tbX, 0, nPts); double[] tukey = SquidMathUtils.tukeysBiweight(tbX, 6); biWtMean = tukey[0]; biWtSigma = tukey[1]; DescriptiveStatistics stats = new DescriptiveStatistics(tbX); double biWtErr95Median = stats.getPercentile(50); double median = median(tbX); double medianConf = medianConfLevel(nPts); double medianPlusErr = medianUpperLim(tbX) - median; double medianMinusErr = median - medianLowerLim(tbX); } // determine whether to return internal or external if (extMean != 0.0) { retVal = new double[][] { { extMean, extSigma, extMeanErr68, extMeanErr95, MSWD, probability, 1.0 }, // contains zeroes for each reject values }; } else { retVal = new double[][] { { intMean, intSigmaMean, intErr68, intMeanErr95, MSWD, probability, 0.0 }, // contains zeroes for each reject values }; } } return retVal; }
From source file:org.hawkular.client.test.metrics.openshift.CollectionRateDetailTest.java
private void getData(String metricID, String testID, long start, long end, Duration timeBucket) { Reporter.log("Fetching large data set... may take a couple minutes", true); List<DataPoint<Double>> rawData = client().metrics().gauge() .findGaugeDataWithId(metricID, String.valueOf(start), String.valueOf(end), null, null, null) .getEntity();//from ww w .j a v a 2 s . c o m Assert.assertNotNull(rawData, testID); Reporter.log("raw datapoints: " + rawData.size(), true); List<Long> zeroList = findZeroValues(rawData); Assert.assertTrue(zeroList == null || zeroList.size() == 0, testID); Map<Long, Integer> hist = OpenshiftBaseTest.makeHistogram(rawData, timeBucket); Double[] result = hist.entrySet().stream().map(x -> new Double(x.getValue())) .toArray(size -> new Double[size]); double[] d = ArrayUtils.toPrimitive(result); // drop the first and last as they are usually outliers double[] samples = Arrays.copyOfRange(d, 1, d.length - 1); DescriptiveStatistics stats = new DescriptiveStatistics(samples); Reporter.log(hist.toString(), true); Reporter.log("size: " + stats.getN(), true); Reporter.log("min/max: " + stats.getMin() + "/" + stats.getMax(), true); Reporter.log("mean: " + stats.getMean(), true); Reporter.log("variance: " + stats.getVariance(), true); Reporter.log("stddev: " + stats.getStandardDeviation(), true); }
From source file:org.lightjason.agentspeak.action.buildin.math.statistic.EStatisticValue.java
/** * returns a statistic value//from ww w. j a v a 2s .c o m * * @param p_statistic statistic object * @return statistic value */ public final double value(final DescriptiveStatistics p_statistic) { switch (this) { case GEOMETRICMEAN: return p_statistic.getGeometricMean(); case MAX: return p_statistic.getMax(); case MIN: return p_statistic.getMin(); case COUNT: return p_statistic.getN(); case POPULATIONVARIANCE: return p_statistic.getPopulationVariance(); case QUADRATICMEAN: return p_statistic.getQuadraticMean(); case STANDARDDEVIATION: return p_statistic.getStandardDeviation(); case SUM: return p_statistic.getSum(); case SUMSQUARE: return p_statistic.getSumsq(); case VARIANCE: return p_statistic.getVariance(); case MEAN: return p_statistic.getMean(); case KURTIOSIS: return p_statistic.getKurtosis(); default: throw new CIllegalStateException( org.lightjason.agentspeak.common.CCommon.languagestring(this, "unknown", this)); } }
From source file:org.lightjason.agentspeak.action.builtin.math.statistic.EStatisticValue.java
/** * returns a statistic value//ww w. j ava 2 s . c o m * * @param p_statistic statistic object * @return statistic value */ public final double value(@Nonnull final DescriptiveStatistics p_statistic) { switch (this) { case GEOMETRICMEAN: return p_statistic.getGeometricMean(); case MAX: return p_statistic.getMax(); case MIN: return p_statistic.getMin(); case COUNT: return p_statistic.getN(); case POPULATIONVARIANCE: return p_statistic.getPopulationVariance(); case QUADRATICMEAN: return p_statistic.getQuadraticMean(); case STANDARDDEVIATION: return p_statistic.getStandardDeviation(); case SUM: return p_statistic.getSum(); case SUMSQUARE: return p_statistic.getSumsq(); case VARIANCE: return p_statistic.getVariance(); case MEAN: return p_statistic.getMean(); case KURTIOSIS: return p_statistic.getKurtosis(); default: throw new CIllegalStateException( org.lightjason.agentspeak.common.CCommon.languagestring(this, "unknown", this)); } }