Example usage for org.apache.commons.math3.stat.descriptive DescriptiveStatistics getVariance

Introduction

In this page you can find the example usage for org.apache.commons.math3.stat.descriptive DescriptiveStatistics getVariance.

Prototype

public double getVariance() 

Source Link

Document

Returns the (sample) variance of the available values.

Usage

From source file:com.github.jessemull.microflexbigdecimal.stat.PopulationVarianceWeightsTest.java

/**
 * Tests the aggregated plate statistics method using an array.
 *///from   w  w w  .  j  av  a 2s. c  o m
@Test
public void testAggregatedSetArray() {

    WellSet[] setArray = new WellSet[array.length];

    for (int i = 0; i < setArray.length; i++) {
        setArray[i] = array[i].dataSet();
    }

    Map<WellSet, BigDecimal> aggregatedReturnedMap = variance.setsAggregated(setArray, weights, mc);
    Map<WellSet, BigDecimal> aggregatedResultMap = new TreeMap<WellSet, BigDecimal>();

    for (WellSet set : setArray) {

        List<BigDecimal> resultList = new ArrayList<BigDecimal>();

        for (Well well : set) {

            List<BigDecimal> input = well.data();

            for (int i = 0; i < input.size(); i++) {
                resultList.add(input.get(i).multiply(new BigDecimal(weights[i])));
            }

        }

        double[] inputAggregated = new double[resultList.size()];

        for (int i = 0; i < resultList.size(); i++) {
            inputAggregated[i] = resultList.get(i).doubleValue();
        }

        DescriptiveStatistics statAggregated = new DescriptiveStatistics(inputAggregated);
        double resultAggregatedDouble = statAggregated.getVariance();
        resultAggregatedDouble *= resultList.size() - 1;
        resultAggregatedDouble /= resultList.size();

        BigDecimal aggregatedResult = new BigDecimal(resultAggregatedDouble, mc);
        aggregatedResultMap.put(set, aggregatedResult);
    }

    for (WellSet set : setArray) {

        BigDecimal result = aggregatedResultMap.get(set);
        BigDecimal returned = aggregatedReturnedMap.get(set);

        BigDecimal[] corrected = correctRoundingErrors(result, returned);

        assertEquals(corrected[0], corrected[1]);
    }

}

---

From source file:com.github.jessemull.microflexbiginteger.stat.PopulationVarianceWeightsTest.java

/**
 * Tests the aggregated plate statistics method using the values between the indices of
 * the array./*from w  ww .j a  va2  s .  co  m*/
 */
@Test
public void testAggregatedSetArrayIndices() {

    int begin = random.nextInt(arrayIndices[0].first().size() - 4);
    int end = begin + random.nextInt(3) + 3;

    WellSet[] setArrayIndices = new WellSet[arrayIndices.length];

    for (int i = 0; i < setArrayIndices.length; i++) {
        setArrayIndices[i] = arrayIndices[i].dataSet();
    }

    Map<WellSet, BigDecimal> aggregatedReturnedMap = variance.setsAggregated(setArrayIndices, weightsIndices,
            begin, end - begin, mc);
    Map<WellSet, BigDecimal> aggregatedResultMap = new TreeMap<WellSet, BigDecimal>();

    for (WellSet set : setArrayIndices) {

        List<BigDecimal> resultList = new ArrayList<BigDecimal>();

        for (Well well : set) {

            List<BigDecimal> input = well.toBigDecimal().subList(begin, end);

            for (int i = 0; i < input.size(); i++) {
                resultList.add(input.get(i).multiply(new BigDecimal(weightsIndices[i])));
            }

        }

        double[] inputAggregated = new double[resultList.size()];

        for (int i = 0; i < resultList.size(); i++) {
            inputAggregated[i] = resultList.get(i).doubleValue();
        }

        DescriptiveStatistics statAggregated = new DescriptiveStatistics(inputAggregated);
        double resultAggregatedDouble = statAggregated.getVariance();
        resultAggregatedDouble *= resultList.size() - 1;
        resultAggregatedDouble /= resultList.size();

        BigDecimal aggregatedResult = new BigDecimal(resultAggregatedDouble, mc);

        aggregatedResultMap.put(set, aggregatedResult);
    }

    for (WellSet plate : setArrayIndices) {

        BigDecimal result = aggregatedResultMap.get(plate);
        BigDecimal returned = aggregatedReturnedMap.get(plate);
        BigDecimal[] corrected = correctRoundingErrors(result, returned);

        assertEquals(corrected[0], corrected[1]);
    }
}

---

From source file:com.github.jessemull.microflex.stat.statbigdecimal.PopulationVarianceBigDecimalWeightsTest.java

/**
 * Tests the aggregated plate statistics method using the values between the indices of
 * the collection.//from w  w  w. jav  a2s.  c  om
 */
@Test
public void testAggregatedSetCollectionIndices() {

    int begin = random.nextInt(arrayIndices[0].first().size() - 4);
    int end = begin + random.nextInt(3) + 3;

    List<WellSetBigDecimal> collection = new ArrayList<WellSetBigDecimal>();

    for (PlateBigDecimal plate : arrayIndices) {
        collection.add(plate.dataSet());
    }

    Map<WellSetBigDecimal, BigDecimal> aggregatedReturnedMap = variance.setsAggregated(collection,
            weightsIndices, begin, end - begin, mc);
    Map<WellSetBigDecimal, BigDecimal> aggregatedResultMap = new TreeMap<WellSetBigDecimal, BigDecimal>();

    for (WellSetBigDecimal set : collection) {

        List<BigDecimal> resultList = new ArrayList<BigDecimal>();

        for (WellBigDecimal well : set) {

            List<BigDecimal> input = well.data().subList(begin, end);

            for (int i = 0; i < input.size(); i++) {
                resultList.add(input.get(i).multiply(new BigDecimal(weightsIndices[i])));
            }

        }

        double[] inputAggregated = new double[resultList.size()];

        for (int i = 0; i < resultList.size(); i++) {
            inputAggregated[i] = resultList.get(i).doubleValue();
        }

        DescriptiveStatistics statAggregated = new DescriptiveStatistics(inputAggregated);
        double resultAggregatedDouble = statAggregated.getVariance();
        resultAggregatedDouble *= resultList.size() - 1;
        resultAggregatedDouble /= resultList.size();

        BigDecimal aggregatedResult = new BigDecimal(resultAggregatedDouble, mc);
        aggregatedResultMap.put(set, aggregatedResult);
    }

    for (WellSetBigDecimal set : collection) {

        BigDecimal result = aggregatedResultMap.get(set);
        BigDecimal returned = aggregatedReturnedMap.get(set);
        BigDecimal[] corrected = correctRoundingErrors(result, returned);

        assertEquals(corrected[0], corrected[1]);
    }
}

---

From source file:com.github.jessemull.microflexbigdecimal.stat.PopulationVarianceWeightsTest.java

/**
 * Tests the aggregated plate statistics method using the values between the indices of
 * the collection./*  w  w w. j  a v a  2 s . co  m*/
 */
@Test
public void testAggregatedSetCollectionIndices() {

    int begin = random.nextInt(arrayIndices[0].first().size() - 4);
    int end = begin + random.nextInt(3) + 3;

    List<WellSet> collection = new ArrayList<WellSet>();

    for (Plate plate : arrayIndices) {
        collection.add(plate.dataSet());
    }

    Map<WellSet, BigDecimal> aggregatedReturnedMap = variance.setsAggregated(collection, weightsIndices, begin,
            end - begin, mc);
    Map<WellSet, BigDecimal> aggregatedResultMap = new TreeMap<WellSet, BigDecimal>();

    for (WellSet set : collection) {

        List<BigDecimal> resultList = new ArrayList<BigDecimal>();

        for (Well well : set) {

            List<BigDecimal> input = well.data().subList(begin, end);

            for (int i = 0; i < input.size(); i++) {
                resultList.add(input.get(i).multiply(new BigDecimal(weightsIndices[i])));
            }

        }

        double[] inputAggregated = new double[resultList.size()];

        for (int i = 0; i < resultList.size(); i++) {
            inputAggregated[i] = resultList.get(i).doubleValue();
        }

        DescriptiveStatistics statAggregated = new DescriptiveStatistics(inputAggregated);
        double resultAggregatedDouble = statAggregated.getVariance();
        resultAggregatedDouble *= resultList.size() - 1;
        resultAggregatedDouble /= resultList.size();

        BigDecimal aggregatedResult = new BigDecimal(resultAggregatedDouble, mc);
        aggregatedResultMap.put(set, aggregatedResult);
    }

    for (WellSet set : collection) {

        BigDecimal result = aggregatedResultMap.get(set);
        BigDecimal returned = aggregatedReturnedMap.get(set);
        BigDecimal[] corrected = correctRoundingErrors(result, returned);

        assertEquals(corrected[0], corrected[1]);
    }
}

---

From source file:com.github.jessemull.microflex.stat.statbigdecimal.PopulationVarianceBigDecimalWeightsTest.java

/**
 * Tests the aggregated plate statistics method using the values between the indices of
 * the array./*  ww w .  j a v  a 2 s.c  o m*/
 */
@Test
public void testAggregatedSetArrayIndices() {

    int begin = random.nextInt(arrayIndices[0].first().size() - 4);
    int end = begin + random.nextInt(3) + 3;

    WellSetBigDecimal[] setArrayIndices = new WellSetBigDecimal[arrayIndices.length];

    for (int i = 0; i < setArrayIndices.length; i++) {
        setArrayIndices[i] = arrayIndices[i].dataSet();
    }

    Map<WellSetBigDecimal, BigDecimal> aggregatedReturnedMap = variance.setsAggregated(setArrayIndices,
            weightsIndices, begin, end - begin, mc);
    Map<WellSetBigDecimal, BigDecimal> aggregatedResultMap = new TreeMap<WellSetBigDecimal, BigDecimal>();

    for (WellSetBigDecimal set : setArrayIndices) {

        List<BigDecimal> resultList = new ArrayList<BigDecimal>();

        for (WellBigDecimal well : set) {

            List<BigDecimal> input = well.data().subList(begin, end);

            for (int i = 0; i < input.size(); i++) {
                resultList.add(input.get(i).multiply(new BigDecimal(weightsIndices[i])));
            }

        }

        double[] inputAggregated = new double[resultList.size()];

        for (int i = 0; i < resultList.size(); i++) {
            inputAggregated[i] = resultList.get(i).doubleValue();
        }

        DescriptiveStatistics statAggregated = new DescriptiveStatistics(inputAggregated);
        double resultAggregatedDouble = statAggregated.getVariance();
        resultAggregatedDouble *= resultList.size() - 1;
        resultAggregatedDouble /= resultList.size();

        BigDecimal aggregatedResult = new BigDecimal(resultAggregatedDouble, mc);

        aggregatedResultMap.put(set, aggregatedResult);
    }

    for (WellSetBigDecimal plate : setArrayIndices) {

        BigDecimal result = aggregatedResultMap.get(plate);
        BigDecimal returned = aggregatedReturnedMap.get(plate);
        BigDecimal[] corrected = correctRoundingErrors(result, returned);

        assertEquals(corrected[0], corrected[1]);
    }
}

---

From source file:com.github.jessemull.microflexbigdecimal.stat.PopulationVarianceWeightsTest.java

/**
 * Tests the aggregated plate statistics method using the values between the indices of
 * the array./*w  w w .  ja  v a  2s  .  c om*/
 */
@Test
public void testAggregatedSetArrayIndices() {

    int begin = random.nextInt(arrayIndices[0].first().size() - 4);
    int end = begin + random.nextInt(3) + 3;

    WellSet[] setArrayIndices = new WellSet[arrayIndices.length];

    for (int i = 0; i < setArrayIndices.length; i++) {
        setArrayIndices[i] = arrayIndices[i].dataSet();
    }

    Map<WellSet, BigDecimal> aggregatedReturnedMap = variance.setsAggregated(setArrayIndices, weightsIndices,
            begin, end - begin, mc);
    Map<WellSet, BigDecimal> aggregatedResultMap = new TreeMap<WellSet, BigDecimal>();

    for (WellSet set : setArrayIndices) {

        List<BigDecimal> resultList = new ArrayList<BigDecimal>();

        for (Well well : set) {

            List<BigDecimal> input = well.data().subList(begin, end);

            for (int i = 0; i < input.size(); i++) {
                resultList.add(input.get(i).multiply(new BigDecimal(weightsIndices[i])));
            }

        }

        double[] inputAggregated = new double[resultList.size()];

        for (int i = 0; i < resultList.size(); i++) {
            inputAggregated[i] = resultList.get(i).doubleValue();
        }

        DescriptiveStatistics statAggregated = new DescriptiveStatistics(inputAggregated);
        double resultAggregatedDouble = statAggregated.getVariance();
        resultAggregatedDouble *= resultList.size() - 1;
        resultAggregatedDouble /= resultList.size();

        BigDecimal aggregatedResult = new BigDecimal(resultAggregatedDouble, mc);

        aggregatedResultMap.put(set, aggregatedResult);
    }

    for (WellSet plate : setArrayIndices) {

        BigDecimal result = aggregatedResultMap.get(plate);
        BigDecimal returned = aggregatedReturnedMap.get(plate);
        BigDecimal[] corrected = correctRoundingErrors(result, returned);

        assertEquals(corrected[0], corrected[1]);
    }
}

---

From source file:com.caseystella.analytics.outlier.streaming.mad.SketchyMovingMADIntegrationTest.java

@Test
public void runAccuracyBenchmark() throws IOException {
    Map<String, List<String>> benchmarks = JSONUtil.INSTANCE.load(
            new FileInputStream(new File(new File(benchmarkRoot), "combined_labels.json")),
            new TypeReference<Map<String, List<String>>>() {
            });/*from  w  ww  .j a v a 2 s. c  o  m*/
    Assert.assertTrue(benchmarks.size() > 0);
    Map<ConfusionMatrix.ConfusionEntry, Long> overallConfusionMatrix = new HashMap<>();
    DescriptiveStatistics globalExpectedScores = new DescriptiveStatistics();
    long total = 0;
    for (Map.Entry<String, List<String>> kv : benchmarks.entrySet()) {
        File dataFile = new File(new File(benchmarkRoot), kv.getKey());
        File plotFile = new File(new File(benchmarkRoot), kv.getKey() + ".dat");
        Assert.assertTrue(dataFile.exists());
        Set<Long> expectedOutliers = Sets.newHashSet(Iterables.transform(kv.getValue(), STR_TO_TS));
        OutlierRunner runner = new OutlierRunner(outlierConfig, extractorConfigStr);
        final long[] numObservations = { 0L };
        final long[] lastTimestamp = { Long.MIN_VALUE };
        final DescriptiveStatistics timeDiffStats = new DescriptiveStatistics();
        final Map<Long, Outlier> outlierMap = new HashMap<>();
        final PrintWriter pw = new PrintWriter(plotFile);
        List<Outlier> outliers = runner.run(dataFile, 1, EnumSet.of(Severity.SEVERE_OUTLIER),
                new Function<Map.Entry<DataPoint, Outlier>, Void>() {
                    @Nullable
                    @Override
                    public Void apply(@Nullable Map.Entry<DataPoint, Outlier> kv) {
                        DataPoint dataPoint = kv.getKey();
                        Outlier outlier = kv.getValue();
                        pw.println(dataPoint.getTimestamp() + " " + outlier.getDataPoint().getValue() + " "
                                + ((outlier.getSeverity() == Severity.SEVERE_OUTLIER) ? "outlier" : "normal"));
                        outlierMap.put(dataPoint.getTimestamp(), outlier);
                        numObservations[0] += 1;
                        if (lastTimestamp[0] != Long.MIN_VALUE) {
                            timeDiffStats.addValue(dataPoint.getTimestamp() - lastTimestamp[0]);
                        }
                        lastTimestamp[0] = dataPoint.getTimestamp();
                        return null;
                    }
                });
        pw.close();
        total += numObservations[0];
        Set<Long> calculatedOutliers = Sets
                .newHashSet(Iterables.transform(outliers, OutlierRunner.OUTLIER_TO_TS));
        double stdDevDiff = Math.sqrt(timeDiffStats.getVariance());
        System.out.println("Running data from " + kv.getKey() + " - E[time delta]: "
                + ConfusionMatrix.timeConversion((long) timeDiffStats.getMean()) + ", StdDev[time delta]: "
                + ConfusionMatrix.timeConversion((long) stdDevDiff) + " mean: " + runner.getMean());
        Map<ConfusionMatrix.ConfusionEntry, Long> confusionMatrix = ConfusionMatrix.getConfusionMatrix(
                expectedOutliers, calculatedOutliers, numObservations[0], (long) timeDiffStats.getMean(), 3 //stdDevDiff > 30000?0:3
                , outlierMap, globalExpectedScores);

        ConfusionMatrix.printConfusionMatrix(confusionMatrix);
        overallConfusionMatrix = ConfusionMatrix.merge(overallConfusionMatrix, confusionMatrix);
    }
    System.out.println("Really ran " + total);
    ConfusionMatrix.printConfusionMatrix(overallConfusionMatrix);
    ConfusionMatrix.printStats("Global Expected Outlier Scores", globalExpectedScores);
}

---

From source file:edu.snu.leader.util.ParseableStatistics.java

/**
 * Called immediately after evaluation occurs.
 *
 * @param state The current state of evolution
 *//*from   w ww  .java 2s  .c  o m*/
@Override
public void postEvaluationStatistics(final EvolutionState state) {
    // Before we do anything, get the time
    long evalTime = (System.currentTimeMillis() - _evalStartTime);
    println("eval-time = " + evalTime, state);
    println("eval-time-human = " + TimeUnit.MILLISECONDS.toMinutes(evalTime) + "m "
            + TimeUnit.MILLISECONDS.toSeconds(evalTime) + "s", state);
    _evalTotalTime += evalTime;

    // Call the superclass impl
    super.postEvaluationStatistics(state);

    // Define the variables to prevent a lot of gc
    Individual bestOfGenInd = null;
    Individual currentInd = null;
    Subpopulation subPop = null;
    int subPopSize = 0;
    String prefix = null;
    double indFitness = 0.0d;

    // Get the statistics objects
    DescriptiveStatistics fitnessStats = new DescriptiveStatistics();

    // Iterate over the sub-populations
    for (int i = 0; i < state.population.subpops.length; i++) {
        // Save some commonly accessed variables here
        subPop = state.population.subpops[i];
        subPopSize = subPop.individuals.length;
        prefix = "subpop[" + _2_DIGIT_FORMATTER.format(i) + "].";

        // Iterate over all the individuals in the sub-population
        bestOfGenInd = null;
        //            _bestFound[i] = bestOfGenInd;
        //            _bestFoundGen[i] = state.generation;
        for (int j = 0; j < subPopSize; j++) {
            // Get the current individual
            currentInd = subPop.individuals[j];

            // Get the fitness statistic
            indFitness = ((SimpleFitness) currentInd.fitness).fitness();
            fitnessStats.addValue(indFitness);

            // Is this individual the best found for this subpopulation
            // for this generation?
            if ((null == bestOfGenInd) || (currentInd.fitness.betterThan(bestOfGenInd.fitness))) {
                bestOfGenInd = currentInd;

                // Is it the best of the run?
                if ((_bestFound[i] == null) || (currentInd.fitness.betterThan(_bestFound[i].fitness))) {
                    // Yup
                    _bestFound[i] = currentInd;
                    _bestFoundGen[i] = state.generation;
                }
            }
        }

        // Compute and log the mean values and variance of the fitness stats
        println(prefix + "fitness-mean = " + fitnessStats.getMean(), state);
        println(prefix + "fitness-variance = " + fitnessStats.getVariance(), state);
        println(prefix + "fitness-std-dev = " + fitnessStats.getStandardDeviation(), state);

        // Display the best individual's stats
        print(buildIndDescription(bestOfGenInd, state, true, prefix + "best-individual."), state);

        println(prefix + "best-individual-found-so-far.fitness = "
                + ((SimpleFitness) _bestFound[i].fitness).fitness(), state);
        println(prefix + "best-individual-found-so-far.generation = " + _bestFoundGen[i], state);
    }

    state.output.flush();
}

---

From source file:com.facebook.presto.tests.AbstractTestQueries.java

@Test
public void testTableSamplePoissonizedRescaled() throws Exception {
    DescriptiveStatistics stats = new DescriptiveStatistics();

    long total = (long) computeExpected("SELECT COUNT(*) FROM orders", ImmutableList.of(BIGINT))
            .getMaterializedRows().get(0).getField(0);

    for (int i = 0; i < 100; i++) {
        String value = (String) computeActual(
                "SELECT COUNT(*) FROM orders TABLESAMPLE POISSONIZED (50) RESCALED APPROXIMATE AT 95 CONFIDENCE")
                        .getMaterializedRows().get(0).getField(0);
        stats.addValue(Long.parseLong(value.split(" ")[0]) * 1.0 / total);
    }//from   w ww.java  2s .c om

    double mean = stats.getGeometricMean();
    assertTrue(mean > 0.90 && mean < 1.1, format("Expected sample to be rescaled to ~1.0, but was %s", mean));
    assertTrue(stats.getVariance() > 0, "Samples all had the exact same size");
}

---

From source file:org.apache.accumulo.core.file.rfile.RolllingStatsTest.java

private static void checkAgreement(DescriptiveStatistics ds, RollingStats rs) {
    // getting stats from ds is expensive, so do it once... otherwise unit test takes 11 sec
    // instead of 5 secs
    double expMean = ds.getMean();
    double expVar = ds.getVariance();
    double expStdDev = Math.sqrt(expVar);

    assertFuzzyEquals(expMean, rs.getMean());
    assertFuzzyEquals(expVar, rs.getVariance());
    assertFuzzyEquals(expStdDev, rs.getStandardDeviation());

    assertTrue(expMean >= 0);// www .  j av a 2  s  .c  o  m
    assertTrue(rs.getMean() >= 0);
    assertTrue(expVar >= 0);
    assertTrue(rs.getVariance() >= 0);
    assertTrue(expStdDev >= 0);
    assertTrue(rs.getStandardDeviation() >= 0);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• »