Example usage for org.apache.commons.math.stat.descriptive DescriptiveStatistics getPercentile

List of usage examples for org.apache.commons.math.stat.descriptive DescriptiveStatistics getPercentile

  1. HOME
  2. Java
  3. org.apache.commons
  4. org.apache.commons.math.stat.descriptive.*
  5. DescriptiveStatistics
  6. getPercentile

Introduction

In this page you can find the example usage for org.apache.commons.math.stat.descriptive DescriptiveStatistics getPercentile.

Prototype

public double getPercentile(double p)

Source Link

Document

Returns an estimate for the pth percentile of the stored values.

Usage

From source file:playground.johannes.gsv.sim.cadyts.ODCountsAnalyzer.java

@Override
public void notifyAfterMobsim(AfterMobsimEvent event) {
    Network network = event.getControler().getScenario().getNetwork();
    DescriptiveStatistics diff = new DescriptiveStatistics();
    DescriptiveStatistics absDiff = new DescriptiveStatistics();
    DescriptiveStatistics error = new DescriptiveStatistics();
    DescriptiveStatistics absError = new DescriptiveStatistics();

    try {//from ww  w. j a v  a  2  s . co  m
        String file = event.getControler().getControlerIO().getIterationFilename(event.getIteration(),
                "odCounts.txt");
        BufferedWriter writer = new BufferedWriter(new FileWriter(file));

        writer.write("id\tobs\tsim");

        writer.newLine();

        for (Count count : counts.getCounts().values()) {
            if (count.getLocId().toString().startsWith(ODCalibrator.VIRTUAL_ID_PREFIX)) {
                Link link = network.getLinks().get(count.getLocId());
                double refVal = count.getMaxVolume().getValue() * 24;
                double simVal = simResults.getSimValue(link, 0, 86400, TYPE.COUNT_VEH);

                double err = (simVal - refVal) / refVal;
                error.addValue(err);
                absError.addValue(Math.abs(err));

                double delta = simVal - refVal;
                diff.addValue(delta);
                absDiff.addValue(Math.abs(delta));

                writer.write(link.getId().toString());
                writer.write("\t");
                writer.write(String.valueOf(refVal));
                writer.write("\t");
                writer.write(String.valueOf(simVal));
                writer.newLine();
            }
        }
        writer.close();

        logger.info(String.format("OD-relations diff: avr = %s, median = %s, var = %s, min = %s, max = %s",
                diff.getMean(), diff.getPercentile(50), diff.getVariance(), diff.getMin(), diff.getMax()));
        logger.info(
                String.format("OD-relations absolute diff: avr = %s, median = %s, var = %s, min = %s, max = %s",
                        absDiff.getMean(), absDiff.getPercentile(50), absDiff.getVariance(), absDiff.getMin(),
                        absDiff.getMax()));

        logger.info(String.format(
                "Relative OD-relations error: avr = %s, median = %s, var = %s, min = %s, max = %s",
                error.getMean(), error.getPercentile(50), error.getVariance(), error.getMin(), error.getMax()));
        logger.info(String.format(
                "Absolute relative OD-relations error: avr = %s, median = %s, var = %s, min = %s, max = %s",
                absError.getMean(), absError.getPercentile(50), absError.getVariance(), absError.getMin(),
                absError.getMax()));

        file = event.getControler().getControlerIO().getOutputFilename("odCountsDiff.txt");
        writeStats(file, diff, event.getIteration());

        file = event.getControler().getControlerIO().getOutputFilename("odCountsAbsDiff.txt");
        writeStats(file, absDiff, event.getIteration());

        file = event.getControler().getControlerIO().getOutputFilename("odCountsError.txt");
        writeStats(file, error, event.getIteration());

        file = event.getControler().getControlerIO().getOutputFilename("odCountsAbsError.txt");
        writeStats(file, absError, event.getIteration());
    } catch (IOException e) {
        e.printStackTrace();
    }
}

From source file:playground.johannes.gsv.sim.cadyts.ODCountsAnalyzer.java

private void writeStats(String file, DescriptiveStatistics stats, int iteration) throws IOException {
    boolean append = true;
    if (iteration == 0)
        append = false;/*from  w w w. j a  va 2 s . c o  m*/

    BufferedWriter writer = new BufferedWriter(new FileWriter(file, append));
    if (!append) {
        writer.write("avr\tmedian\tvar\tmin\tmax");
        writer.newLine();
    }

    writer.write(String.valueOf(stats.getMean()));
    writer.write("\t");
    writer.write(String.valueOf(stats.getPercentile(50)));
    writer.write("\t");
    writer.write(String.valueOf(stats.getVariance()));
    writer.write("\t");
    writer.write(String.valueOf(stats.getMin()));
    writer.write("\t");
    writer.write(String.valueOf(stats.getMax()));
    writer.newLine();
    writer.close();
}

From source file:playground.johannes.sna.util.TXTWriter.java

public static void writeStatistics(TDoubleObjectHashMap<DescriptiveStatistics> statsMap, String xLab,
        String file) throws IOException {
    double[] keys = statsMap.keys();
    Arrays.sort(keys);//w  ww  .  ja  va 2  s  .co m

    BufferedWriter writer = new BufferedWriter(new FileWriter(file));

    writer.write(xLab);
    writer.write(TAB);
    writer.write("mean");
    writer.write(TAB);
    writer.write("median");
    writer.write(TAB);
    writer.write("min");
    writer.write(TAB);
    writer.write("max");
    writer.write(TAB);
    writer.write("n");
    writer.newLine();

    for (double key : keys) {
        DescriptiveStatistics stats = statsMap.get(key);

        writer.write(String.valueOf(key));
        writer.write(TAB);
        writer.write(String.valueOf(stats.getMean()));
        writer.write(TAB);
        writer.write(String.valueOf(stats.getPercentile(50)));
        writer.write(TAB);
        writer.write(String.valueOf(stats.getMin()));
        writer.write(TAB);
        writer.write(String.valueOf(stats.getMax()));
        writer.write(TAB);
        writer.write(String.valueOf(stats.getN()));
        writer.newLine();
    }

    writer.close();
}

From source file:playground.johannes.socialnetworks.snowball2.sim.postprocess.ConfidenceInterval.java

/**
 * @param args//from  ww w.j  a v  a2  s  .  co  m
 * @throws IOException 
 */
public static void main(String[] args) throws IOException {
    String rootDir = args[0];

    int dumpStart = Integer.parseInt(args[1]);
    int dumpEnd = Integer.parseInt(args[2]);
    ;
    int dumpStep = Integer.parseInt(args[3]);

    String constParamKey = args[4];
    int constParam = Integer.parseInt(args[5]);
    String property = args[6];
    String dumpProperty = args[7];
    String output = args[8];

    final double mean = Double.parseDouble(args[9]);
    double confProba = Double.parseDouble(args[10]);
    String mode = args[11];

    TIntObjectHashMap<TIntDoubleHashMap> table = new TIntObjectHashMap<TIntDoubleHashMap>();
    SortedSet<Integer> dumpKeys = new TreeSet<Integer>();

    for (int dumpKey = dumpStart; dumpKey <= dumpEnd; dumpKey += dumpStep) {
        TIntDoubleHashMap row = new TIntDoubleHashMap();

        BufferedReader valueReader;
        BufferedReader dumpReader;
        if (constParamKey.equalsIgnoreCase("alpha")) {
            String path = String.format("%1$s/seed.%2$s/alpha.%3$s/%4$s.txt", rootDir, dumpKey, constParam,
                    property);
            valueReader = new BufferedReader(new FileReader(path));
            System.out.println("Loading file " + path);

            path = String.format("%1$s/seed.%2$s/alpha.%3$s/%4$s.avr.txt", rootDir, dumpKey, constParam,
                    dumpProperty);
            dumpReader = new BufferedReader(new FileReader(path));
            System.out.println("Loading file " + path);
        } else if (constParamKey.equalsIgnoreCase("seed")) {
            String path = String.format("%1$s/seed.%2$s/alpha.%3$s/%4$s.txt", rootDir, constParam, dumpKey,
                    property);
            valueReader = new BufferedReader(new FileReader(path));
            System.out.println("Loading file " + path);

            path = String.format("%1$s/seed.%2$s/alpha.%3$s/%4$s.avr.txt", rootDir, constParam, dumpKey,
                    dumpProperty);
            dumpReader = new BufferedReader(new FileReader(path));
            System.out.println("Loading file " + path);
        } else
            throw new IllegalArgumentException(
                    String.format("Constant parameter %1$s unknown.", constParamKey));

        String header = valueReader.readLine();
        String keys[] = header.split("\t");
        int cols = keys.length;
        String valueLine;
        Map<String, TDoubleArrayList> matrix = new HashMap<String, TDoubleArrayList>();
        while ((valueLine = valueReader.readLine()) != null) {
            String[] tokens = valueLine.split("\t");
            for (int i = 0; i < cols; i++) {
                TDoubleArrayList list = matrix.get(keys[i]);
                if (list == null) {
                    list = new TDoubleArrayList();
                    matrix.put(keys[i], list);
                }

                list.add(Double.parseDouble(tokens[i]));
            }
        }

        String dumpLine;
        Map<String, String> dumpMapping = new HashMap<String, String>();
        while ((dumpLine = dumpReader.readLine()) != null) {
            String[] tokens = dumpLine.split("\t");
            dumpMapping.put(tokens[0], tokens[1]);
        }

        for (Entry<String, TDoubleArrayList> entry : matrix.entrySet()) {
            DescriptiveStatistics stats = new DescriptiveStatistics();

            double vals[] = entry.getValue().toNativeArray();
            for (double val : vals) {
                if (!Double.isNaN(val)) {
                    double relerr;
                    if (mode.equals("abs")) {
                        relerr = Math.abs((val - mean) / mean);
                    } else {
                        relerr = (val - mean) / mean;
                    }
                    stats.addValue(relerr);

                }
            }
            if (stats.getN() < 50) {
                System.err.println("Less than 50 samples. Ignoring dump.");
            } else {
                double conf;
                if (mode.equals("abs"))
                    conf = stats.getPercentile(confProba);
                else if (mode.equals("pos")) {
                    confProba = (100 - confProba) / 2.0;
                    conf = stats.getPercentile(100 - confProba);
                } else if (mode.equals("neg")) {
                    confProba = (100 - confProba) / 2.0;
                    conf = stats.getPercentile(confProba);
                } else {
                    throw new IllegalArgumentException(String.format("Mode %1$s unknown.", mode));
                }
                // int key = Integer.parseInt(keys[i]);
                String keyStr = entry.getKey();
                if (!dumpMapping.get(keyStr).equals("null")) {
                    int key = (int) Double.parseDouble(dumpMapping.get(keyStr));
                    row.put(key, conf);
                    dumpKeys.add(key);
                } else {
                    System.err.println("Null key");
                }
            }
        }
        table.put(dumpKey, row);
    }

    write(table, output, dumpKeys);
}

From source file:playground.johannes.socialnetworks.snowball2.sim.postprocess.PercentileTest.java

/**
 * @param args/*  w ww .  j  a va 2s  .  c om*/
 */
public static void main(String[] args) {
    DescriptiveStatistics stats = new DescriptiveStatistics();

    for (int i = 0; i < 20; i++) {
        stats.addValue(i);
    }

    System.out.println(stats.getPercentile(10));

}

From source file:pro.foundev.strategies.BenchmarkStrategy.java

private void exec(Runnable runnable, String name, int runs, BenchmarkReport report) {
    logger.info("Starting run of " + name);
    DescriptiveStatistics stats = new DescriptiveStatistics();

    Stopwatch timer = new Stopwatch();
    for (int i = 0; i < runs; i++) {
        timer.start();//w  ww.  j a v a2s.  com
        runnable.run();
        timer.stop();
        logger.info("Time to execute load run #" + i + " it took " + timer);
        stats.addValue(timer.elapsed(TimeUnit.MILLISECONDS));
        timer.reset();
    }
    logger.info("Finished run of " + name);
    report.addLine(name, stats.getMin(), stats.getMax(), stats.getPercentile(50), stats.getPercentile(90),
            stats.getMean());
}

From source file:rs.fon.whibo.GDT.component.removeInsignificantAttributes.ChiSquareTestCategorical.java

@Override
public LinkedList<Attribute> removeAttributes(ExampleSet exampleSet,
        LinkedList<Attribute> attributesForSplitting) {

    // checks if the example set is pure, and if it is, it exits the method
    Attribute label = exampleSet.getAttributes().getLabel();
    if (Tools.getAllCategories(exampleSet, label).size() < 2)
        return attributesForSplitting;

    // selects the attributes to be evaluated for removal (by calculating
    // chi-square probability for each attribute)
    ArrayList<Attribute> attributesToRemove = new ArrayList<Attribute>();
    ArrayList<Double> attributeProbabilities = new ArrayList<Double>();
    for (Attribute attr : attributesForSplitting)
        if (attr.isNominal()) {
            // calculate chi-square probability of the attribute
            double probability = 0;
            try {
                long[][] matrixForAttribute = getContigencyTable(exampleSet, attr);
                ChiSquareTestImpl chiTest = new ChiSquareTestImpl();
                probability = chiTest.chiSquareTest(matrixForAttribute);
            } catch (MathException me) {
                // System.out.println("Error in calculating math formula (chiTest)");
            }/*from  w w  w. j a  v  a2s  .c  om*/
            // add the attribute to the list
            attributesToRemove.add(attr);
            attributeProbabilities.add(new Double(probability));
        }

    // calculates the percentile of the required percentage. Percentile
    // variable in code represents the percentage of attributes to be kept
    // (not removed)
    double percentile;
    DescriptiveStatistics stat = new DescriptiveStatistics();
    for (Double d : attributeProbabilities)
        stat.addValue(d.doubleValue());
    percentile = stat.getPercentile((1 - Percentage_Remove) * 100);

    // evaluates attributes and chooses the ones for removal (actually saves
    // the ones not for removal)
    Iterator<Attribute> iattr = attributesToRemove.iterator();
    Iterator<Double> iprob = attributeProbabilities.iterator();
    while (iattr.hasNext()) {
        iattr.next();
        Double prob = iprob.next();
        if (Use_Percentage_Instead == 0) {
            if (prob <= Alpha_Value) {
                iattr.remove();
                iprob.remove();
            }
        } else {
            if (prob <= percentile) {
                iattr.remove();
                iprob.remove();
            }
        }
    }

    // removes the attributes
    for (Attribute attr : attributesToRemove)
        attributesForSplitting.remove(attr);
    return attributesForSplitting;
}

From source file:rs.fon.whibo.GDT.component.removeInsignificantAttributes.FTestNumerical.java

public LinkedList<Attribute> removeAttributes(ExampleSet exampleSet,
        LinkedList<Attribute> attributesForSplitting) {
    // checks if the example set is pure, and if it is, it exits the method
    Attribute label = exampleSet.getAttributes().getLabel();
    if (Tools.getAllCategories(exampleSet, label).size() < 2)
        return attributesForSplitting;

    // selects the attributes to be evaluated for removal (by calculating
    // F-test probability for each attribute)
    ArrayList<Attribute> attributesToRemove = new ArrayList<Attribute>();
    ArrayList<Double> attributeProbabilities = new ArrayList<Double>();
    for (Attribute attr : attributesForSplitting)
        if (attr.isNumerical()) {
            // calculate F-test probability of the attribute
            double probability = 0;
            try {

                OneWayAnova fTest = new OneWayAnovaImpl();
                List<double[]> paramForFTest = getArraysByLabel(exampleSet, attr);

                // tests if no arrays for f-test has fewer that 2 elements
                boolean fTestImpossible = false;
                for (double[] i : paramForFTest)
                    if (i.length < 2)
                        fTestImpossible = true;

                // calculates ftest probability
                if (!fTestImpossible)
                    probability = fTest.anovaPValue(paramForFTest);

            } catch (Exception e) {
                // System.out.println("Error in calculating math formula (FTest)");
            }/*from w w w .j  a  va 2 s  . c  o  m*/
            // add the attribute to the list
            attributesToRemove.add(attr);
            attributeProbabilities.add(new Double(probability));
        }

    if (attributesToRemove.size() == 0)
        return attributesForSplitting;

    // calculates the percentile of the required percentage. Percentile
    // variable in code represents the percentage of attributes to be kept
    // (not removed)
    double percentile;
    DescriptiveStatistics stat = new DescriptiveStatistics();
    for (Double d : attributeProbabilities)
        stat.addValue(d.doubleValue());
    percentile = stat.getPercentile((1 - Percentage_Remove) * 100);

    // evaluates attributes and chooses the ones for removal (actually saves
    // the ones not for removal)
    Iterator<Attribute> iattr = attributesToRemove.iterator();
    Iterator<Double> iprob = attributeProbabilities.iterator();
    while (iattr.hasNext()) {
        iattr.next();
        Double prob = iprob.next();
        if (Use_Percentage_Instead == 0) {
            if (prob <= Alpha_Value) {
                iattr.remove();
                iprob.remove();
            }
        } else {
            if (prob <= percentile) {
                iattr.remove();
                iprob.remove();
            }
        }
    }

    // removes the attributes
    for (Attribute attr : attributesToRemove)
        attributesForSplitting.remove(attr);
    return attributesForSplitting;

}