List of usage examples for org.apache.commons.math3.stat.descriptive DescriptiveStatistics addValue
public void addValue(double v)
From source file:org.wso2.carbon.ml.core.spark.models.ext.AnomalyDetectionModel.java
/** * This method is to get the percentile distances map * key : percentile value/*w w w . j a va 2 s. c om*/ * value : distance value * This will return cluster boundary distance values with respect to each percentile */ private Map<Integer, Double> getPercentileDistancesMap(double percentileValue) { // Get a DescriptiveStatistics instance DescriptiveStatistics stats = new DescriptiveStatistics(); /* * key : percentile value * value : distance value */ Map<Integer, Double> percentilesMap = new HashMap<Integer, Double>(); // calculating percentile distance of each cluster for (int clusterIndex = 0; clusterIndex < clusterIndexToDistancesListMap.size(); clusterIndex++) { for (double distance : clusterIndexToDistancesListMap.get(clusterIndex)) { stats.addValue(distance); } double percentileDistance = stats.getPercentile(percentileValue); percentilesMap.put(clusterIndex, percentileDistance); stats.clear(); } return percentilesMap; }
From source file:org.wso2.carbon.ml.core.spark.models.ext.AnomalyDetectionModel.java
/** * This method is to get the percentile distance to a given cluster *///ww w.j a v a2s . com private double getPercentileDistance(double percentileValue, int clusterIndex) { // Get a DescriptiveStatistics instance DescriptiveStatistics stats = new DescriptiveStatistics(); // calculating percentile distance for (double distance : clusterIndexToDistancesListMap.get(clusterIndex)) { stats.addValue(distance); } double percentileDistance = stats.getPercentile(percentileValue); stats.clear(); return percentileDistance; }
From source file:org.wso2.extension.siddhi.execution.var.backtest.BacktestIncrementalTest.java
private void runViolationTest() { Formatter formatter = null;/*from w ww . j a v a 2s . com*/ try { formatter = new Formatter(new File(METHOD + "-" + DATA_SET + "-" + VAR_CI + ".csv")); formatter.format("%s,%s%n", "VaR", "Loss"); } catch (FileNotFoundException e) { e.printStackTrace(); } DescriptiveStatistics lossStat = new DescriptiveStatistics( lossList.stream().mapToDouble(Double::doubleValue).toArray()); DescriptiveStatistics varStat = new DescriptiveStatistics( varList.stream().mapToDouble(Double::doubleValue).toArray()); DescriptiveStatistics violationStat = new DescriptiveStatistics(); int failCount = 0; int numberOfData = varList.size(); for (int i = 0; i < numberOfData - 1; i++) { formatter.format("%.3f,%.3f%n", varList.get(i), lossList.get(i + 1)); if (lossList.get(i + 1) < varList.get(i)) { violationStat.addValue(varList.get(i)); failCount++; } } formatter.close(); System.out.println("Number of data : " + numberOfData); System.out.printf("Loss Mean : %.2f\n", lossStat.getMean()); System.out.printf("Var Mean : %.2f\n", varStat.getMean()); System.out.printf("Number of violations : %d\n", failCount); System.out.printf("Mean Violation : %.2f\n", violationStat.getMean()); System.out.printf("Violation Rate : %.2f%%\n", ((double) failCount) / (numberOfData - 1) * 100); }
From source file:org.wso2.extension.siddhi.execution.var.models.historical.HistoricalVaRCalculator.java
/** * @return the var of the portfolio Calculate the contribution of the changed asset to the portfolio loss and then * adjust the previous loss value using it. A distribution is constructed using those loss values and the ultimate * VaR value is obtained./*from w ww . j a v a 2 s. c o m*/ */ @Override public Double processData(Portfolio portfolio, Event event) { HistoricalPortfolio historicalPortfolio = (HistoricalPortfolio) portfolio; String symbol = event.getSymbol(); HistoricalAsset asset = (HistoricalAsset) getAssetPool().get(symbol); double[] currentSimulatedPriceList = asset.getCurrentSimulatedPriceList(); double[] cumulativeLossValues = historicalPortfolio.getCumulativeLossValues(); //there should be at least one return value if (asset.getNumberOfReturnValues() > 0) { DescriptiveStatistics descriptiveStatistics = new DescriptiveStatistics( currentSimulatedPriceList.length); //first time for the portfolio if (cumulativeLossValues == null) { cumulativeLossValues = new double[getBatchSize() - 1]; } double previousSimulatedPriceList[] = asset.getPreviousSimulatedPriceList(); int previousQty = portfolio.getPreviousAssetQuantities(symbol); int currentQty = portfolio.getCurrentAssetQuantities(symbol); //incrementally calculate the cumulative loss value //new cumulative loss value = previous loss value + adjustment for (int i = 0; i < currentSimulatedPriceList.length; i++) { //2nd time for the portfolio if (i < previousSimulatedPriceList.length) { cumulativeLossValues[i] = cumulativeLossValues[i] - (previousSimulatedPriceList[i] * previousQty); } //incrementally calculate the cumulative loss value cumulativeLossValues[i] += (currentSimulatedPriceList[i] * currentQty); descriptiveStatistics.addValue(cumulativeLossValues[i]); } historicalPortfolio.setCumulativeLossValues(cumulativeLossValues); return descriptiveStatistics.getPercentile((1 - getConfidenceInterval()) * 100); } return null; }
From source file:performancestatisticsasset.DistributionSet.java
public void setDistributionSet(String targetGroup, String targetTask, String targetMeasure, RecordList dataSet) {/*from w w w . ja v a2 s. c o m*/ //Determine the number of trials selected by group and by task int trials = 0; int len = dataSet.records.size(); for (int i = 0; i < len; i++) { if (dataSet.records.get(i).getTrialNumber() > trials) trials = dataSet.records.get(i).getTrialNumber(); } Distribution tempDist; DescriptiveStatistics tempStat; //For each trial of the set do for (int i = 0; i < trials; i++) { tempDist = new Distribution(); tempStat = new DescriptiveStatistics(); //Select data for (int j = 0; j < len; j++) { //If the current record is of the correct trial if ((dataSet.records.get(j).getTrialNumber() == i + 1) && (targetGroup.equals(dataSet.records.get(j).getGroupID())) && (targetTask.equals(dataSet.records.get(j).getTaskID()))) { //Fill distribution switch (targetMeasure) { case "time": tempStat.addValue(dataSet.records.get(j).getTimeToComplete()); break; case "perf": tempStat.addValue(dataSet.records.get(j).getPerformance()); break; } } } //Transfer the computed statistics to tempDist tempDist.max = tempStat.getMax(); tempDist.min = tempStat.getMin(); tempDist.sum = tempStat.getSum(); tempDist.variance = tempStat.getVariance(); tempDist.mean = tempStat.getMean(); tempDist.stdDev = tempStat.getStandardDeviation(); tempDist.skewness = tempStat.getSkewness(); tempDist.kurtosis = tempStat.getKurtosis(); tempDist.n = tempStat.getN(); //Add tempDist to distributionSet distributionSet.add(tempDist); } }
From source file:performancestatisticsasset.DistributionSet.java
public void setDistributionSet(String targetTask, String targetMeasure, RecordList dataSet) { //Determine the number of trials selected by task int trials = 0; int len = dataSet.records.size(); for (int i = 0; i < len; i++) { if (dataSet.records.get(i).getTrialNumber() > trials) trials = dataSet.records.get(i).getTrialNumber(); }/*from www.j a va2 s. co m*/ Distribution tempDist; DescriptiveStatistics tempStat; //For each trial of the set do for (int i = 0; i < trials; i++) { tempDist = new Distribution(); tempStat = new DescriptiveStatistics(); //Select data for (int j = 0; j < len; j++) { //If the current record is of the correct trial if ((dataSet.records.get(j).getTrialNumber() == i + 1) && (targetTask.equals(dataSet.records.get(j).getTaskID()))) { //Fill distribution switch (targetMeasure) { case "time": tempStat.addValue(dataSet.records.get(j).getTimeToComplete()); break; case "perf": tempStat.addValue(dataSet.records.get(j).getPerformance()); break; } } } //Transfer the computed statistics to tempDist tempDist.max = tempStat.getMax(); tempDist.min = tempStat.getMin(); tempDist.sum = tempStat.getSum(); tempDist.variance = tempStat.getVariance(); tempDist.mean = tempStat.getMean(); tempDist.stdDev = tempStat.getStandardDeviation(); tempDist.skewness = tempStat.getSkewness(); tempDist.kurtosis = tempStat.getKurtosis(); tempDist.n = tempStat.getN(); //Add tempDist to distributionSet distributionSet.add(tempDist); } }
From source file:Pooling.collectStatsTime.java
public static void main(String[] args) throws IOException { /* int maxN = 1000; double[] prob = {0.25,0.5,0.75,1.00}; int[] maxPoolSize = {35};/*w w w.j a v a2s . c o m*/ int nTests = 50; String folder = "randGraphsSimRes"; for (int t = 0; t < maxPoolSize.length; t++) { FileWriter fw = new FileWriter(folder + File.separator + "statsDesignTime" + maxPoolSize[t] + ".txt"); for (int n = 10; n <= maxN; n+=10) { DescriptiveStatistics stats = new DescriptiveStatistics(); for (int i = 1; i <= nTests; i++) { String resfile = "testResults_" + n + "_" + maxPoolSize[t] + "_" + i + ".txt"; BufferedReader br = new BufferedReader(new FileReader(folder + File.separator + resfile)); System.out.println(resfile); String s = br.readLine(); StringTokenizer st = new StringTokenizer(s," "); st.nextToken(); int k = Integer.parseInt(st.nextToken()); stats.addValue(((double)k)/1000); } fw.write(n + " " + stats.getMean() + " " + (stats.getStandardDeviation()/Math.sqrt(nTests)) + "\n"); } fw.close(); } */ int[] ns = { 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130, 140, 150 }; int[] maxPoolSizes = { 15, 25, 35 }; int nTests = 10; for (int sz = 0; sz < maxPoolSizes.length; sz++) { FileWriter fw = new FileWriter("statisticsTime_" + maxPoolSizes[sz] + ".txt"); for (int in = 0; in < ns.length; in++) { DescriptiveStatistics stats = new DescriptiveStatistics(); for (int it = 0; it < nTests; it++) { String outdir = "Test_" + maxPoolSizes[sz] + "_" + ns[in] + "_" + it; String repFile = outdir + File.separator + "report.txt"; System.out.println(outdir); BufferedReader br = new BufferedReader(new FileReader(repFile)); String s = ""; for (int i = 0; i < 6; i++) s = br.readLine(); StringTokenizer st = new StringTokenizer(s, ":"); st.nextToken(); double tm = Double.parseDouble(st.nextToken()); stats.addValue(tm); } double maxtime = stats.getMax(); DescriptiveStatistics stats1 = new DescriptiveStatistics(); for (int i = 0; i < nTests; i++) if (stats.getElement(i) != maxtime) stats1.addValue(stats.getElement(i)); fw.write(ns[in] + " " + stats1.getMean() + " " + (stats1.getStandardDeviation() / Math.sqrt((nTests - 1))) + "\n"); } fw.close(); } }
From source file:reflex.module.ReflexStatistics.java
public ReflexValue statistics(List<ReflexValue> params) { if (params.size() != 1) { throw new ReflexException(-1, "statistics needs one list parameter"); }// w w w. java2s. com if (!params.get(0).isList()) { throw new ReflexException(-1, "statistics needs one list parameter"); } DescriptiveStatistics stats = new DescriptiveStatistics(); List<ReflexValue> values = params.get(0).asList(); for (ReflexValue v : values) { stats.addValue(v.asDouble()); } Map<String, ReflexValue> ret = new HashMap<String, ReflexValue>(); ret.put("mean", new ReflexValue(stats.getMean())); ret.put("std", new ReflexValue(stats.getStandardDeviation())); ret.put("median", new ReflexValue(stats.getPercentile(50))); return new ReflexValue(ret); }
From source file:saffranexperiment.Main.java
/** * @param simulationData A 5D {@link java.util.Arrays Array} with the * following structure:/*from w w w . j a v a2 s. c om*/ * * <ul> * <li>First dimension: participant types</li> * <li>Second dimension: repeats</li> * <li>Third dimension: experiments</li> * <li>Fourth dimension: participants</li> * <li>Fifth dimension:</li> * <ol type="1"> * <li>Familiar word 1 presentation time</li> * <li>Familiar word 2 presentation time</li> * <li>Novel word 1 presentation time</li> * <li>Novel word 2 presentation time</li> * </ol> * </ul> * * For example, simulationData[7][8][1][17][2] should return the presentation * time for the first novel word achieved by participant 18 in the 9th repeat * of experiment 2 when participant type is set to 8. * * @return A 4D {@link java.util.Arrays Array} with the following structure: * * <ul> * <li>First dimension: participant types</li> * <li>Second dimension: repeats</li> * <li>Third dimension: experiments</li> * <li>Fourth dimension:</li> * <ol type="1"> * <li>Familiar word presentation time mean</li> * <li>Familiar word presentation time standard deviation</li> * <li>Novel word presentation time mean</li> * <li>Novel word presentation time standard deviation</li> * <li>t value for familiar/novel word presentation time means</li> * <li>p value for familiar/novel word presentation time means</li> * </ol> * </ul> * * For example, assigning the result to a variable j and invoking * j[5][3][1][4] would return the t-value associated with the second * experiment of the fourth repeat for participant type 6. */ private static double[][][][] calculateMeanSdTAndPValuesForEachExperimentRepeat( double[][][][][] simulationData) { double[][][][] values = new double[_totalParticipantTypes][_totalRepeats][2][6]; for (int participantType = 0; participantType < _totalParticipantTypes; participantType++) { for (int repeat = 0; repeat < _totalRepeats; repeat++) { for (int experiment = 0; experiment < 2; experiment++) { List<Double> familWordPresentationTimes = new ArrayList(); List<Double> novelWordPresentationTimes = new ArrayList(); for (int participant = 0; participant < _totalParticipants; participant++) { familWordPresentationTimes .add(simulationData[participantType][repeat][experiment][participant][0]); familWordPresentationTimes .add(simulationData[participantType][repeat][experiment][participant][1]); novelWordPresentationTimes .add(simulationData[participantType][repeat][experiment][participant][2]); novelWordPresentationTimes .add(simulationData[participantType][repeat][experiment][participant][3]); } //Calculate familiar word presentation time mean for experiment. DescriptiveStatistics familWordDescStat = new DescriptiveStatistics(); familWordPresentationTimes.forEach((value) -> familWordDescStat.addValue(value)); values[participantType][repeat][experiment][0] = familWordDescStat.getMean(); values[participantType][repeat][experiment][1] = familWordDescStat.getStandardDeviation(); //Calculate novel word presentation time mean for experiment. DescriptiveStatistics novelWordDescStat = new DescriptiveStatistics(); novelWordPresentationTimes.forEach((value) -> novelWordDescStat.addValue(value)); values[participantType][repeat][experiment][2] = novelWordDescStat.getMean(); values[participantType][repeat][experiment][3] = novelWordDescStat.getStandardDeviation(); //Convert lists containing familiar and novel presentation times to //arrays so we can use the Apache stats library functions for //calculating t and p values. double[] familWordPresentationTimesArr = convertDoubleListToDoubleArray( familWordPresentationTimes); double[] novelWordPresentationTimesArr = convertDoubleListToDoubleArray( novelWordPresentationTimes); //Calculate t value between familiar and novel word presentation times. values[participantType][repeat][0][4] = Math .abs(TestUtils.pairedT(familWordPresentationTimesArr, novelWordPresentationTimesArr)); //Calculate p value between familiar and novel word presentation times. values[participantType][repeat][0][5] = TestUtils.pairedTTest(familWordPresentationTimesArr, novelWordPresentationTimesArr); } } } return values; }
From source file:saffranexperiment.Main.java
/** * @param data A 4D {@link java.util.Arrays Array} with the following * structure (can be obtained from {@link * #calculateMeanSdTAndPValuesForEachExperimentRepeat(double[][][][][])}: * //from w ww .j a va2 s . c o m * <ul> * <li>First dimension: participant types</li> * <li>Second dimension: repeats</li> * <li>Third dimension: experiments</li> * <li>Fourth dimension:</li> * <ol type="1"> * <li>Familiar word presentation time mean</li> * <li>Familiar word presentation time standard deviation</li> * <li>Novel word presentation time mean</li> * <li>Novel word presentation time standard deviation</li> * <li>t value for familiar/novel word presentation time means</li> * <li>p value for familiar/novel word presentation time means</li> * </ol> * </ul> * * For example, invoking data[5][3][1][4] would return the t-value associated * with the second experiment of the fourth repeat for participant type 6. * * @return A 3D {@link java.util.Arrays Array}: * <ul> * <li>1st dimension: participant types</li> * <li>2nd dimension: experiments</li> * <li>3rd dimension</li> * <ol type="1"> * <li>Mean of familiar word presentation time repeat means</li> * <li>Standard error of familiar word presentation time repeat means</li> * <li>Mean of novel word presentation time repeat means</li> * <li>Standard error of novel word presentation time repeat means</li> * </ol> * </ul> * * For example, assigning the result to a variable j and invoking j[5][0][1] * would return the standard error of familiar word presentation time repeat * means associated with the repeats of experiment 1 for participant type 6. */ private static double[][][] calculateMeanAndStandardErrorOfMeanFamiliarAndNovelWordPresentationTimesForEachExperiment( double[][][][] data) { double[][][] participantTypeExperimentAverageFamiliarNovelValues = new double[_totalParticipantTypes][2][4]; for (int participantType = 0; participantType < _totalParticipantTypes; participantType++) { DescriptiveStatistics expt1FamiliarWordValues = new DescriptiveStatistics(); DescriptiveStatistics expt1NovelWordValues = new DescriptiveStatistics(); DescriptiveStatistics expt2FamiliarWordValues = new DescriptiveStatistics(); DescriptiveStatistics expt2NovelWordValues = new DescriptiveStatistics(); for (int repeat = 0; repeat < _totalRepeats; repeat++) { expt1FamiliarWordValues.addValue(data[participantType][repeat][0][0]); expt1NovelWordValues.addValue(data[participantType][repeat][0][2]); expt2FamiliarWordValues.addValue(data[participantType][repeat][1][0]); expt2NovelWordValues.addValue(data[participantType][repeat][1][2]); } participantTypeExperimentAverageFamiliarNovelValues[participantType][0][0] = expt1FamiliarWordValues .getMean(); participantTypeExperimentAverageFamiliarNovelValues[participantType][0][1] = expt1FamiliarWordValues .getStandardDeviation() / (Math.sqrt(_totalRepeats)); participantTypeExperimentAverageFamiliarNovelValues[participantType][0][2] = expt1NovelWordValues .getMean(); participantTypeExperimentAverageFamiliarNovelValues[participantType][0][3] = expt1NovelWordValues .getStandardDeviation() / (Math.sqrt(_totalRepeats)); participantTypeExperimentAverageFamiliarNovelValues[participantType][1][0] = expt2FamiliarWordValues .getMean(); participantTypeExperimentAverageFamiliarNovelValues[participantType][1][1] = expt2FamiliarWordValues .getStandardDeviation() / (Math.sqrt(_totalRepeats)); participantTypeExperimentAverageFamiliarNovelValues[participantType][1][2] = expt2NovelWordValues .getMean(); participantTypeExperimentAverageFamiliarNovelValues[participantType][1][3] = expt2NovelWordValues .getStandardDeviation() / (Math.sqrt(_totalRepeats)); } return participantTypeExperimentAverageFamiliarNovelValues; }