List of usage examples for org.apache.commons.math.stat.descriptive.moment StandardDeviation StandardDeviation
public StandardDeviation()
From source file:org.gwaspi.statistics.ChiSqrBoundaryCalculator.java
protected static void calculateChisqrBoundaryByStDev() throws IOException, MathException { FileWriter repFW = new FileWriter(boundaryPath); BufferedWriter repBW = new BufferedWriter(repFW); NetcdfFile ncfile = NetcdfFile.open(netCDFFile); List<Dimension> dims = ncfile.getDimensions(); Dimension sizeDim = dims.get(0); Dimension simsDim = dims.get(1); String varName = "distributions"; Variable distributions = ncfile.findVariable(varName); try {/*from w w w . j av a 2 s .c o m*/ for (int i = 0; i < pointsNb; i++) { //distributions(i:i:1, 0:simsNb:1) ArrayDouble.D2 rdDoubleArrayD2 = (ArrayDouble.D2) distributions .read(i + ":" + i + ":1, 0:" + (simsDim.getLength() - 1) + ":1"); ArrayDouble.D1 rdDoubleArrayD1 = (D1) rdDoubleArrayD2.reduce(); double sampleSize = rdDoubleArrayD2.getSize(); double currentTot = 0; double[] allValues = new double[(int) sampleSize]; for (int j = 0; j < sampleSize; j++) { allValues[j] = rdDoubleArrayD1.get(j); currentTot += rdDoubleArrayD1.get(j); } StandardDeviation stdDev = new StandardDeviation(); double stdDevValue = stdDev.evaluate(allValues); double currentAvg = currentTot / simNb; double low95 = currentAvg - (2 * stdDevValue); // Display 2 standard deviations double top95 = currentAvg + (2 * stdDevValue); // Display 2 standard deviations StringBuilder sb = new StringBuilder(); sb.append(top95); sb.append(","); sb.append(currentAvg); sb.append(","); sb.append(low95); repBW.append(sb + "\n"); } } catch (IOException ex) { log.error("Cannot read data", ex); } catch (InvalidRangeException ex) { log.error("Cannot read data", ex); } repBW.close(); repFW.close(); log.info("Confidence boundary created for {} points", N); }
From source file:playground.wrashid.parkingSearch.withindayFW.zhCity.ParkingCostOptimizerZH.java
public void logParkingPriceStats(Controler controler, int iteration) { // String iterationFilenameTxt = // controler.getControlerIO().getIterationFilename(controler.getIterationNumber(), // "parkingPrices.txt"); Collection<Double> streetParkingPricesMorning = new LinkedList<Double>(); Collection<Double> garageParkingPricesMorning = new LinkedList<Double>(); Collection<Double> streetParkingPricesAfternoon = new LinkedList<Double>(); Collection<Double> garageParkingPricesAfternoon = new LinkedList<Double>(); double[] morningArray; double[] afterNoonArray; separateStreetAndGarageParkingPrices(publicParkingPricePerHourInTheMorning, streetParkingPricesMorning, garageParkingPricesMorning); separateStreetAndGarageParkingPrices(publicParkingPricePerHourInTheAfternoon, streetParkingPricesAfternoon, garageParkingPricesAfternoon); morningArray = Collections.convertDoubleCollectionToArray(streetParkingPricesMorning); afterNoonArray = Collections.convertDoubleCollectionToArray(streetParkingPricesAfternoon); log.info("it-" + iteration + " average street parking price (morning/afternoon): " + (new Mean()).evaluate(morningArray) + "/" + new Mean().evaluate(afterNoonArray)); log.info("it-" + iteration + " standardDeviation street parking price (morning/afternoon):" + (new StandardDeviation()).evaluate(morningArray) + "/" + new StandardDeviation().evaluate(afterNoonArray)); morningArray = Collections.convertDoubleCollectionToArray(garageParkingPricesMorning); afterNoonArray = Collections.convertDoubleCollectionToArray(garageParkingPricesAfternoon); log.info("it-" + iteration + " average garage parking price (morning/afternoon): " + (new Mean()).evaluate(morningArray) + "/" + new Mean().evaluate(afterNoonArray)); log.info("it-" + iteration + " standardDeviation garage parking price (morning/afternoon):" + (new StandardDeviation()).evaluate(morningArray) + "/" + new StandardDeviation().evaluate(afterNoonArray)); if (GlobalParkingSearchParams.writeDetailedOutput(iteration)) { writeoutPrices(controler, iteration); }// ww w. ja va 2s .c o m }
From source file:rrlFramework.RRLExperiment.java
/** * Compiles the performance files together into a single file, detailing the * average, min and max performances.// w w w. j a v a 2 s . c o m * * @param runEnd * The last run. * @param byEpisode * If the performances are being combined by episode (in * intervals) or by regular CE interval. */ private long combineTempFiles(File performanceFile, int runEnd, long experimentStart) throws Exception { List<List<Float[]>> performances = new ArrayList<List<Float[]>>(); float min = Float.MAX_VALUE; int minRun = -1; float max = -Float.MAX_VALUE; int maxRun = -1; double[] episodeLengths = new double[runEnd]; double[] numSlots = new double[runEnd]; long averageRunTime = 0; File combinedPerfFile = performanceFile; if (Config.getInstance().getGeneratorFile() != null) { combinedPerfFile = new File(performanceFile.getAbsolutePath() + "greedy"); ProgramArgument.PERFORMANCE_EPISODE_GAP .setDoubleValue(ProgramArgument.PERFORMANCE_TESTING_SIZE.intValue() * ProgramArgument.POLICY_REPEATS.intValue()); } if (!combinedPerfFile.exists()) combinedPerfFile.createNewFile(); // For every performance file for (int i = 0; i < runEnd; i++) { File tempPerf = new File(Config.TEMP_FOLDER + "/" + performanceFile + i); if (!Performance.readRawPerformanceFile(tempPerf, true)) { System.err.println("Error reading performance file."); return 0; } List<Float[]> thisRunPerformances = new ArrayList<Float[]>(); performances.add(thisRunPerformances); // Run through the performances and place them in the matrix SortedMap<Integer, Float[]> runPerformances = Performance.getPerformanceArray(); averageRunTime += Performance.getRunTime(); Iterator<Integer> iter = runPerformances.keySet().iterator(); Integer current = iter.next(); Integer previous = null; int currentKeyframeEpisode = ProgramArgument.PERFORMANCE_EPISODE_GAP.intValue(); // Run through the performances, using linear interpolation to // get estimates of the performance at a given interval. do { // If the current segment is further along than the current // value, advance to the next value. while (currentKeyframeEpisode > current) { previous = current; if (iter.hasNext()) current = iter.next(); else break; } // If the keyframe isn't up to the first episode, just use // the current value Float[] episodePerformance = runPerformances.get(current); if (previous == null) { // Add to the previous value. thisRunPerformances.add(episodePerformance); } else { // Interpolate from the previous value to the current // one. Float[] interpolatedPerformance = new Float[episodePerformance.length]; if (previous == current) { interpolatedPerformance = episodePerformance; } else { Float[] prevPerformance = runPerformances.get(previous); for (int j = 0; j < episodePerformance.length; j++) { Float currPerf = episodePerformance[j]; Float prevPerf = prevPerformance[j]; // Adjust for null elites if (j == PerformanceDetails.ELITEMAX.ordinal() || j == PerformanceDetails.ELITEMEAN.ordinal()) { if (currPerf == null) currPerf = episodePerformance[PerformanceDetails.MEAN.ordinal()]; if (prevPerf == null) prevPerf = prevPerformance[PerformanceDetails.MEAN.ordinal()]; } if (currPerf == null || prevPerf == null) interpolatedPerformance[j] = null; else interpolatedPerformance[j] = (currPerf - prevPerf) * (1f * (currentKeyframeEpisode - previous) / (current - previous)) + prevPerf; } } // Add to the performances thisRunPerformances.add(interpolatedPerformance); } // To the next increment currentKeyframeEpisode += ProgramArgument.PERFORMANCE_EPISODE_GAP.intValue(); } while (currentKeyframeEpisode <= runPerformances.lastKey()); Float[] lastPerf = runPerformances.get(runPerformances.lastKey()); thisRunPerformances.add(lastPerf); System.out.println(runPerformances.get(runPerformances.lastKey())[PerformanceDetails.MEAN.ordinal()]); // Find min or max runs float runVal = runPerformances.get(runPerformances.lastKey())[PerformanceDetails.MEAN.ordinal()]; if (runVal < min) { min = runVal; minRun = i; } if (runVal > max) { max = runVal; maxRun = i; } episodeLengths[i] = runPerformances.lastKey(); } // Calculate the average and print out the stats FileWriter writer = new FileWriter(combinedPerfFile); BufferedWriter buf = new BufferedWriter(writer); Config.writeFileHeader(buf, Config.getInstance().getGoal()); buf.write( "Episode\tAverage\tSD\tMin\tMax\tElite-Average\tElite-SD\tNumSlots\tSlots-SD\tNumRules\tRules-SD\n"); boolean moreEpisodes = true; int index = 0; Mean mean = new Mean(); StandardDeviation sd = new StandardDeviation(); while (moreEpisodes) { moreEpisodes = false; // Compile the array of performances for the given index double[][] performanceArray = new double[PerformanceDetails.values().length][performances.size()]; double maxVal = 0; double minVal = 0; for (int run = 0; run < performances.size(); run++) { List<Float[]> runPerformanceList = performances.get(run); int thisIndex = Math.min(index, runPerformanceList.size() - 1); if (index < runPerformanceList.size() - 1) moreEpisodes = true; Float[] performanceDetails = runPerformanceList.get(thisIndex); for (int j = 0; j < performanceDetails.length; j++) { if (performanceDetails[j] != null) performanceArray[j][run] = performanceDetails[j]; } // Max and min if (run == minRun) minVal = performanceArray[PerformanceDetails.MEAN.ordinal()][run]; if (run == maxRun) maxVal = performanceArray[PerformanceDetails.MEAN.ordinal()][run]; } // Find the statistics int episodeNum = (index + 1) * ProgramArgument.PERFORMANCE_EPISODE_GAP.intValue(); buf.write(episodeNum + "\t" + mean.evaluate(performanceArray[PerformanceDetails.MEAN.ordinal()]) + "\t" + sd.evaluate(performanceArray[PerformanceDetails.MEAN.ordinal()]) + "\t" + minVal + "\t" + maxVal + "\t" + mean.evaluate(performanceArray[PerformanceDetails.ELITEMEAN.ordinal()]) + "\t" + sd.evaluate(performanceArray[PerformanceDetails.ELITEMEAN.ordinal()]) + "\t" + mean.evaluate(performanceArray[PerformanceDetails.NUMSLOTS.ordinal()]) + "\t" + sd.evaluate(performanceArray[PerformanceDetails.NUMSLOTS.ordinal()]) + "\t" + mean.evaluate(performanceArray[PerformanceDetails.NUMRULES.ordinal()]) + "\t" + sd.evaluate(performanceArray[PerformanceDetails.NUMRULES.ordinal()]) + "\n"); index++; } averageRunTime /= runEnd; buf.write("Average Run Time: " + toTimeFormat(averageRunTime) + "\n"); // Write the average episode length buf.write("\nAverage episode length: " + mean.evaluate(episodeLengths) + " +- " + sd.evaluate(episodeLengths) + "\n"); buf.write("\nAverage num slots: " + mean.evaluate(numSlots) + " +- " + sd.evaluate(numSlots) + "\n"); buf.close(); writer.close(); return averageRunTime; }
From source file:state.FindOutliers.java
public static void main(String[] args) throws Exception { final double threshold = args.length == 0 ? 2.0 : Double.parseDouble(args[0]); Topology t = new Topology("StandardDeviationFilter"); final Random rand = new Random(); // Produce a stream of random double values with a normal // distribution, mean 0.0 and standard deviation 1. TStream<Double> values = t.limitedSource(new Supplier<Double>() { private static final long serialVersionUID = 1L; @Override/* w w w . ja va 2 s .c o m*/ public Double get() { return rand.nextGaussian(); } }, 100000); /* * Filters the values based on calculating the mean and standard * deviation from the incoming data. In this case only outliers are * present in the output stream outliers. A outlier is defined as one * more than (threshold*standard deviation) from the mean. * * This demonstrates an anonymous functional logic class that is * stateful. The two fields mean and sd maintain their values across * multiple invocations of the test method, that is for multiple tuples. * * Note both Mean & StandardDeviation classes are serializable. */ TStream<Double> outliers = values.filter(new Predicate<Double>() { private static final long serialVersionUID = 1L; private final Mean mean = new Mean(); private final StandardDeviation sd = new StandardDeviation(); @Override public boolean test(Double tuple) { mean.increment(tuple); sd.increment(tuple); double multpleSd = threshold * sd.getResult(); double absMean = Math.abs(mean.getResult()); double absTuple = Math.abs(tuple); return absTuple > absMean + multpleSd; } }); outliers.print(); StreamsContextFactory.getEmbedded().submit(t).get(); }