List of usage examples for org.apache.commons.math.stat.descriptive.moment Mean evaluate
@Override public double evaluate(final double[] values)
From source file:de.tudarmstadt.ukp.dkpro.tc.crfsuite.CRFSuiteBatchCrossValidationReport.java
@Override public void execute() throws Exception { StorageService store = getContext().getStorageService(); FlexTable<String> table = FlexTable.forClass(String.class); Map<String, List<Double>> key2resultValues = new HashMap<String, List<Double>>(); for (TaskContextMetadata subcontext : getSubtasks()) { String name = ExperimentCrossValidation.class.getSimpleName(); // one CV batch (which internally ran numFolds times) if (subcontext.getLabel().startsWith(name)) { Map<String, String> discriminatorsMap = store .retrieveBinary(subcontext.getId(), Task.DISCRIMINATORS_KEY, new PropertiesAdapter()) .getMap();//from w w w . j a v a 2s.c o m File eval = store.getStorageFolder(subcontext.getId(), EVAL_FILE_NAME + SUFFIX_CSV); Map<String, String> resultMap = new HashMap<String, String>(); String[][] evalMatrix = null; int i = 0; for (String line : FileUtils.readLines(eval)) { String[] tokenizedLine = StrTokenizer.getCSVInstance(line).getTokenArray(); if (evalMatrix == null) { evalMatrix = new String[FileUtils.readLines(eval).size()][tokenizedLine.length]; } evalMatrix[i] = tokenizedLine; i++; } // columns for (int j = 0; j < evalMatrix[0].length; j++) { String header = evalMatrix[0][j]; String[] vals = new String[evalMatrix.length - 1]; // rows for (int k = 1; k < evalMatrix.length; k++) { if (evalMatrix[k][j].equals("null")) { vals[k - 1] = String.valueOf(0.); } else { vals[k - 1] = evalMatrix[k][j]; } } Mean mean = new Mean(); Sum sum = new Sum(); StandardDeviation std = new StandardDeviation(); double[] dVals = new double[vals.length]; Set<String> sVals = new HashSet<String>(); for (int k = 0; k < vals.length; k++) { try { dVals[k] = Double.parseDouble(vals[k]); sVals = null; } catch (NumberFormatException e) { dVals = null; sVals.add(vals[k]); } } if (dVals != null) { if (nonAveragedResultsMeasures.contains(header)) { resultMap.put(header + foldSum, String.valueOf(sum.evaluate(dVals))); } else { resultMap.put(header + foldAveraged, String.valueOf( mean.evaluate(dVals) + "\u00B1" + String.valueOf(std.evaluate(dVals)))); } } else { if (sVals.size() > 1) { resultMap.put(header, "---"); } else { resultMap.put(header, vals[0]); } } } String key = getKey(discriminatorsMap); List<Double> results; if (key2resultValues.get(key) == null) { results = new ArrayList<Double>(); } else { results = key2resultValues.get(key); } key2resultValues.put(key, results); Map<String, String> values = new HashMap<String, String>(); Map<String, String> cleanedDiscriminatorsMap = new HashMap<String, String>(); for (String disc : discriminatorsMap.keySet()) { if (!ReportUtils.containsExcludePattern(disc, discriminatorsToExclude)) { cleanedDiscriminatorsMap.put(disc, discriminatorsMap.get(disc)); } } values.putAll(cleanedDiscriminatorsMap); values.putAll(resultMap); table.addRow(subcontext.getLabel(), values); } } getContext().getLoggingService().message(getContextLabel(), ReportUtils.getPerformanceOverview(table)); // Excel cannot cope with more than 255 columns if (table.getColumnIds().length <= 255) { getContext().storeBinary(EVAL_FILE_NAME + "_compact" + SUFFIX_EXCEL, table.getExcelWriter()); } getContext().storeBinary(EVAL_FILE_NAME + "_compact" + SUFFIX_CSV, table.getCsvWriter()); table.setCompact(false); // Excel cannot cope with more than 255 columns if (table.getColumnIds().length <= 255) { getContext().storeBinary(EVAL_FILE_NAME + SUFFIX_EXCEL, table.getExcelWriter()); } getContext().storeBinary(EVAL_FILE_NAME + SUFFIX_CSV, table.getCsvWriter()); // output the location of the batch evaluation folder // otherwise it might be hard for novice users to locate this File dummyFolder = store.getStorageFolder(getContext().getId(), "dummy"); // TODO can we also do this without creating and deleting the dummy folder? getContext().getLoggingService().message(getContextLabel(), "Storing detailed results in:\n" + dummyFolder.getParent() + "\n"); dummyFolder.delete(); }
From source file:org.apache.drill.exec.physical.impl.orderedpartitioner.TestOrderedPartitionExchange.java
/** * Starts two drillbits and runs a physical plan with a Mock scan, project, OrderedParititionExchange, Union Exchange, * and sort. The final sort is done first on the partition column, and verifies that the partitions are correct, in that * all rows in partition 0 should come in the sort order before any row in partition 1, etc. Also verifies that the standard * deviation of the size of the partitions is less than one tenth the mean size of the partitions, because we expect all * the partitions to be roughly equal in size. * @throws Exception//from w w w . jav a 2 s . c o m */ @Test public void twoBitTwoExchangeRun() throws Exception { RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet(); try (Drillbit bit1 = new Drillbit(CONFIG, serviceSet); Drillbit bit2 = new Drillbit(CONFIG, serviceSet); DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator());) { bit1.run(); bit2.run(); client.connect(); List<QueryDataBatch> results = client.runQuery( org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile("/sender/ordered_exchange.json"), Charsets.UTF_8)); int count = 0; List<Integer> partitionRecordCounts = Lists.newArrayList(); for (QueryDataBatch b : results) { if (b.getData() != null) { int rows = b.getHeader().getRowCount(); count += rows; RecordBatchLoader loader = new RecordBatchLoader( new BootStrapContext(DrillConfig.create()).getAllocator()); loader.load(b.getHeader().getDef(), b.getData()); BigIntVector vv1 = (BigIntVector) loader.getValueAccessorById(BigIntVector.class, loader .getValueVectorId(new SchemaPath("col1", ExpressionPosition.UNKNOWN)).getFieldIds()) .getValueVector(); Float8Vector vv2 = (Float8Vector) loader.getValueAccessorById(Float8Vector.class, loader .getValueVectorId(new SchemaPath("col2", ExpressionPosition.UNKNOWN)).getFieldIds()) .getValueVector(); IntVector pVector = (IntVector) loader.getValueAccessorById(IntVector.class, loader.getValueVectorId(new SchemaPath("partition", ExpressionPosition.UNKNOWN)) .getFieldIds()) .getValueVector(); long previous1 = Long.MIN_VALUE; double previous2 = Double.MIN_VALUE; int partPrevious = -1; long current1 = Long.MIN_VALUE; double current2 = Double.MIN_VALUE; int partCurrent = -1; int partitionRecordCount = 0; for (int i = 0; i < rows; i++) { previous1 = current1; previous2 = current2; partPrevious = partCurrent; current1 = vv1.getAccessor().get(i); current2 = vv2.getAccessor().get(i); partCurrent = pVector.getAccessor().get(i); Assert.assertTrue(current1 >= previous1); if (current1 == previous1) { Assert.assertTrue(current2 <= previous2); } if (partCurrent == partPrevious || partPrevious == -1) { partitionRecordCount++; } else { partitionRecordCounts.add(partitionRecordCount); partitionRecordCount = 0; } } partitionRecordCounts.add(partitionRecordCount); loader.clear(); } b.release(); } double[] values = new double[partitionRecordCounts.size()]; int i = 0; for (Integer rc : partitionRecordCounts) { values[i++] = rc.doubleValue(); } StandardDeviation stdDev = new StandardDeviation(); Mean mean = new Mean(); double std = stdDev.evaluate(values); double m = mean.evaluate(values); System.out.println("mean: " + m + " std dev: " + std); //Assert.assertTrue(std < 0.1 * m); assertEquals(31000, count); } }
From source file:org.coode.distance.entityrelevance.AbstractRankingRelevancePolicy.java
private final double computeMean() { List<Double> rankingValues = new ArrayList<Double>(); for (RankingSlot<P, Double> slot : this.getRanking().getRanking()) { Double value = slot.getValue(); Iterator<P> iterator = slot.getMembers().iterator(); while (iterator.hasNext()) { iterator.next();//from w w w . j a va 2s .c o m rankingValues.add(value); } } double[] values = new double[rankingValues.size()]; int i = 0; for (Double d : rankingValues) { values[i] = d; i++; } Mean m = new Mean(); return m.evaluate(values); }
From source file:org.graphstream.algorithm.measure.CommunityDistribution.java
@Override /**// w w w.j a v a 2 s . c o m * Computes and update the statistical information on size distribution. * * @complexity O(C), where C is the expected number of communities. */ public void compute() { if (graphChanged) { // Default measure is the number of communities M = (float) communities.size(); // Update the smallest/biggest community // and creates the size distribution int maxSize = 0; int minSize = Integer.MAX_VALUE; double[] distribution = new double[(int) M]; int k = 0; Mean mean = new Mean(); StandardDeviation stdev = new StandardDeviation(); for (Object c : communities.keySet()) { distribution[k++] = (communities.get(c)).size(); if ((communities.get(c)).size() > maxSize) { biggestCommunity = c; maxSize = (communities.get(c)).size(); } if ((communities.get(c)).size() < minSize) { smallestCommunity = c; minSize = (communities.get(c)).size(); } } // Compute the statistical moments avgSize = (float) mean.evaluate(distribution); stdevSize = (float) stdev.evaluate(distribution); graphChanged = false; } }
From source file:org.graphstream.algorithm.measure.MobileCommunityMeasure.java
public void compute() { if (graphChanged) { // Default measure is the number of communities M = (float) communities.size(); double[] avgValueDistribution = new double[(int) M]; double[] stdValueDistribution = new double[(int) M]; // double[] sizeDistribution = new double[(int) M]; int k = 0; for (Object c : communities.keySet()) { HashSet<Node> nodes = communities.get(c); int size = (int) nodes.size(); double[] valueDistribution = new double[size]; int j = 0; Mean mean = new Mean(); StandardDeviation stdev = new StandardDeviation(); // System.out.println("com " + c + ", nodes "+ size); for (Node n : nodes) { valueDistribution[j++] = n.getAttribute(this.mobMarker) == null ? 0.0 : (Double) n.getAttribute(this.mobMarker); }/*ww w . ja v a 2 s .c o m*/ // Compute the statistical moments float avgValue = (float) mean.evaluate(valueDistribution); float stdevValue = (float) stdev.evaluate(valueDistribution); avgValueDistribution[k] = avgValue; stdValueDistribution[k] = stdevValue; ++k; } // Compute the statistical moments Mean mean = new Mean(); StandardDeviation stdev = new StandardDeviation(); avgValue = (float) mean.evaluate(avgValueDistribution); stdevValue = (float) stdev.evaluate(avgValueDistribution); avgStddev = (float) mean.evaluate(stdValueDistribution); graphChanged = false; } }
From source file:rrlFramework.RRLExperiment.java
/** * Compiles the performance files together into a single file, detailing the * average, min and max performances./*from w ww . java 2 s .co m*/ * * @param runEnd * The last run. * @param byEpisode * If the performances are being combined by episode (in * intervals) or by regular CE interval. */ private long combineTempFiles(File performanceFile, int runEnd, long experimentStart) throws Exception { List<List<Float[]>> performances = new ArrayList<List<Float[]>>(); float min = Float.MAX_VALUE; int minRun = -1; float max = -Float.MAX_VALUE; int maxRun = -1; double[] episodeLengths = new double[runEnd]; double[] numSlots = new double[runEnd]; long averageRunTime = 0; File combinedPerfFile = performanceFile; if (Config.getInstance().getGeneratorFile() != null) { combinedPerfFile = new File(performanceFile.getAbsolutePath() + "greedy"); ProgramArgument.PERFORMANCE_EPISODE_GAP .setDoubleValue(ProgramArgument.PERFORMANCE_TESTING_SIZE.intValue() * ProgramArgument.POLICY_REPEATS.intValue()); } if (!combinedPerfFile.exists()) combinedPerfFile.createNewFile(); // For every performance file for (int i = 0; i < runEnd; i++) { File tempPerf = new File(Config.TEMP_FOLDER + "/" + performanceFile + i); if (!Performance.readRawPerformanceFile(tempPerf, true)) { System.err.println("Error reading performance file."); return 0; } List<Float[]> thisRunPerformances = new ArrayList<Float[]>(); performances.add(thisRunPerformances); // Run through the performances and place them in the matrix SortedMap<Integer, Float[]> runPerformances = Performance.getPerformanceArray(); averageRunTime += Performance.getRunTime(); Iterator<Integer> iter = runPerformances.keySet().iterator(); Integer current = iter.next(); Integer previous = null; int currentKeyframeEpisode = ProgramArgument.PERFORMANCE_EPISODE_GAP.intValue(); // Run through the performances, using linear interpolation to // get estimates of the performance at a given interval. do { // If the current segment is further along than the current // value, advance to the next value. while (currentKeyframeEpisode > current) { previous = current; if (iter.hasNext()) current = iter.next(); else break; } // If the keyframe isn't up to the first episode, just use // the current value Float[] episodePerformance = runPerformances.get(current); if (previous == null) { // Add to the previous value. thisRunPerformances.add(episodePerformance); } else { // Interpolate from the previous value to the current // one. Float[] interpolatedPerformance = new Float[episodePerformance.length]; if (previous == current) { interpolatedPerformance = episodePerformance; } else { Float[] prevPerformance = runPerformances.get(previous); for (int j = 0; j < episodePerformance.length; j++) { Float currPerf = episodePerformance[j]; Float prevPerf = prevPerformance[j]; // Adjust for null elites if (j == PerformanceDetails.ELITEMAX.ordinal() || j == PerformanceDetails.ELITEMEAN.ordinal()) { if (currPerf == null) currPerf = episodePerformance[PerformanceDetails.MEAN.ordinal()]; if (prevPerf == null) prevPerf = prevPerformance[PerformanceDetails.MEAN.ordinal()]; } if (currPerf == null || prevPerf == null) interpolatedPerformance[j] = null; else interpolatedPerformance[j] = (currPerf - prevPerf) * (1f * (currentKeyframeEpisode - previous) / (current - previous)) + prevPerf; } } // Add to the performances thisRunPerformances.add(interpolatedPerformance); } // To the next increment currentKeyframeEpisode += ProgramArgument.PERFORMANCE_EPISODE_GAP.intValue(); } while (currentKeyframeEpisode <= runPerformances.lastKey()); Float[] lastPerf = runPerformances.get(runPerformances.lastKey()); thisRunPerformances.add(lastPerf); System.out.println(runPerformances.get(runPerformances.lastKey())[PerformanceDetails.MEAN.ordinal()]); // Find min or max runs float runVal = runPerformances.get(runPerformances.lastKey())[PerformanceDetails.MEAN.ordinal()]; if (runVal < min) { min = runVal; minRun = i; } if (runVal > max) { max = runVal; maxRun = i; } episodeLengths[i] = runPerformances.lastKey(); } // Calculate the average and print out the stats FileWriter writer = new FileWriter(combinedPerfFile); BufferedWriter buf = new BufferedWriter(writer); Config.writeFileHeader(buf, Config.getInstance().getGoal()); buf.write( "Episode\tAverage\tSD\tMin\tMax\tElite-Average\tElite-SD\tNumSlots\tSlots-SD\tNumRules\tRules-SD\n"); boolean moreEpisodes = true; int index = 0; Mean mean = new Mean(); StandardDeviation sd = new StandardDeviation(); while (moreEpisodes) { moreEpisodes = false; // Compile the array of performances for the given index double[][] performanceArray = new double[PerformanceDetails.values().length][performances.size()]; double maxVal = 0; double minVal = 0; for (int run = 0; run < performances.size(); run++) { List<Float[]> runPerformanceList = performances.get(run); int thisIndex = Math.min(index, runPerformanceList.size() - 1); if (index < runPerformanceList.size() - 1) moreEpisodes = true; Float[] performanceDetails = runPerformanceList.get(thisIndex); for (int j = 0; j < performanceDetails.length; j++) { if (performanceDetails[j] != null) performanceArray[j][run] = performanceDetails[j]; } // Max and min if (run == minRun) minVal = performanceArray[PerformanceDetails.MEAN.ordinal()][run]; if (run == maxRun) maxVal = performanceArray[PerformanceDetails.MEAN.ordinal()][run]; } // Find the statistics int episodeNum = (index + 1) * ProgramArgument.PERFORMANCE_EPISODE_GAP.intValue(); buf.write(episodeNum + "\t" + mean.evaluate(performanceArray[PerformanceDetails.MEAN.ordinal()]) + "\t" + sd.evaluate(performanceArray[PerformanceDetails.MEAN.ordinal()]) + "\t" + minVal + "\t" + maxVal + "\t" + mean.evaluate(performanceArray[PerformanceDetails.ELITEMEAN.ordinal()]) + "\t" + sd.evaluate(performanceArray[PerformanceDetails.ELITEMEAN.ordinal()]) + "\t" + mean.evaluate(performanceArray[PerformanceDetails.NUMSLOTS.ordinal()]) + "\t" + sd.evaluate(performanceArray[PerformanceDetails.NUMSLOTS.ordinal()]) + "\t" + mean.evaluate(performanceArray[PerformanceDetails.NUMRULES.ordinal()]) + "\t" + sd.evaluate(performanceArray[PerformanceDetails.NUMRULES.ordinal()]) + "\n"); index++; } averageRunTime /= runEnd; buf.write("Average Run Time: " + toTimeFormat(averageRunTime) + "\n"); // Write the average episode length buf.write("\nAverage episode length: " + mean.evaluate(episodeLengths) + " +- " + sd.evaluate(episodeLengths) + "\n"); buf.write("\nAverage num slots: " + mean.evaluate(numSlots) + " +- " + sd.evaluate(numSlots) + "\n"); buf.close(); writer.close(); return averageRunTime; }