List of usage examples for org.apache.commons.math3.stat.descriptive DescriptiveStatistics DescriptiveStatistics
public DescriptiveStatistics()
From source file:de.tudarmstadt.ukp.dkpro.core.performance.Stopwatch.java
@Override public void collectionProcessComplete() throws AnalysisEngineProcessException { super.collectionProcessComplete(); if (isDownstreamTimer()) { getLogger().info("Results from Timer '" + timerName + "' after processing all documents."); DescriptiveStatistics statTimes = new DescriptiveStatistics(); for (Long timeValue : times) { statTimes.addValue((double) timeValue / 1000); }// w w w. j a v a 2s.co m double sum = statTimes.getSum(); double mean = statTimes.getMean(); double stddev = statTimes.getStandardDeviation(); StringBuilder sb = new StringBuilder(); sb.append("Estimate after processing " + times.size() + " documents."); sb.append("\n"); Formatter formatter = new Formatter(sb, Locale.US); formatter.format("Aggregated time: %,.1fs\n", sum); formatter.format("Time / Document: %,.3fs (%,.3fs)\n", mean, stddev); formatter.close(); getLogger().info(sb.toString()); if (outputFile != null) { try { Properties props = new Properties(); props.setProperty(KEY_SUM, "" + sum); props.setProperty(KEY_MEAN, "" + mean); props.setProperty(KEY_STDDEV, "" + stddev); OutputStream out = new FileOutputStream(outputFile); props.store(out, "timer " + timerName + " result file"); } catch (FileNotFoundException e) { throw new AnalysisEngineProcessException(e); } catch (IOException e) { throw new AnalysisEngineProcessException(e); } } } }
From source file:com.intuit.tank.persistence.databases.MetricsCalculator.java
/** * @param object2//from ww w.ja v a2 s . co m * @param start * @{inheritDoc */ public void retrieveAndCalculateTimingData(@Nonnull String jobId, Date start, Date end) { MethodTimer mt = new MethodTimer(LOG, this.getClass(), "retrieveAndCalculateSummaryTimingCsv"); int period = 15; Writer csvFile = null; CSVWriter csvWriter = null; InputStream is = null; try { ResultsReader resultsReader = ReportingFactory.getResultsReader(); String fileName = "timing_" + new TankConfig().getInstanceName() + "_" + jobId + ".csv.gz"; File f = File.createTempFile("timing", ".csv.gz"); csvFile = new OutputStreamWriter(new GZIPOutputStream(new FileOutputStream(f))); csvWriter = new CSVWriter(csvFile); int count = 0; Object nextToken = null; csvWriter.writeNext(FIELDS); do { PagedTimingResults results = resultsReader.getPagedTimingResults(jobId, nextToken); for (TankResult result : results.getResults()) { count++; String[] entryArray = getCsvArray(result); csvWriter.writeNext(entryArray); if (count % 1000 == 0) { csvWriter.flush(); } double d = result.getResponseTime(); if (!skipDate(result.getTimeStamp(), start, end)) { DescriptiveStatistics statistics = summaryResults.get(result.getRequestName()); if (statistics == null) { statistics = new DescriptiveStatistics(); summaryResults.put(result.getRequestName(), statistics); } statistics.addValue(d); } if (result.getTimeStamp() != null) { Date periodDate = TimeUtil.normalizeToPeriod(period, result.getTimeStamp()); DescriptiveStatistics bucketStats = getBucketStats(result.getRequestName(), period, periodDate); bucketStats.addValue(d); } } nextToken = results.getNextToken(); } while (nextToken != null); csvWriter.flush(); csvWriter.close(); csvWriter = null; IOUtils.closeQuietly(csvFile); FileStorage fileStorage = FileStorageFactory.getFileStorage(new TankConfig().getTimingDir(), false); FileData fd = new FileData("", fileName); is = new FileInputStream(f); fileStorage.storeFileData(fd, is); mt.endAndLog(); LOG.info("Processed " + count + " total items for job " + jobId); } catch (Exception e) { if (e instanceof RuntimeException) { throw (RuntimeException) e; } throw new RuntimeException(e); } finally { if (csvWriter != null) { try { csvWriter.close(); } catch (IOException e) { // swallow LOG.warn("Error closing csv file: " + e); } } IOUtils.closeQuietly(csvFile); IOUtils.closeQuietly(is); } }
From source file:ijfx.core.stats.DefaultImageStatisticsService.java
@Override public DescriptiveStatistics getPlaneDescriptiveStatistics(Dataset dataset, long[] position) { DescriptiveStatistics stats = new DescriptiveStatistics(); double width = dataset.max(0) + 1; double height = dataset.max(1) + 1; RandomAccess<RealType<?>> randomAccess = dataset.randomAccess(); randomAccess.setPosition(position);/*from w ww . j av a 2 s . co m*/ for (int x = 0; x < width; x++) { randomAccess.setPosition(x, 0); for (int y = 0; y < height; y++) { randomAccess.setPosition(y, 1); stats.addValue(randomAccess.get().getRealDouble()); } } return stats; }
From source file:io.yields.math.framework.DomainTest.java
@Explore(name = "Test Variable Distribution", dataProvider = DataProviders.FixedMersenneTwisterDataProvider.class) @Exploration(name = "Test Function", context = FunctionExplorerContext.class, group = "domain") public void testVariableDistribution(Explorer<Double> explorer) { assertThat(explorer.all().count()).isEqualTo(explorer.valid().count()); assertThat(explorer.valid().count()).isEqualTo(explorer.valid().count()); assertThat(explorer.invalid().count()).isEqualTo(0); assertThat(explorer.propertyError().count()).isEqualTo(0); DescriptiveStatistics stats = new DescriptiveStatistics(); explorer.all().forEach(result -> stats.addValue(result.getFunctionOutcome().orElse(0d))); assertThat(stats.getMean()).isEqualTo(0, delta(0.1)); assertThat(stats.getMax()).isEqualTo(1, delta(0.1)); assertThat(stats.getMin()).isEqualTo(-1, delta(0.1)); }
From source file:com.linuxbox.enkive.teststats.StatsDayGrainTest.java
@Test public void consolidationMethods() { List<Map<String, Object>> consolidatedData = grain.consolidateData(); assertTrue("the consolidated data is null", consolidatedData != null); String methods[] = { CONSOLIDATION_AVG, CONSOLIDATION_MAX, CONSOLIDATION_MIN }; DescriptiveStatistics statsMaker = new DescriptiveStatistics(); statsMaker.addValue(111);/*from w w w. j a va2 s .co m*/ statsMaker.addValue(11); statsMaker.addValue(1); Map<String, Object> statData = new HashMap<String, Object>(); for (String method : methods) { grain.methodMapBuilder(method, statsMaker, statData); } assertTrue("methodMapBuilder returned null", statData != null); }
From source file:edu.snu.leader.hierarchy.simple.DefaultReporter.java
/** * Report the final results of the simulation * * @see edu.snu.leader.hierarchy.simple.Reporter#reportFinalResults() */// www .ja v a2 s . c o m @Override public void reportFinalResults() { // Create some handy variables long firstActiveTimestep = Long.MAX_VALUE; long lastActiveTimestep = Long.MIN_VALUE; int initiatorCount = 0; // Gather some statistics DescriptiveStatistics immediateFollowerStats = new DescriptiveStatistics(); DescriptiveStatistics initiatorDistanceStats = new DescriptiveStatistics(); DescriptiveStatistics activeTimestepStats = new DescriptiveStatistics(); // Iterate through all the individuals Iterator<Individual> indIter = _simState.getAllIndividuals().iterator(); while (indIter.hasNext()) { Individual ind = indIter.next(); // Get some statistics immediateFollowerStats.addValue(ind.getImmediateFollowerCount()); initiatorDistanceStats.addValue(ind.getDistanceToInitiator()); activeTimestepStats.addValue(ind.getActiveTimestep()); // Build the prefix String prefix = "individual." + ind.getID() + "."; // Log out important information _writer.println(prefix + "group-id = " + ind.getGroupID()); _writer.println(prefix + "active-timestep = " + ind.getActiveTimestep()); _writer.println(prefix + "immediate-follower-count = " + ind.getImmediateFollowerCount()); _writer.println(prefix + "total-follower-count = " + ind.getTotalFollowerCount()); _writer.println(prefix + "distance-to-initiator = " + ind.getDistanceToInitiator()); _writer.println(prefix + "location = " + ind.getLocation().getX() + " " + ind.getLocation().getY()); _writer.println(prefix + "threshold = " + ind.getThreshold()); _writer.println(prefix + "skill = " + ind.getSkill()); _writer.println(prefix + "confidence = " + ind.getConfidence()); _writer.println(prefix + "reputation = " + ind.getReputation()); _writer.println(prefix + "boldness = " + ind.getBoldness()); // Get the leader's ID, if it exists Object leaderID = ""; if (null != ind.getLeader()) { leaderID = ind.getLeader().getIndividual().getID(); } else { ++initiatorCount; } _writer.println(prefix + "leader = " + leaderID); // Build the list of neighbor ID's StringBuilder builder = new StringBuilder(); Iterator<Neighbor> neighborIter = ind.getNearestNeighbors().iterator(); while (neighborIter.hasNext()) { builder.append(neighborIter.next().getIndividual().getID()); builder.append(" "); } _writer.println(prefix + "nearest-neighbors = " + builder.toString()); // Build the list of follower ID's builder = new StringBuilder(); neighborIter = ind.getFollowers().iterator(); while (neighborIter.hasNext()) { builder.append(neighborIter.next().getIndividual().getID()); builder.append(" "); } _writer.println(prefix + "immediate-followers = " + builder.toString()); // Check the activity time if (firstActiveTimestep > ind.getActiveTimestep()) { firstActiveTimestep = ind.getActiveTimestep(); } if (lastActiveTimestep < ind.getActiveTimestep()) { lastActiveTimestep = ind.getActiveTimestep(); } _writer.println(); } // Log the simulation information _writer.println("simulation.first-active-timestep = " + firstActiveTimestep); _writer.println("simulation.last-active-timestep = " + lastActiveTimestep); _writer.println("simulation.initiator-count = " + initiatorCount); // Log the stats _writer.println("statistics.immediate-followers.mean = " + immediateFollowerStats.getMean()); _writer.println( "statistics.immediate-followers.std-dev = " + immediateFollowerStats.getStandardDeviation()); _writer.println("statistics.immediate-followers.min = " + immediateFollowerStats.getMin()); _writer.println("statistics.immediate-followers.max = " + immediateFollowerStats.getMax()); _writer.println("statistics.initiator-distance.mean = " + initiatorDistanceStats.getMean()); _writer.println("statistics.initiator-distance.std-dev = " + initiatorDistanceStats.getStandardDeviation()); _writer.println("statistics.initiator-distance.min = " + initiatorDistanceStats.getMin()); _writer.println("statistics.initiator-distance.max = " + initiatorDistanceStats.getMax()); _writer.println("statistics.active-timestep.mean = " + activeTimestepStats.getMean()); _writer.println("statistics.active-timestep.std-dev = " + activeTimestepStats.getStandardDeviation()); _writer.println("statistics.active-timestep.min = " + activeTimestepStats.getMin()); _writer.println("statistics.active-timestep.max = " + activeTimestepStats.getMax()); // Log out the stop time _writer.println(); _writer.println(_STATS_SPACER); _writer.println("# Finished: " + (new Date())); // Close out the writer _writer.close(); }
From source file:com.itemanalysis.jmetrik.stats.descriptives.DescriptiveAnalysis.java
public void summarize() throws SQLException { Statement stmt = null;// ww w .jav a 2 s .com ResultSet rs = null; DescriptiveStatistics temp = null; Table sqlTable = new Table(tableName.getNameForDatabase()); SelectQuery select = new SelectQuery(); for (VariableAttributes v : variables) { select.addColumn(sqlTable, v.getName().nameForDatabase()); } stmt = conn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY); rs = stmt.executeQuery(select.toString()); double value = Double.NaN; while (rs.next()) { for (VariableAttributes v : variables) { temp = data.get(v); if (temp == null) { temp = new DescriptiveStatistics(); data.put(v, temp); } //only increment for non null doubles value = rs.getDouble(v.getName().nameForDatabase()); if (!rs.wasNull()) { temp.addValue(value); } } updateProgress(); } rs.close(); stmt.close(); for (VariableAttributes v : data.keySet()) { publishTable(v); } }
From source file:de.tudarmstadt.ukp.experiments.argumentation.convincingness.graph.VisualizationTest.java
/** * Computes max. transitivity score for the given node (as described in the paper) * <p/>/*w ww. j a va 2 s . c om*/ * It uses Bellman-Ford algorithm to compute the shortest and longest paths * * @param graph graph (must be DAG) * @return statistics */ private static int computeMaxTransitivityScore(Graph graph, Node sourceNode) { if (sourceNode.getOutDegree() == 0) { return 0; } // find all out-degree > 1 nodes Set<Node> targetNodes = new HashSet<>(); for (Node n : graph) { if (n.getInDegree() > 1) { targetNodes.add(n); } } FileSourceDGS source = new FileSourceDGS(); source.addSink(graph); DescriptiveStatistics result = new DescriptiveStatistics(); // set positive weight first for (Edge e : graph.getEdgeSet()) { e.setAttribute(WEIGHT, 1.0); } BellmanFord bfShortest = new BellmanFord(WEIGHT, sourceNode.getId()); bfShortest.init(graph); bfShortest.compute(); // now negative weight for longest-path for (Edge e : graph.getEdgeSet()) { e.setAttribute(WEIGHT, -1.0); } BellmanFord bfLongest = new BellmanFord(WEIGHT, sourceNode.getId()); bfLongest.init(graph); bfLongest.compute(); int maxTransitivityScore = Integer.MIN_VALUE; for (Node targetNode : targetNodes) { Path shortestPath = bfShortest.getShortestPath(targetNode); Path longestPath = bfLongest.getShortestPath(targetNode); int shortestPathLength = shortestPath.getEdgeCount(); int longestPathLength = longestPath.getEdgeCount(); if (shortestPathLength == 1 && longestPathLength > 1) { // update statistics maxTransitivityScore = Math.max(maxTransitivityScore, longestPathLength); } } // none found maxTransitivityScore = Math.max(maxTransitivityScore, 0); return maxTransitivityScore; }
From source file:de.tudarmstadt.ukp.dkpro.core.performance.ThroughputTestAE.java
public String getPerformanceaAnalysis() { StringBuilder sb = new StringBuilder(); long sumMillis = 0; for (double timeValue : times) { sumMillis += timeValue;/* w ww .j a v a 2 s .co m*/ } DescriptiveStatistics statTimes = new DescriptiveStatistics(); for (Long timeValue : times) { statTimes.addValue((double) timeValue / 1000); } sb.append("Estimate after processing " + times.size() + " documents."); sb.append(LF); Formatter formatter = new Formatter(sb, Locale.US); formatter.format("Time / Document: %,.3f (%,.3f)\n", statTimes.getMean(), statTimes.getStandardDeviation()); formatter.format("Time / 10^4 Token: %,.3f\n", getNormalizedTime(sumMillis, nrofTokens, 1000)); formatter.format("Time / 10^4 Sentences: %,.3f\n", getNormalizedTime(sumMillis, nrofSentences, 1000)); formatter.close(); return sb.toString(); }
From source file:com.caseystella.analytics.outlier.streaming.mad.SketchyMovingMADIntegrationTest.java
@Test public void runAccuracyBenchmark() throws IOException { Map<String, List<String>> benchmarks = JSONUtil.INSTANCE.load( new FileInputStream(new File(new File(benchmarkRoot), "combined_labels.json")), new TypeReference<Map<String, List<String>>>() { });/*from ww w . ja v a 2 s . c o m*/ Assert.assertTrue(benchmarks.size() > 0); Map<ConfusionMatrix.ConfusionEntry, Long> overallConfusionMatrix = new HashMap<>(); DescriptiveStatistics globalExpectedScores = new DescriptiveStatistics(); long total = 0; for (Map.Entry<String, List<String>> kv : benchmarks.entrySet()) { File dataFile = new File(new File(benchmarkRoot), kv.getKey()); File plotFile = new File(new File(benchmarkRoot), kv.getKey() + ".dat"); Assert.assertTrue(dataFile.exists()); Set<Long> expectedOutliers = Sets.newHashSet(Iterables.transform(kv.getValue(), STR_TO_TS)); OutlierRunner runner = new OutlierRunner(outlierConfig, extractorConfigStr); final long[] numObservations = { 0L }; final long[] lastTimestamp = { Long.MIN_VALUE }; final DescriptiveStatistics timeDiffStats = new DescriptiveStatistics(); final Map<Long, Outlier> outlierMap = new HashMap<>(); final PrintWriter pw = new PrintWriter(plotFile); List<Outlier> outliers = runner.run(dataFile, 1, EnumSet.of(Severity.SEVERE_OUTLIER), new Function<Map.Entry<DataPoint, Outlier>, Void>() { @Nullable @Override public Void apply(@Nullable Map.Entry<DataPoint, Outlier> kv) { DataPoint dataPoint = kv.getKey(); Outlier outlier = kv.getValue(); pw.println(dataPoint.getTimestamp() + " " + outlier.getDataPoint().getValue() + " " + ((outlier.getSeverity() == Severity.SEVERE_OUTLIER) ? "outlier" : "normal")); outlierMap.put(dataPoint.getTimestamp(), outlier); numObservations[0] += 1; if (lastTimestamp[0] != Long.MIN_VALUE) { timeDiffStats.addValue(dataPoint.getTimestamp() - lastTimestamp[0]); } lastTimestamp[0] = dataPoint.getTimestamp(); return null; } }); pw.close(); total += numObservations[0]; Set<Long> calculatedOutliers = Sets .newHashSet(Iterables.transform(outliers, OutlierRunner.OUTLIER_TO_TS)); double stdDevDiff = Math.sqrt(timeDiffStats.getVariance()); System.out.println("Running data from " + kv.getKey() + " - E[time delta]: " + ConfusionMatrix.timeConversion((long) timeDiffStats.getMean()) + ", StdDev[time delta]: " + ConfusionMatrix.timeConversion((long) stdDevDiff) + " mean: " + runner.getMean()); Map<ConfusionMatrix.ConfusionEntry, Long> confusionMatrix = ConfusionMatrix.getConfusionMatrix( expectedOutliers, calculatedOutliers, numObservations[0], (long) timeDiffStats.getMean(), 3 //stdDevDiff > 30000?0:3 , outlierMap, globalExpectedScores); ConfusionMatrix.printConfusionMatrix(confusionMatrix); overallConfusionMatrix = ConfusionMatrix.merge(overallConfusionMatrix, confusionMatrix); } System.out.println("Really ran " + total); ConfusionMatrix.printConfusionMatrix(overallConfusionMatrix); ConfusionMatrix.printStats("Global Expected Outlier Scores", globalExpectedScores); }