List of usage examples for org.apache.commons.math3.stat.descriptive DescriptiveStatistics getMean
public double getMean()
From source file:org.alfresco.bm.api.v1.ResultsRestAPI.java
/** * Retrieve an approximate number of results, allowing for a smoothing factor * (<a href=http://en.wikipedia.org/wiki/Moving_average#Simple_moving_average>Simple Moving Average</a>) - * the number of data results to including in the moving average. * /* ww w . j a va 2 s. c o m*/ * @param fromTime the approximate time to start from * @param timeUnit the units of the 'reportPeriod' (default SECONDS). See {@link TimeUnit}. * @param reportPeriod how often a result should be output. This is expressed as a multiple of the 'timeUnit'. * @param smoothing the number of results to include in the Simple Moving Average calculations * @param chartOnly <tt>true</tt> to filter out results that are not of interest in performance charts * * @return JSON representing the event start time (x-axis) and the smoothed average execution time * along with data such as the events per second, failures per second, etc. */ @GET @Path("/ts") @Produces(MediaType.APPLICATION_JSON) public String getTimeSeriesResults(@DefaultValue("0") @QueryParam("fromTime") long fromTime, @DefaultValue("SECONDS") @QueryParam("timeUnit") String timeUnit, @DefaultValue("1") @QueryParam("reportPeriod") long reportPeriod, @DefaultValue("1") @QueryParam("smoothing") int smoothing, @DefaultValue("true") @QueryParam("chartOnly") boolean chartOnly) { if (logger.isDebugEnabled()) { logger.debug("Inbound: " + "[test:" + test + ",fromTime:" + fromTime + ",timeUnit:" + timeUnit + ",reportPeriod:" + reportPeriod + ",smoothing:" + smoothing + ",chartOnly:" + chartOnly + "]"); } if (reportPeriod < 1) { throwAndLogException(Status.BAD_REQUEST, "'reportPeriod' must be 1 or more."); } if (smoothing < 1) { throwAndLogException(Status.BAD_REQUEST, "'smoothing' must be 1 or more."); } TimeUnit timeUnitEnum = null; try { timeUnitEnum = TimeUnit.valueOf(timeUnit.toUpperCase()); } catch (Exception e) { // Invalid time unit throwAndLogException(Status.BAD_REQUEST, e); } final ResultService resultService = getResultService(); // Calculate the window size long reportPeriodMs = timeUnitEnum.toMillis(reportPeriod); long windowSize = reportPeriodMs * smoothing; // This is just too convenient an API final BasicDBList events = new BasicDBList(); ResultHandler handler = new ResultHandler() { @Override public boolean processResult(long fromTime, long toTime, Map<String, DescriptiveStatistics> statsByEventName, Map<String, Integer> failuresByEventName) throws Throwable { for (Map.Entry<String, DescriptiveStatistics> entry : statsByEventName.entrySet()) { String eventName = entry.getKey(); DescriptiveStatistics stats = entry.getValue(); Integer failures = failuresByEventName.get(eventName); if (failures == null) { logger.error("Found null failure count: " + entry); // Do nothing with it and stop return false; } // Per second double numPerSec = (double) stats.getN() / ((double) (toTime - fromTime) / 1000.0); double failuresPerSec = (double) failures / ((double) (toTime - fromTime) / 1000.0); // Push into an object DBObject eventObj = BasicDBObjectBuilder.start().add("time", toTime).add("name", eventName) .add("mean", stats.getMean()).add("min", stats.getMin()).add("max", stats.getMax()) .add("stdDev", stats.getStandardDeviation()).add("num", stats.getN()) .add("numPerSec", numPerSec).add("fail", failures).add("failPerSec", failuresPerSec) .get(); // Add the object to the list of events events.add(eventObj); } // Go for the next result return true; } }; try { // Get all the results resultService.getResults(handler, fromTime, windowSize, reportPeriodMs, chartOnly); // Muster into JSON String json = events.toString(); // Done if (logger.isDebugEnabled()) { int jsonLen = json.length(); if (jsonLen < 500) { logger.debug("Outbound: " + json); } else { logger.debug("Outbound: " + json.substring(0, 250) + " ... " + json.substring(jsonLen - 250, jsonLen)); } } return json; } catch (WebApplicationException e) { throw e; } catch (Exception e) { throwAndLogException(Status.INTERNAL_SERVER_ERROR, e); return null; } }
From source file:org.alfresco.bm.event.mongo.MongoResultServiceTest.java
/** * Test the case where the reporting period is smaller than the stats window *///from www. j a va 2s. c o m @Test public void getCheckedResultsUsingHandler() { pumpRecords(10); final AtomicInteger count = new AtomicInteger(); final Map<String, DescriptiveStatistics> lastStatsByEventName = new HashMap<String, DescriptiveStatistics>( 17); resultService.getResults(new ResultHandler() { @Override public boolean processResult(long fromTime, long toTime, Map<String, DescriptiveStatistics> statsByEventName, Map<String, Integer> failuresByEventName) throws Throwable { // Always keep the last stats lastStatsByEventName.clear(); lastStatsByEventName.putAll(statsByEventName); count.incrementAndGet(); return true; } }, 0L, 200L, 10L, false); // Check assertEquals(10, count.get()); // Now go through the last stats received // Check it against the last window size List<String> names = resultService.getEventNames(); for (String eventName : names) { List<EventRecord> eventResults = resultService.getResults(eventName, 0, 1000); DescriptiveStatistics eventStats = new DescriptiveStatistics(); for (EventRecord eventRecord : eventResults) { eventStats.addValue(eventRecord.getTime()); } DescriptiveStatistics lastEventStats = lastStatsByEventName.get(eventName); assertNotNull("No last report for event '" + eventName + "'.", lastEventStats); // Now check that this matched the last report exactly assertEquals("Mean for '" + eventName + "' was not correct. ", (long) Math.floor(eventStats.getMean()), (long) Math.floor(lastStatsByEventName.get(eventName).getMean())); } }
From source file:org.alfresco.bm.report.XLSXReporter.java
private void createEventSheets(final XSSFWorkbook workbook) { // Create the fonts we need Font fontBold = workbook.createFont(); fontBold.setBoldweight(Font.BOLDWEIGHT_BOLD); // Create the styles we need CreationHelper helper = workbook.getCreationHelper(); final XSSFCellStyle dataStyle = workbook.createCellStyle(); dataStyle.setAlignment(HorizontalAlignment.RIGHT); final XSSFCellStyle headerStyle = workbook.createCellStyle(); headerStyle.setAlignment(HorizontalAlignment.RIGHT); headerStyle.setFont(fontBold);//from ww w .j ava2 s.c om final XSSFCellStyle dateStyle = workbook.createCellStyle(); dateStyle.setDataFormat(helper.createDataFormat().getFormat("HH:mm:ss")); // Calculate a good window size ResultService resultService = getResultService(); EventRecord firstResult = resultService.getFirstResult(); EventRecord lastResult = resultService.getLastResult(); if (firstResult == null || lastResult == null) { return; } long start = firstResult.getStartTime(); long end = lastResult.getStartTime(); long windowSize = AbstractEventReporter.getWindowSize(start, end, 100); // Well-known window sizes // Keep track of sheets by event name. Note that XLSX truncates sheets to 31 chars, so use 28 chars and ~01, ~02 final Map<String, String> sheetNames = new HashMap<String, String>(31); final Map<String, XSSFSheet> sheets = new HashMap<String, XSSFSheet>(31); final Map<String, AtomicInteger> rowNums = new HashMap<String, AtomicInteger>(31); ResultHandler handler = new ResultHandler() { @Override public boolean processResult(long fromTime, long toTime, Map<String, DescriptiveStatistics> statsByEventName, Map<String, Integer> failuresByEventName) throws Throwable { // Get or create a sheet for each event for (String eventName : statsByEventName.keySet()) { // What sheet name to we use? String sheetName = sheetNames.get(eventName); if (sheetName == null) { sheetName = eventName; if (eventName.length() > 28) { int counter = 1; // Find a sheet name not in use while (true) { sheetName = eventName.substring(0, 28); sheetName = String.format("%s~%02d", sheetName, counter); // Have we used this, yet? if (sheets.containsKey(sheetName)) { // Yes, we have used it. counter++; continue; } // This is unique break; } } sheetNames.put(eventName, sheetName); } // Get and create the sheet, if necessary XSSFSheet sheet = sheets.get(sheetName); if (sheet == null) { // Create try { sheet = workbook.createSheet(sheetName); sheets.put(sheetName, sheet); sheet.getHeader().setCenter(title + " - " + eventName); sheet.getPrintSetup().setFitWidth((short) 1); sheet.getPrintSetup().setLandscape(true); } catch (Exception e) { logger.error("Unable to create workbook sheet for event: " + eventName, e); continue; } // Intro XSSFCell cell = sheet.createRow(0).createCell(0); cell.setCellValue(title + " - " + eventName + ":"); cell.setCellStyle(headerStyle); // Headings XSSFRow row = sheet.createRow(1); cell = row.createCell(0); cell.setCellStyle(headerStyle); cell.setCellValue("time"); cell = row.createCell(1); cell.setCellStyle(headerStyle); cell.setCellValue("mean"); cell = row.createCell(2); cell.setCellStyle(headerStyle); cell.setCellValue("min"); cell = row.createCell(3); cell.setCellStyle(headerStyle); cell.setCellValue("max"); cell = row.createCell(4); cell.setCellStyle(headerStyle); cell.setCellValue("stdDev"); cell = row.createCell(5); cell.setCellStyle(headerStyle); cell.setCellValue("num"); cell = row.createCell(6); cell.setCellStyle(headerStyle); cell.setCellValue("numPerSec"); cell = row.createCell(7); cell.setCellStyle(headerStyle); cell.setCellValue("fail"); cell = row.createCell(8); cell.setCellStyle(headerStyle); cell.setCellValue("failPerSec"); // Size the columns sheet.autoSizeColumn(0); sheet.autoSizeColumn(1); sheet.autoSizeColumn(2); sheet.autoSizeColumn(3); sheet.autoSizeColumn(4); sheet.autoSizeColumn(5); sheet.autoSizeColumn(6); sheet.autoSizeColumn(7); sheet.autoSizeColumn(8); } AtomicInteger rowNum = rowNums.get(eventName); if (rowNum == null) { rowNum = new AtomicInteger(2); rowNums.put(eventName, rowNum); } DescriptiveStatistics stats = statsByEventName.get(eventName); Integer failures = failuresByEventName.get(eventName); double numPerSec = (double) stats.getN() / ((double) (toTime - fromTime) / 1000.0); double failuresPerSec = (double) failures / ((double) (toTime - fromTime) / 1000.0); XSSFRow row = sheet.createRow(rowNum.getAndIncrement()); XSSFCell cell; cell = row.createCell(0, Cell.CELL_TYPE_NUMERIC); cell.setCellStyle(dateStyle); cell.setCellValue(new Date(toTime)); cell = row.createCell(5, Cell.CELL_TYPE_NUMERIC); cell.setCellValue(stats.getN()); cell = row.createCell(6, Cell.CELL_TYPE_NUMERIC); cell.setCellValue(numPerSec); cell = row.createCell(7, Cell.CELL_TYPE_NUMERIC); cell.setCellValue(failures); cell = row.createCell(8, Cell.CELL_TYPE_NUMERIC); cell.setCellValue(failuresPerSec); // Leave out values if there is no mean if (Double.isNaN(stats.getMean())) { continue; } cell = row.createCell(1, Cell.CELL_TYPE_NUMERIC); cell.setCellValue(stats.getMean()); cell = row.createCell(2, Cell.CELL_TYPE_NUMERIC); cell.setCellValue(stats.getMin()); cell = row.createCell(3, Cell.CELL_TYPE_NUMERIC); cell.setCellValue(stats.getMax()); cell = row.createCell(4, Cell.CELL_TYPE_NUMERIC); cell.setCellValue(stats.getStandardDeviation()); } return true; } }; resultService.getResults(handler, start, windowSize, windowSize, false); // Create charts in the sheets for (String eventName : sheetNames.keySet()) { // Get the sheet name String sheetName = sheetNames.get(eventName); if (sheetName == null) { logger.error("Did not find sheet for event: " + eventName); continue; } // Get the sheet XSSFSheet sheet = sheets.get(sheetName); if (sheet == null) { logger.error("Did not find sheet for name: " + sheetName); continue; } // What row did we get up to AtomicInteger rowNum = rowNums.get(eventName); if (rowNum == null) { logger.error("Did not find row number for event: " + sheetName); continue; } // This axis is common to both charts ChartDataSource<Number> xTime = DataSources.fromNumericCellRange(sheet, new CellRangeAddress(1, rowNum.intValue() - 1, 0, 0)); // Graph of event times XSSFDrawing drawingTimes = sheet.createDrawingPatriarch(); ClientAnchor anchorTimes = drawingTimes.createAnchor(0, 0, 0, 0, 0, 5, 15, 25); Chart chartTimes = drawingTimes.createChart(anchorTimes); ChartLegend legendTimes = chartTimes.getOrCreateLegend(); legendTimes.setPosition(LegendPosition.BOTTOM); LineChartData chartDataTimes = chartTimes.getChartDataFactory().createLineChartData(); ChartAxis bottomAxisTimes = chartTimes.getChartAxisFactory().createCategoryAxis(AxisPosition.BOTTOM); bottomAxisTimes.setNumberFormat("#,##0;-#,##0"); ValueAxis leftAxisTimes = chartTimes.getChartAxisFactory().createValueAxis(AxisPosition.LEFT); // Mean ChartDataSource<Number> yMean = DataSources.fromNumericCellRange(sheet, new CellRangeAddress(1, rowNum.intValue() - 1, 1, 1)); LineChartSeries yMeanSerie = chartDataTimes.addSeries(xTime, yMean); yMeanSerie.setTitle(title + " - " + eventName + ": Mean (ms)"); // Std Dev ChartDataSource<Number> yStdDev = DataSources.fromNumericCellRange(sheet, new CellRangeAddress(1, rowNum.intValue() - 1, 4, 4)); LineChartSeries yStdDevSerie = chartDataTimes.addSeries(xTime, yStdDev); yStdDevSerie.setTitle(title + " - " + eventName + ": Standard Deviation (ms)"); // Plot event times chartTimes.plot(chartDataTimes, bottomAxisTimes, leftAxisTimes); // Graph of event volumes // Graph of event times XSSFDrawing drawingVolumes = sheet.createDrawingPatriarch(); ClientAnchor anchorVolumes = drawingVolumes.createAnchor(0, 0, 0, 0, 0, 25, 15, 35); Chart chartVolumes = drawingVolumes.createChart(anchorVolumes); ChartLegend legendVolumes = chartVolumes.getOrCreateLegend(); legendVolumes.setPosition(LegendPosition.BOTTOM); LineChartData chartDataVolumes = chartVolumes.getChartDataFactory().createLineChartData(); ChartAxis bottomAxisVolumes = chartVolumes.getChartAxisFactory() .createCategoryAxis(AxisPosition.BOTTOM); bottomAxisVolumes.setNumberFormat("#,##0;-#,##0"); ValueAxis leftAxisVolumes = chartVolumes.getChartAxisFactory().createValueAxis(AxisPosition.LEFT); // Number per second ChartDataSource<Number> yNumPerSec = DataSources.fromNumericCellRange(sheet, new CellRangeAddress(1, rowNum.intValue() - 1, 6, 6)); LineChartSeries yNumPerSecSerie = chartDataVolumes.addSeries(xTime, yNumPerSec); yNumPerSecSerie.setTitle(title + " - " + eventName + ": Events per Second"); // Failures per second ChartDataSource<Number> yFailPerSec = DataSources.fromNumericCellRange(sheet, new CellRangeAddress(1, rowNum.intValue() - 1, 8, 8)); LineChartSeries yFailPerSecSerie = chartDataVolumes.addSeries(xTime, yFailPerSec); yFailPerSecSerie.setTitle(title + " - " + eventName + ": Failures per Second"); // Plot volumes chartVolumes.plot(chartDataVolumes, bottomAxisVolumes, leftAxisVolumes); } }
From source file:org.apache.accumulo.core.file.rfile.RolllingStatsTest.java
private static void checkAgreement(DescriptiveStatistics ds, RollingStats rs) { // getting stats from ds is expensive, so do it once... otherwise unit test takes 11 sec // instead of 5 secs double expMean = ds.getMean(); double expVar = ds.getVariance(); double expStdDev = Math.sqrt(expVar); assertFuzzyEquals(expMean, rs.getMean()); assertFuzzyEquals(expVar, rs.getVariance()); assertFuzzyEquals(expStdDev, rs.getStandardDeviation()); assertTrue(expMean >= 0);// w w w .j a v a 2 s .com assertTrue(rs.getMean() >= 0); assertTrue(expVar >= 0); assertTrue(rs.getVariance() >= 0); assertTrue(expStdDev >= 0); assertTrue(rs.getStandardDeviation() >= 0); }
From source file:org.apache.groovy.perf.CompilerPerformanceTest.java
public static void main(String[] args) throws Exception { List<File> sources = new ArrayList<>(); List<URL> classpath = new ArrayList<>(); boolean isCp = false; for (String arg : args) { if ("-cp".equals(arg)) { isCp = true;//from ww w. j ava2s.c o m } else if (isCp) { classpath.add(new File(arg).toURI().toURL()); } else { sources.add(new File(arg)); } } ScriptCompilationExecuter executer = new ScriptCompilationExecuter( sources.toArray(new File[sources.size()]), classpath); System.out.println("Using Groovy " + GROOVY_VERSION); DescriptiveStatistics stats = new DescriptiveStatistics(); for (int i = 0; i < WARMUP + REPEAT; i++) { if (i < WARMUP) { System.out.println("Warmup #" + (i + 1)); } else { System.out.println("Round #" + (i - WARMUP)); } long dur = executer.execute(); System.gc(); System.out.printf("Compile time = %dms%n", dur); if (i >= WARMUP) { stats.addValue((double) dur); } } System.out.println("Compilation took " + stats.getMean() + "ms " + stats.getStandardDeviation() + "ms"); FileWriter wrt = new FileWriter(new File("target/compilation-stats.csv"), true); wrt.append(String.format("%s;%s;%s\n", GROOVY_VERSION, stats.getMean(), stats.getStandardDeviation())); wrt.close(); }
From source file:org.apache.hadoop.hive.metastore.tools.BenchmarkSuite.java
/** * Get new statistics that excludes values beyond mean +/- 2 * stdev * * @param data Source data// w w w .j ava 2 s.c om * @return new {@link @DescriptiveStatistics objects with sanitized data} */ private static DescriptiveStatistics sanitize(@NotNull DescriptiveStatistics data) { double meanValue = data.getMean(); double delta = MARGIN * meanValue; double minVal = meanValue - delta; double maxVal = meanValue + delta; return new DescriptiveStatistics( Arrays.stream(data.getValues()).filter(x -> x > minVal && x < maxVal).toArray()); }
From source file:org.apache.hadoop.hive.metastore.tools.BenchmarkSuite.java
/** * Produce printable result//from w ww. j a v a2s. c o m * @param fmt text formatter - destination of formatted results. * @param name benchmark name * @param stats benchmark data */ private void displayStats(@NotNull Formatter fmt, @NotNull String name, @NotNull DescriptiveStatistics stats) { double mean = stats.getMean(); double err = stats.getStandardDeviation() / mean * 100; long conv = scale.toNanos(1); fmt.format("%-30s %-8.4g %-8.4g %-8.4g %-8.4g %-8.4g%n", name, mean / conv, median(stats) / conv, stats.getMin() / conv, stats.getMax() / conv, err); }
From source file:org.apache.hadoop.hive.metastore.tools.BenchmarkSuite.java
/** * Produce results in printable CSV format, separated by separator. * @param fmt text formatter - destination of formatted results. * @param name benchmark name/* www . j ava 2 s . c om*/ * @param stats benchmark data * @param separator field separator */ private void displayCSV(@NotNull Formatter fmt, @NotNull String name, @NotNull DescriptiveStatistics stats, @NotNull String separator) { double mean = stats.getMean(); double err = stats.getStandardDeviation() / mean * 100; long conv = scale.toNanos(1); fmt.format("%s%s%g%s%g%s%g%s%g%s%g%n", name, separator, mean / conv, separator, median(stats) / conv, separator, stats.getMin() / conv, separator, stats.getMax() / conv, separator, err); }
From source file:org.apache.metron.common.math.stats.OnlineStatisticsProviderTest.java
public static void validateStatisticsProvider(StatisticsProvider statsProvider, SummaryStatistics summaryStats, DescriptiveStatistics stats) { //N/*from w w w . j a v a2 s. c o m*/ Assert.assertEquals(statsProvider.getCount(), stats.getN()); //sum Assert.assertEquals(statsProvider.getSum(), stats.getSum(), 1e-3); //sum of squares Assert.assertEquals(statsProvider.getSumSquares(), stats.getSumsq(), 1e-3); //sum of squares Assert.assertEquals(statsProvider.getSumLogs(), summaryStats.getSumOfLogs(), 1e-3); //Mean Assert.assertEquals(statsProvider.getMean(), stats.getMean(), 1e-3); //Quadratic Mean Assert.assertEquals(statsProvider.getQuadraticMean(), summaryStats.getQuadraticMean(), 1e-3); //SD Assert.assertEquals(statsProvider.getStandardDeviation(), stats.getStandardDeviation(), 1e-3); //Variance Assert.assertEquals(statsProvider.getVariance(), stats.getVariance(), 1e-3); //Min Assert.assertEquals(statsProvider.getMin(), stats.getMin(), 1e-3); //Max Assert.assertEquals(statsProvider.getMax(), stats.getMax(), 1e-3); //Kurtosis Assert.assertEquals(stats.getKurtosis(), statsProvider.getKurtosis(), 1e-3); //Skewness Assert.assertEquals(stats.getSkewness(), statsProvider.getSkewness(), 1e-3); for (double d = 10.0; d < 100.0; d += 10) { //This is a sketch, so we're a bit more forgiving here in our choice of \epsilon. Assert.assertEquals("Percentile mismatch for " + d + "th %ile", statsProvider.getPercentile(d), stats.getPercentile(d), 1e-2); } }
From source file:org.apache.metron.common.stellar.benchmark.Microbenchmark.java
public static String describe(DescriptiveStatistics stats, Double[] percentiles) { StringBuilder sb = new StringBuilder(); sb.append(String.format("round: mean of %dms [+-%d], measured %d rounds;\n", (long) stats.getMean(), (long) stats.getStandardDeviation(), stats.getN())); sb.append("\tMin - " + (long) stats.getMin() + "\n"); for (double pctile : percentiles) { sb.append("\t" + pctile + " - " + stats.getPercentile(pctile) + "\n"); }/*ww w . j a v a 2s . c o m*/ sb.append("\tMax - " + (long) stats.getMax()); return sb.toString(); }