List of usage examples for org.apache.commons.math3.stat.descriptive DescriptiveStatistics getMin
public double getMin()
From source file:knop.psfj.FovDataSet.java
/** * Gets the min and max./* w ww .j ava2 s. co m*/ * * @param yColumn the y column * @return the min and max */ public double[] getMinAndMax(String yColumn) { double[] minAndMax = new double[3]; DescriptiveStatistics stats = getColumnStatistics(yColumn); double line; double min; double max; if (yColumn.equals("z_profile")) { minAndMax[0] = -1; minAndMax[1] = 1; minAndMax[2] = 0; return minAndMax; } // scale to the theoritical value if (scalingMode == SCALE_TO_THEORITICAL) { // there is a theoretical value if (getTheoriticalValue(yColumn) != null) { line = getTheoriticalValue(yColumn); max = line * 2; min = line / 2; } // no theoritical value but the min and max are fixed else if (getMetaDataValueAsDouble(yColumn + "_interval_min") != null) { min = getMetaDataValueAsDouble(yColumn + "_interval_min"); max = getMetaDataValueAsDouble(yColumn + "_interval_max"); line = -1; } // no theoritical value, then we can only scale to min and max else { min = stats.getMin(); max = stats.getMax(); line = -1; } } else { min = stats.getMin(); max = stats.getMax(); if (getTheoriticalValue(yColumn) != null) { line = getTheoriticalValue(yColumn); } else { line = -1; } } minAndMax[0] = min; minAndMax[1] = max; minAndMax[2] = line; return minAndMax; }
From source file:gdsc.smlm.ij.plugins.pcpalm.PCPALMMolecules.java
/** * Calculate the average precision by fitting a skewed Gaussian to the histogram of the precision distribution. * <p>//www .j a v a 2 s .co m * A simple mean and SD of the histogram is computed. If the mean of the Skewed Gaussian does not fit within 3 SDs * of the simple mean then the simple mean is returned. * * @param molecules * @param title * the plot title (null if no plot should be displayed) * @param histogramBins * @param logFitParameters * Record the fit parameters to the ImageJ log * @param removeOutliers * The distribution is created using all values within 1.5x the inter-quartile range (IQR) of the data * @return The average precision */ public double calculateAveragePrecision(ArrayList<Molecule> molecules, String title, int histogramBins, boolean logFitParameters, boolean removeOutliers) { // Plot histogram of the precision float[] data = new float[molecules.size()]; DescriptiveStatistics stats = new DescriptiveStatistics(); double yMin = Double.NEGATIVE_INFINITY, yMax = 0; for (int i = 0; i < data.length; i++) { data[i] = (float) molecules.get(i).precision; stats.addValue(data[i]); } // Set the min and max y-values using 1.5 x IQR if (removeOutliers) { double lower = stats.getPercentile(25); double upper = stats.getPercentile(75); if (Double.isNaN(lower) || Double.isNaN(upper)) { if (logFitParameters) Utils.log("Error computing IQR: %f - %f", lower, upper); } else { double iqr = upper - lower; yMin = FastMath.max(lower - iqr, stats.getMin()); yMax = FastMath.min(upper + iqr, stats.getMax()); if (logFitParameters) Utils.log(" Data range: %f - %f. Plotting 1.5x IQR: %f - %f", stats.getMin(), stats.getMax(), yMin, yMax); } } if (yMin == Double.NEGATIVE_INFINITY) { yMin = stats.getMin(); yMax = stats.getMax(); if (logFitParameters) Utils.log(" Data range: %f - %f", yMin, yMax); } float[][] hist = Utils.calcHistogram(data, yMin, yMax, histogramBins); Plot2 plot = null; if (title != null) { plot = new Plot2(title, "Precision", "Frequency"); float[] xValues = hist[0]; float[] yValues = hist[1]; if (xValues.length > 0) { double xPadding = 0.05 * (xValues[xValues.length - 1] - xValues[0]); plot.setLimits(xValues[0] - xPadding, xValues[xValues.length - 1] + xPadding, 0, Maths.max(yValues) * 1.05); } plot.addPoints(xValues, yValues, Plot2.BAR); Utils.display(title, plot); } // Extract non-zero data float[] x = Arrays.copyOf(hist[0], hist[0].length); float[] y = hist[1]; int count = 0; float dx = (x[1] - x[0]) * 0.5f; for (int i = 0; i < y.length; i++) if (y[i] > 0) { x[count] = x[i] + dx; y[count] = y[i]; count++; } x = Arrays.copyOf(x, count); y = Arrays.copyOf(y, count); // Sense check to fitted data. Get mean and SD of histogram double[] stats2 = Utils.getHistogramStatistics(x, y); double mean = stats2[0]; if (logFitParameters) log(" Initial Statistics: %f +/- %f", stats2[0], stats2[1]); // Standard Gaussian fit double[] parameters = fitGaussian(x, y); if (parameters == null) { log(" Failed to fit initial Gaussian"); return mean; } double newMean = parameters[1]; double error = Math.abs(stats2[0] - newMean) / stats2[1]; if (error > 3) { log(" Failed to fit Gaussian: %f standard deviations from histogram mean", error); return mean; } if (newMean < yMin || newMean > yMax) { log(" Failed to fit Gaussian: %f outside data range %f - %f", newMean, yMin, yMax); return mean; } mean = newMean; if (logFitParameters) log(" Initial Gaussian: %f @ %f +/- %f", parameters[0], parameters[1], parameters[2]); double[] initialSolution = new double[] { parameters[0], parameters[1], parameters[2], -1 }; // Fit to a skewed Gaussian (or appropriate function) double[] skewParameters = fitSkewGaussian(x, y, initialSolution); if (skewParameters == null) { log(" Failed to fit Skewed Gaussian"); return mean; } SkewNormalFunction sn = new SkewNormalFunction(skewParameters); if (logFitParameters) log(" Skewed Gaussian: %f @ %f +/- %f (a = %f) => %f +/- %f", skewParameters[0], skewParameters[1], skewParameters[2], skewParameters[3], sn.getMean(), Math.sqrt(sn.getVariance())); newMean = sn.getMean(); error = Math.abs(stats2[0] - newMean) / stats2[1]; if (error > 3) { log(" Failed to fit Skewed Gaussian: %f standard deviations from histogram mean", error); return mean; } if (newMean < yMin || newMean > yMax) { log(" Failed to fit Skewed Gaussian: %f outside data range %f - %f", newMean, yMin, yMax); return mean; } // Use original histogram x-axis to maintain all the bins if (plot != null) { x = hist[0]; for (int i = 0; i < y.length; i++) x[i] += dx; plot.setColor(Color.red); addToPlot(plot, x, skewParameters, Plot2.LINE); plot.setColor(Color.black); Utils.display(title, plot); } // Return the average precision from the fitted curve return newMean; }
From source file:com.mapd.bench.Benchmark.java
String executeQuery(String sql, int expected, int iterations, int queryNum) { Connection conn = null;//from w ww. jav a2 s . c o m Statement stmt = null; Long firstExecute = 0l; Long firstJdbc = 0l; Long firstIterate = 0l; DescriptiveStatistics statsExecute = new DescriptiveStatistics(); DescriptiveStatistics statsJdbc = new DescriptiveStatistics(); DescriptiveStatistics statsIterate = new DescriptiveStatistics(); DescriptiveStatistics statsTotal = new DescriptiveStatistics(); long totalTime = 0; try { //Open a connection logger.debug("Connecting to database url :" + url); conn = DriverManager.getConnection(url, iUser, iPasswd); long startTime = System.currentTimeMillis(); for (int loop = 0; loop < iterations; loop++) { //Execute a query stmt = conn.createStatement(); long timer = System.currentTimeMillis(); ResultSet rs = stmt.executeQuery(sql); long executeTime = 0; long jdbcTime = 0; // gather internal execute time for MapD as we are interested in that if (driver.equals(JDBC_DRIVER)) { executeTime = stmt.getQueryTimeout(); jdbcTime = (System.currentTimeMillis() - timer) - executeTime; } else { jdbcTime = (System.currentTimeMillis() - timer); executeTime = 0; } // this is fake to get our intenal execute time. logger.debug("Query Timeout/AKA internal Execution Time was " + stmt.getQueryTimeout() + " ms Elapsed time in JVM space was " + (System.currentTimeMillis() - timer) + "ms"); timer = System.currentTimeMillis(); //Extract data from result set int resultCount = 0; while (rs.next()) { Object obj = rs.getObject(1); if (obj != null && obj.equals(statsExecute)) { logger.info("Impossible"); } resultCount++; } long iterateTime = (System.currentTimeMillis() - timer); if (resultCount != expected) { logger.error("Expect " + expected + " actual " + resultCount + " for query " + sql); // don't run anymore break; } if (loop == 0) { firstJdbc = jdbcTime; firstExecute = executeTime; firstIterate = iterateTime; } else { statsJdbc.addValue(jdbcTime); statsExecute.addValue(executeTime); statsIterate.addValue(iterateTime); statsTotal.addValue(jdbcTime + executeTime + iterateTime); } //Clean-up environment rs.close(); stmt.close(); } totalTime = System.currentTimeMillis() - startTime; conn.close(); } catch (SQLException se) { //Handle errors for JDBC se.printStackTrace(); } catch (Exception e) { //Handle errors for Class.forName e.printStackTrace(); } finally { //finally block used to close resources try { if (stmt != null) { stmt.close(); } } catch (SQLException se2) { } // nothing we can do try { if (conn != null) { conn.close(); } } catch (SQLException se) { se.printStackTrace(); } //end finally try } //end try return String.format(lineDescriptor, queryNum, statsTotal.getMean(), statsTotal.getMin(), statsTotal.getMax(), statsTotal.getPercentile(85), statsExecute.getMean(), statsExecute.getMin(), statsExecute.getMax(), statsExecute.getPercentile(85), statsExecute.getPercentile(25), statsExecute.getStandardDeviation(), statsJdbc.getMean(), statsJdbc.getMin(), statsJdbc.getMax(), statsJdbc.getPercentile(85), statsIterate.getMean(), statsIterate.getMin(), statsIterate.getMax(), statsIterate.getPercentile(85), firstExecute, firstJdbc, firstIterate, iterations, totalTime, (long) statsTotal.getSum() + firstExecute + firstJdbc + firstIterate); }
From source file:edu.snu.leader.hierarchy.simple.DefaultReporter.java
/** * Report the final results of the simulation * * @see edu.snu.leader.hierarchy.simple.Reporter#reportFinalResults() *///from w w w.j a va 2 s . c o m @Override public void reportFinalResults() { // Create some handy variables long firstActiveTimestep = Long.MAX_VALUE; long lastActiveTimestep = Long.MIN_VALUE; int initiatorCount = 0; // Gather some statistics DescriptiveStatistics immediateFollowerStats = new DescriptiveStatistics(); DescriptiveStatistics initiatorDistanceStats = new DescriptiveStatistics(); DescriptiveStatistics activeTimestepStats = new DescriptiveStatistics(); // Iterate through all the individuals Iterator<Individual> indIter = _simState.getAllIndividuals().iterator(); while (indIter.hasNext()) { Individual ind = indIter.next(); // Get some statistics immediateFollowerStats.addValue(ind.getImmediateFollowerCount()); initiatorDistanceStats.addValue(ind.getDistanceToInitiator()); activeTimestepStats.addValue(ind.getActiveTimestep()); // Build the prefix String prefix = "individual." + ind.getID() + "."; // Log out important information _writer.println(prefix + "group-id = " + ind.getGroupID()); _writer.println(prefix + "active-timestep = " + ind.getActiveTimestep()); _writer.println(prefix + "immediate-follower-count = " + ind.getImmediateFollowerCount()); _writer.println(prefix + "total-follower-count = " + ind.getTotalFollowerCount()); _writer.println(prefix + "distance-to-initiator = " + ind.getDistanceToInitiator()); _writer.println(prefix + "location = " + ind.getLocation().getX() + " " + ind.getLocation().getY()); _writer.println(prefix + "threshold = " + ind.getThreshold()); _writer.println(prefix + "skill = " + ind.getSkill()); _writer.println(prefix + "confidence = " + ind.getConfidence()); _writer.println(prefix + "reputation = " + ind.getReputation()); _writer.println(prefix + "boldness = " + ind.getBoldness()); // Get the leader's ID, if it exists Object leaderID = ""; if (null != ind.getLeader()) { leaderID = ind.getLeader().getIndividual().getID(); } else { ++initiatorCount; } _writer.println(prefix + "leader = " + leaderID); // Build the list of neighbor ID's StringBuilder builder = new StringBuilder(); Iterator<Neighbor> neighborIter = ind.getNearestNeighbors().iterator(); while (neighborIter.hasNext()) { builder.append(neighborIter.next().getIndividual().getID()); builder.append(" "); } _writer.println(prefix + "nearest-neighbors = " + builder.toString()); // Build the list of follower ID's builder = new StringBuilder(); neighborIter = ind.getFollowers().iterator(); while (neighborIter.hasNext()) { builder.append(neighborIter.next().getIndividual().getID()); builder.append(" "); } _writer.println(prefix + "immediate-followers = " + builder.toString()); // Check the activity time if (firstActiveTimestep > ind.getActiveTimestep()) { firstActiveTimestep = ind.getActiveTimestep(); } if (lastActiveTimestep < ind.getActiveTimestep()) { lastActiveTimestep = ind.getActiveTimestep(); } _writer.println(); } // Log the simulation information _writer.println("simulation.first-active-timestep = " + firstActiveTimestep); _writer.println("simulation.last-active-timestep = " + lastActiveTimestep); _writer.println("simulation.initiator-count = " + initiatorCount); // Log the stats _writer.println("statistics.immediate-followers.mean = " + immediateFollowerStats.getMean()); _writer.println( "statistics.immediate-followers.std-dev = " + immediateFollowerStats.getStandardDeviation()); _writer.println("statistics.immediate-followers.min = " + immediateFollowerStats.getMin()); _writer.println("statistics.immediate-followers.max = " + immediateFollowerStats.getMax()); _writer.println("statistics.initiator-distance.mean = " + initiatorDistanceStats.getMean()); _writer.println("statistics.initiator-distance.std-dev = " + initiatorDistanceStats.getStandardDeviation()); _writer.println("statistics.initiator-distance.min = " + initiatorDistanceStats.getMin()); _writer.println("statistics.initiator-distance.max = " + initiatorDistanceStats.getMax()); _writer.println("statistics.active-timestep.mean = " + activeTimestepStats.getMean()); _writer.println("statistics.active-timestep.std-dev = " + activeTimestepStats.getStandardDeviation()); _writer.println("statistics.active-timestep.min = " + activeTimestepStats.getMin()); _writer.println("statistics.active-timestep.max = " + activeTimestepStats.getMax()); // Log out the stop time _writer.println(); _writer.println(_STATS_SPACER); _writer.println("# Finished: " + (new Date())); // Close out the writer _writer.close(); }
From source file:com.mapd.bench.BenchmarkCloud.java
String executeQuery(Connection conn1, String qid, String sql, int iterations) { Statement stmt = null;//w ww . j a va 2s . c o m Connection conn = getConnection(url, iUser, iPasswd); Long firstExecute = 0l; Long firstJdbc = 0l; Long firstIterate = 0l; DescriptiveStatistics statsExecute = new DescriptiveStatistics(); DescriptiveStatistics statsJdbc = new DescriptiveStatistics(); DescriptiveStatistics statsIterate = new DescriptiveStatistics(); DescriptiveStatistics statsTotal = new DescriptiveStatistics(); long totalTime = 0; int resultCount = 0; try { long startTime = System.currentTimeMillis(); for (int loop = 0; loop < iterations; loop++) { //Execute a query stmt = conn.createStatement(); long timer = System.currentTimeMillis(); if (loop == 0) { System.out.println(String.format("Query Id is %s : query is '%s'", qid, sql)); } ResultSet rs = stmt.executeQuery(sql); long executeTime = 0; long jdbcTime = 0; // gather internal execute time for MapD as we are interested in that if (driver.equals(JDBC_DRIVER)) { executeTime = stmt.getQueryTimeout(); jdbcTime = (System.currentTimeMillis() - timer) - executeTime; } else { jdbcTime = (System.currentTimeMillis() - timer); executeTime = 0; } // this is fake to get our intenal execute time. logger.debug("Query Timeout/AKA internal Execution Time was " + stmt.getQueryTimeout() + " ms Elapsed time in JVM space was " + (System.currentTimeMillis() - timer) + "ms"); timer = System.currentTimeMillis(); //Extract data from result set resultCount = 0; while (rs.next()) { Object obj = rs.getObject(1); if (obj != null && obj.equals(statsExecute)) { logger.info("Impossible"); } resultCount++; } long iterateTime = (System.currentTimeMillis() - timer); // if (resultCount != expected) { // logger.error("Expect " + expected + " actual " + resultCount + " for query " + sql); // // don't run anymore // break; // } if (loop == 0) { firstJdbc = jdbcTime; firstExecute = executeTime; firstIterate = iterateTime; } else { statsJdbc.addValue(jdbcTime); statsExecute.addValue(executeTime); statsIterate.addValue(iterateTime); statsTotal.addValue(jdbcTime + executeTime + iterateTime); } //Clean-up environment rs.close(); stmt.close(); } totalTime = System.currentTimeMillis() - startTime; conn.close(); } catch (SQLException se) { //Handle errors for JDBC se.printStackTrace(); System.exit(4); } catch (Exception e) { //Handle errors for Class.forName e.printStackTrace(); System.exit(3); } finally { //finally block used to close resources try { if (stmt != null) { stmt.close(); } } catch (SQLException se2) { } // nothing we can do try { if (conn != null) { conn.close(); } } catch (SQLException se) { se.printStackTrace(); System.exit(6); } //end finally try } //end try // write it to the db here as well String insertPart = String.format(insertDescriptor, this.rid, this.rTimestamp, url, this.driver, label, gpuCount, this.tableName, qid, resultCount, "", statsTotal.getMean(), statsTotal.getMin(), statsTotal.getMax(), statsTotal.getPercentile(85), statsExecute.getMean(), statsExecute.getMin(), statsExecute.getMax(), statsExecute.getPercentile(85), statsExecute.getPercentile(25), statsExecute.getStandardDeviation(), statsJdbc.getMean(), statsJdbc.getMin(), statsJdbc.getMax(), statsJdbc.getPercentile(85), statsIterate.getMean(), statsIterate.getMin(), statsIterate.getMax(), statsIterate.getPercentile(85), firstExecute, firstJdbc, firstIterate, iterations, totalTime, (long) statsTotal.getSum() + firstExecute + firstJdbc + firstIterate, targetDBVersion); LResult.add("Insert into results values " + insertPart); return String.format(lineDescriptor, qid, statsTotal.getMean(), statsTotal.getMin(), statsTotal.getMax(), statsTotal.getPercentile(85), statsExecute.getMean(), statsExecute.getMin(), statsExecute.getMax(), statsExecute.getPercentile(85), statsExecute.getPercentile(25), statsExecute.getStandardDeviation(), statsJdbc.getMean(), statsJdbc.getMin(), statsJdbc.getMax(), statsJdbc.getPercentile(85), statsIterate.getMean(), statsIterate.getMin(), statsIterate.getMax(), statsIterate.getPercentile(85), firstExecute, firstJdbc, firstIterate, iterations, totalTime, (long) statsTotal.getSum() + firstExecute + firstJdbc + firstIterate); }
From source file:org.alfresco.bm.api.v1.ResultsRestAPI.java
/** * Retrieve an approximate number of results, allowing for a smoothing factor * (<a href=http://en.wikipedia.org/wiki/Moving_average#Simple_moving_average>Simple Moving Average</a>) - * the number of data results to including in the moving average. * /*from w w w. jav a 2s . co m*/ * @param fromTime the approximate time to start from * @param timeUnit the units of the 'reportPeriod' (default SECONDS). See {@link TimeUnit}. * @param reportPeriod how often a result should be output. This is expressed as a multiple of the 'timeUnit'. * @param smoothing the number of results to include in the Simple Moving Average calculations * @param chartOnly <tt>true</tt> to filter out results that are not of interest in performance charts * * @return JSON representing the event start time (x-axis) and the smoothed average execution time * along with data such as the events per second, failures per second, etc. */ @GET @Path("/ts") @Produces(MediaType.APPLICATION_JSON) public String getTimeSeriesResults(@DefaultValue("0") @QueryParam("fromTime") long fromTime, @DefaultValue("SECONDS") @QueryParam("timeUnit") String timeUnit, @DefaultValue("1") @QueryParam("reportPeriod") long reportPeriod, @DefaultValue("1") @QueryParam("smoothing") int smoothing, @DefaultValue("true") @QueryParam("chartOnly") boolean chartOnly) { if (logger.isDebugEnabled()) { logger.debug("Inbound: " + "[test:" + test + ",fromTime:" + fromTime + ",timeUnit:" + timeUnit + ",reportPeriod:" + reportPeriod + ",smoothing:" + smoothing + ",chartOnly:" + chartOnly + "]"); } if (reportPeriod < 1) { throwAndLogException(Status.BAD_REQUEST, "'reportPeriod' must be 1 or more."); } if (smoothing < 1) { throwAndLogException(Status.BAD_REQUEST, "'smoothing' must be 1 or more."); } TimeUnit timeUnitEnum = null; try { timeUnitEnum = TimeUnit.valueOf(timeUnit.toUpperCase()); } catch (Exception e) { // Invalid time unit throwAndLogException(Status.BAD_REQUEST, e); } final ResultService resultService = getResultService(); // Calculate the window size long reportPeriodMs = timeUnitEnum.toMillis(reportPeriod); long windowSize = reportPeriodMs * smoothing; // This is just too convenient an API final BasicDBList events = new BasicDBList(); ResultHandler handler = new ResultHandler() { @Override public boolean processResult(long fromTime, long toTime, Map<String, DescriptiveStatistics> statsByEventName, Map<String, Integer> failuresByEventName) throws Throwable { for (Map.Entry<String, DescriptiveStatistics> entry : statsByEventName.entrySet()) { String eventName = entry.getKey(); DescriptiveStatistics stats = entry.getValue(); Integer failures = failuresByEventName.get(eventName); if (failures == null) { logger.error("Found null failure count: " + entry); // Do nothing with it and stop return false; } // Per second double numPerSec = (double) stats.getN() / ((double) (toTime - fromTime) / 1000.0); double failuresPerSec = (double) failures / ((double) (toTime - fromTime) / 1000.0); // Push into an object DBObject eventObj = BasicDBObjectBuilder.start().add("time", toTime).add("name", eventName) .add("mean", stats.getMean()).add("min", stats.getMin()).add("max", stats.getMax()) .add("stdDev", stats.getStandardDeviation()).add("num", stats.getN()) .add("numPerSec", numPerSec).add("fail", failures).add("failPerSec", failuresPerSec) .get(); // Add the object to the list of events events.add(eventObj); } // Go for the next result return true; } }; try { // Get all the results resultService.getResults(handler, fromTime, windowSize, reportPeriodMs, chartOnly); // Muster into JSON String json = events.toString(); // Done if (logger.isDebugEnabled()) { int jsonLen = json.length(); if (jsonLen < 500) { logger.debug("Outbound: " + json); } else { logger.debug("Outbound: " + json.substring(0, 250) + " ... " + json.substring(jsonLen - 250, jsonLen)); } } return json; } catch (WebApplicationException e) { throw e; } catch (Exception e) { throwAndLogException(Status.INTERNAL_SERVER_ERROR, e); return null; } }
From source file:org.alfresco.bm.report.XLSXReporter.java
private void createEventSheets(final XSSFWorkbook workbook) { // Create the fonts we need Font fontBold = workbook.createFont(); fontBold.setBoldweight(Font.BOLDWEIGHT_BOLD); // Create the styles we need CreationHelper helper = workbook.getCreationHelper(); final XSSFCellStyle dataStyle = workbook.createCellStyle(); dataStyle.setAlignment(HorizontalAlignment.RIGHT); final XSSFCellStyle headerStyle = workbook.createCellStyle(); headerStyle.setAlignment(HorizontalAlignment.RIGHT); headerStyle.setFont(fontBold);//from w w w . j a va 2 s . c o m final XSSFCellStyle dateStyle = workbook.createCellStyle(); dateStyle.setDataFormat(helper.createDataFormat().getFormat("HH:mm:ss")); // Calculate a good window size ResultService resultService = getResultService(); EventRecord firstResult = resultService.getFirstResult(); EventRecord lastResult = resultService.getLastResult(); if (firstResult == null || lastResult == null) { return; } long start = firstResult.getStartTime(); long end = lastResult.getStartTime(); long windowSize = AbstractEventReporter.getWindowSize(start, end, 100); // Well-known window sizes // Keep track of sheets by event name. Note that XLSX truncates sheets to 31 chars, so use 28 chars and ~01, ~02 final Map<String, String> sheetNames = new HashMap<String, String>(31); final Map<String, XSSFSheet> sheets = new HashMap<String, XSSFSheet>(31); final Map<String, AtomicInteger> rowNums = new HashMap<String, AtomicInteger>(31); ResultHandler handler = new ResultHandler() { @Override public boolean processResult(long fromTime, long toTime, Map<String, DescriptiveStatistics> statsByEventName, Map<String, Integer> failuresByEventName) throws Throwable { // Get or create a sheet for each event for (String eventName : statsByEventName.keySet()) { // What sheet name to we use? String sheetName = sheetNames.get(eventName); if (sheetName == null) { sheetName = eventName; if (eventName.length() > 28) { int counter = 1; // Find a sheet name not in use while (true) { sheetName = eventName.substring(0, 28); sheetName = String.format("%s~%02d", sheetName, counter); // Have we used this, yet? if (sheets.containsKey(sheetName)) { // Yes, we have used it. counter++; continue; } // This is unique break; } } sheetNames.put(eventName, sheetName); } // Get and create the sheet, if necessary XSSFSheet sheet = sheets.get(sheetName); if (sheet == null) { // Create try { sheet = workbook.createSheet(sheetName); sheets.put(sheetName, sheet); sheet.getHeader().setCenter(title + " - " + eventName); sheet.getPrintSetup().setFitWidth((short) 1); sheet.getPrintSetup().setLandscape(true); } catch (Exception e) { logger.error("Unable to create workbook sheet for event: " + eventName, e); continue; } // Intro XSSFCell cell = sheet.createRow(0).createCell(0); cell.setCellValue(title + " - " + eventName + ":"); cell.setCellStyle(headerStyle); // Headings XSSFRow row = sheet.createRow(1); cell = row.createCell(0); cell.setCellStyle(headerStyle); cell.setCellValue("time"); cell = row.createCell(1); cell.setCellStyle(headerStyle); cell.setCellValue("mean"); cell = row.createCell(2); cell.setCellStyle(headerStyle); cell.setCellValue("min"); cell = row.createCell(3); cell.setCellStyle(headerStyle); cell.setCellValue("max"); cell = row.createCell(4); cell.setCellStyle(headerStyle); cell.setCellValue("stdDev"); cell = row.createCell(5); cell.setCellStyle(headerStyle); cell.setCellValue("num"); cell = row.createCell(6); cell.setCellStyle(headerStyle); cell.setCellValue("numPerSec"); cell = row.createCell(7); cell.setCellStyle(headerStyle); cell.setCellValue("fail"); cell = row.createCell(8); cell.setCellStyle(headerStyle); cell.setCellValue("failPerSec"); // Size the columns sheet.autoSizeColumn(0); sheet.autoSizeColumn(1); sheet.autoSizeColumn(2); sheet.autoSizeColumn(3); sheet.autoSizeColumn(4); sheet.autoSizeColumn(5); sheet.autoSizeColumn(6); sheet.autoSizeColumn(7); sheet.autoSizeColumn(8); } AtomicInteger rowNum = rowNums.get(eventName); if (rowNum == null) { rowNum = new AtomicInteger(2); rowNums.put(eventName, rowNum); } DescriptiveStatistics stats = statsByEventName.get(eventName); Integer failures = failuresByEventName.get(eventName); double numPerSec = (double) stats.getN() / ((double) (toTime - fromTime) / 1000.0); double failuresPerSec = (double) failures / ((double) (toTime - fromTime) / 1000.0); XSSFRow row = sheet.createRow(rowNum.getAndIncrement()); XSSFCell cell; cell = row.createCell(0, Cell.CELL_TYPE_NUMERIC); cell.setCellStyle(dateStyle); cell.setCellValue(new Date(toTime)); cell = row.createCell(5, Cell.CELL_TYPE_NUMERIC); cell.setCellValue(stats.getN()); cell = row.createCell(6, Cell.CELL_TYPE_NUMERIC); cell.setCellValue(numPerSec); cell = row.createCell(7, Cell.CELL_TYPE_NUMERIC); cell.setCellValue(failures); cell = row.createCell(8, Cell.CELL_TYPE_NUMERIC); cell.setCellValue(failuresPerSec); // Leave out values if there is no mean if (Double.isNaN(stats.getMean())) { continue; } cell = row.createCell(1, Cell.CELL_TYPE_NUMERIC); cell.setCellValue(stats.getMean()); cell = row.createCell(2, Cell.CELL_TYPE_NUMERIC); cell.setCellValue(stats.getMin()); cell = row.createCell(3, Cell.CELL_TYPE_NUMERIC); cell.setCellValue(stats.getMax()); cell = row.createCell(4, Cell.CELL_TYPE_NUMERIC); cell.setCellValue(stats.getStandardDeviation()); } return true; } }; resultService.getResults(handler, start, windowSize, windowSize, false); // Create charts in the sheets for (String eventName : sheetNames.keySet()) { // Get the sheet name String sheetName = sheetNames.get(eventName); if (sheetName == null) { logger.error("Did not find sheet for event: " + eventName); continue; } // Get the sheet XSSFSheet sheet = sheets.get(sheetName); if (sheet == null) { logger.error("Did not find sheet for name: " + sheetName); continue; } // What row did we get up to AtomicInteger rowNum = rowNums.get(eventName); if (rowNum == null) { logger.error("Did not find row number for event: " + sheetName); continue; } // This axis is common to both charts ChartDataSource<Number> xTime = DataSources.fromNumericCellRange(sheet, new CellRangeAddress(1, rowNum.intValue() - 1, 0, 0)); // Graph of event times XSSFDrawing drawingTimes = sheet.createDrawingPatriarch(); ClientAnchor anchorTimes = drawingTimes.createAnchor(0, 0, 0, 0, 0, 5, 15, 25); Chart chartTimes = drawingTimes.createChart(anchorTimes); ChartLegend legendTimes = chartTimes.getOrCreateLegend(); legendTimes.setPosition(LegendPosition.BOTTOM); LineChartData chartDataTimes = chartTimes.getChartDataFactory().createLineChartData(); ChartAxis bottomAxisTimes = chartTimes.getChartAxisFactory().createCategoryAxis(AxisPosition.BOTTOM); bottomAxisTimes.setNumberFormat("#,##0;-#,##0"); ValueAxis leftAxisTimes = chartTimes.getChartAxisFactory().createValueAxis(AxisPosition.LEFT); // Mean ChartDataSource<Number> yMean = DataSources.fromNumericCellRange(sheet, new CellRangeAddress(1, rowNum.intValue() - 1, 1, 1)); LineChartSeries yMeanSerie = chartDataTimes.addSeries(xTime, yMean); yMeanSerie.setTitle(title + " - " + eventName + ": Mean (ms)"); // Std Dev ChartDataSource<Number> yStdDev = DataSources.fromNumericCellRange(sheet, new CellRangeAddress(1, rowNum.intValue() - 1, 4, 4)); LineChartSeries yStdDevSerie = chartDataTimes.addSeries(xTime, yStdDev); yStdDevSerie.setTitle(title + " - " + eventName + ": Standard Deviation (ms)"); // Plot event times chartTimes.plot(chartDataTimes, bottomAxisTimes, leftAxisTimes); // Graph of event volumes // Graph of event times XSSFDrawing drawingVolumes = sheet.createDrawingPatriarch(); ClientAnchor anchorVolumes = drawingVolumes.createAnchor(0, 0, 0, 0, 0, 25, 15, 35); Chart chartVolumes = drawingVolumes.createChart(anchorVolumes); ChartLegend legendVolumes = chartVolumes.getOrCreateLegend(); legendVolumes.setPosition(LegendPosition.BOTTOM); LineChartData chartDataVolumes = chartVolumes.getChartDataFactory().createLineChartData(); ChartAxis bottomAxisVolumes = chartVolumes.getChartAxisFactory() .createCategoryAxis(AxisPosition.BOTTOM); bottomAxisVolumes.setNumberFormat("#,##0;-#,##0"); ValueAxis leftAxisVolumes = chartVolumes.getChartAxisFactory().createValueAxis(AxisPosition.LEFT); // Number per second ChartDataSource<Number> yNumPerSec = DataSources.fromNumericCellRange(sheet, new CellRangeAddress(1, rowNum.intValue() - 1, 6, 6)); LineChartSeries yNumPerSecSerie = chartDataVolumes.addSeries(xTime, yNumPerSec); yNumPerSecSerie.setTitle(title + " - " + eventName + ": Events per Second"); // Failures per second ChartDataSource<Number> yFailPerSec = DataSources.fromNumericCellRange(sheet, new CellRangeAddress(1, rowNum.intValue() - 1, 8, 8)); LineChartSeries yFailPerSecSerie = chartDataVolumes.addSeries(xTime, yFailPerSec); yFailPerSecSerie.setTitle(title + " - " + eventName + ": Failures per Second"); // Plot volumes chartVolumes.plot(chartDataVolumes, bottomAxisVolumes, leftAxisVolumes); } }
From source file:org.apache.hadoop.hive.metastore.tools.BenchmarkSuite.java
/** * Produce printable result// w ww.j a v a2 s .c o m * @param fmt text formatter - destination of formatted results. * @param name benchmark name * @param stats benchmark data */ private void displayStats(@NotNull Formatter fmt, @NotNull String name, @NotNull DescriptiveStatistics stats) { double mean = stats.getMean(); double err = stats.getStandardDeviation() / mean * 100; long conv = scale.toNanos(1); fmt.format("%-30s %-8.4g %-8.4g %-8.4g %-8.4g %-8.4g%n", name, mean / conv, median(stats) / conv, stats.getMin() / conv, stats.getMax() / conv, err); }
From source file:org.apache.hadoop.hive.metastore.tools.BenchmarkSuite.java
/** * Produce results in printable CSV format, separated by separator. * @param fmt text formatter - destination of formatted results. * @param name benchmark name/*from w w w .j a va 2s.com*/ * @param stats benchmark data * @param separator field separator */ private void displayCSV(@NotNull Formatter fmt, @NotNull String name, @NotNull DescriptiveStatistics stats, @NotNull String separator) { double mean = stats.getMean(); double err = stats.getStandardDeviation() / mean * 100; long conv = scale.toNanos(1); fmt.format("%s%s%g%s%g%s%g%s%g%s%g%n", name, separator, mean / conv, separator, median(stats) / conv, separator, stats.getMin() / conv, separator, stats.getMax() / conv, separator, err); }
From source file:org.apache.metron.common.math.stats.OnlineStatisticsProviderTest.java
public static void validateStatisticsProvider(StatisticsProvider statsProvider, SummaryStatistics summaryStats, DescriptiveStatistics stats) { //N/*w ww .ja v a 2 s.c om*/ Assert.assertEquals(statsProvider.getCount(), stats.getN()); //sum Assert.assertEquals(statsProvider.getSum(), stats.getSum(), 1e-3); //sum of squares Assert.assertEquals(statsProvider.getSumSquares(), stats.getSumsq(), 1e-3); //sum of squares Assert.assertEquals(statsProvider.getSumLogs(), summaryStats.getSumOfLogs(), 1e-3); //Mean Assert.assertEquals(statsProvider.getMean(), stats.getMean(), 1e-3); //Quadratic Mean Assert.assertEquals(statsProvider.getQuadraticMean(), summaryStats.getQuadraticMean(), 1e-3); //SD Assert.assertEquals(statsProvider.getStandardDeviation(), stats.getStandardDeviation(), 1e-3); //Variance Assert.assertEquals(statsProvider.getVariance(), stats.getVariance(), 1e-3); //Min Assert.assertEquals(statsProvider.getMin(), stats.getMin(), 1e-3); //Max Assert.assertEquals(statsProvider.getMax(), stats.getMax(), 1e-3); //Kurtosis Assert.assertEquals(stats.getKurtosis(), statsProvider.getKurtosis(), 1e-3); //Skewness Assert.assertEquals(stats.getSkewness(), statsProvider.getSkewness(), 1e-3); for (double d = 10.0; d < 100.0; d += 10) { //This is a sketch, so we're a bit more forgiving here in our choice of \epsilon. Assert.assertEquals("Percentile mismatch for " + d + "th %ile", statsProvider.getPercentile(d), stats.getPercentile(d), 1e-2); } }