List of usage examples for org.apache.commons.math.stat.descriptive DescriptiveStatistics getMin
public double getMin()
From source file:graticules2wld.Main.java
/** * @param args/*from w w w.j a v a2s . com*/ * @throws Exception */ public static void main(String[] args) throws Exception { /* parse the command line arguments */ // create the command line parser CommandLineParser parser = new PosixParser(); // create the Options Options options = new Options(); options.addOption("x", "originx", true, "x component of projected coordinates of upper left pixel"); options.addOption("y", "originy", true, "y component of projected coordinates of upper left pixel"); options.addOption("u", "tometers", true, "multiplication factor to get source units into meters"); options.addOption("h", "help", false, "prints this usage page"); options.addOption("d", "debug", false, "prints debugging information to stdout"); double originNorthing = 0; double originEasting = 0; String inputFileName = null; String outputFileName = null; try { // parse the command line arguments CommandLine line = parser.parse(options, args); if (line.hasOption("help")) printUsage(0); // print usage then exit using a non error exit status if (line.hasOption("debug")) debug = true; // these arguments are required if (!line.hasOption("originy") || !line.hasOption("originx")) printUsage(1); originNorthing = Double.parseDouble(line.getOptionValue("originy")); originEasting = Double.parseDouble(line.getOptionValue("originx")); if (line.hasOption("tometers")) unitsToMeters = Double.parseDouble(line.getOptionValue("tometers")); // two args should be left. the input csv file name and the output wld file name. String[] iofiles = line.getArgs(); if (iofiles.length < 2) { printUsage(1); } inputFileName = iofiles[0]; outputFileName = iofiles[1]; } catch (ParseException exp) { System.err.println("Unexpected exception:" + exp.getMessage()); System.exit(1); } // try to open the input file for reading and the output file for writing File graticulesCsvFile; BufferedReader csvReader = null; File wldFile; BufferedWriter wldWriter = null; try { graticulesCsvFile = new File(inputFileName); csvReader = new BufferedReader(new FileReader(graticulesCsvFile)); } catch (IOException exp) { System.err.println("Could not open input file for reading: " + inputFileName); System.exit(1); } try { wldFile = new File(outputFileName); wldWriter = new BufferedWriter(new FileWriter(wldFile)); } catch (IOException exp) { System.err.println("Could not open output file for writing: " + outputFileName); System.exit(1); } // list of lon graticules and lat graticules ArrayList<Graticule> lonGrats = new ArrayList<Graticule>(); ArrayList<Graticule> latGrats = new ArrayList<Graticule>(); // read the source CSV and convert its information into the two ArrayList<Graticule> data structures readCSV(csvReader, lonGrats, latGrats); // we now need to start finding the world file paramaters DescriptiveStatistics stats = new DescriptiveStatistics(); // find theta and phi for (Graticule g : latGrats) { stats.addValue(g.angle()); } double theta = stats.getMean(); // we use the mean of the lat angles as theta if (debug) System.out.println("theta range = " + Math.toDegrees(stats.getMax() - stats.getMin())); stats.clear(); for (Graticule g : lonGrats) { stats.addValue(g.angle()); } double phi = stats.getMean(); // ... and the mean of the lon angles for phi if (debug) System.out.println("phi range = " + Math.toDegrees(stats.getMax() - stats.getMin())); stats.clear(); // print these if in debug mode if (debug) { System.out.println("theta = " + Math.toDegrees(theta) + "deg"); System.out.println("phi = " + Math.toDegrees(phi) + "deg"); } // find x and y (distance beteen pixels in map units) Collections.sort(latGrats); Collections.sort(lonGrats); int prevMapValue = 0; //fixme: how to stop warning about not being initilised? Line2D prevGratPixelSys = new Line2D.Double(); boolean first = true; for (Graticule g : latGrats) { if (!first) { int deltaMapValue = Math.abs(g.realValue() - prevMapValue); double deltaPixelValue = (g.l.ptLineDist(prevGratPixelSys.getP1()) + (g.l.ptLineDist(prevGratPixelSys.getP2()))) / 2; double delta = deltaMapValue / deltaPixelValue; stats.addValue(delta); } else { first = false; prevMapValue = g.realValue(); prevGratPixelSys = (Line2D) g.l.clone(); } } double y = stats.getMean(); if (debug) System.out.println("y range = " + (stats.getMax() - stats.getMin())); stats.clear(); first = true; for (Graticule g : lonGrats) { if (!first) { int deltaMapValue = g.realValue() - prevMapValue; double deltaPixelValue = (g.l.ptLineDist(prevGratPixelSys.getP1()) + (g.l.ptLineDist(prevGratPixelSys.getP2()))) / 2; double delta = deltaMapValue / deltaPixelValue; stats.addValue(delta); } else { first = false; prevMapValue = g.realValue(); prevGratPixelSys = (Line2D) g.l.clone(); } } double x = stats.getMean(); if (debug) System.out.println("x range = " + (stats.getMax() - stats.getMin())); stats.clear(); if (debug) { System.out.println("x = " + x); System.out.println("y = " + y); } SimpleRegression regression = new SimpleRegression(); // C, F are translation terms: x, y map coordinates of the center of the upper-left pixel for (Graticule g : latGrats) { // find perp dist to pixel space 0,0 Double perpPixelDist = g.l.ptLineDist(new Point2D.Double(0, 0)); // find the map space distance from this graticule to the center of the 0,0 pixel Double perpMapDist = perpPixelDist * y; // perpMapDist / perpPixelDist = y regression.addData(perpMapDist, g.realValue()); } double F = regression.getIntercept(); regression.clear(); for (Graticule g : lonGrats) { // find perp dist to pixel space 0,0 Double perpPixelDist = g.l.ptLineDist(new Point2D.Double(0, 0)); // find the map space distance from this graticule to the center of the 0,0 pixel Double perpMapDist = perpPixelDist * x; // perpMapDist / perpPixelDist = x regression.addData(perpMapDist, g.realValue()); } double C = regression.getIntercept(); regression.clear(); if (debug) { System.out.println("Upper Left pixel has coordinates " + C + ", " + F); } // convert to meters C *= unitsToMeters; F *= unitsToMeters; // C,F store the projected (in map units) coordinates of the upper left pixel. // originNorthing,originEasting is the offset we need to apply to 0,0 to push the offsets into our global coordinate system C = originEasting + C; F = originNorthing + F; // calculate the affine transformation matrix elements double D = -1 * x * unitsToMeters * Math.sin(theta); double A = x * unitsToMeters * Math.cos(theta); double B = y * unitsToMeters * Math.sin(phi); // if should be negative, it'll formed by negative sin double E = -1 * y * unitsToMeters * Math.cos(phi); /* * Line 1: A: pixel size in the x-direction in map units/pixel * Line 2: D: rotation about y-axis * Line 3: B: rotation about x-axis * Line 4: E: pixel size in the y-direction in map units, almost always negative[3] * Line 5: C: x-coordinate of the center of the upper left pixel * Line 6: F: y-coordinate of the center of the upper left pixel */ if (debug) { System.out.println("A = " + A); System.out.println("D = " + D); System.out.println("B = " + B); System.out.println("E = " + E); System.out.println("C = " + C); System.out.println("F = " + F); // write the world file System.out.println(); System.out.println("World File:"); System.out.println(A); System.out.println(D); System.out.println(B); System.out.println(E); System.out.println(C); System.out.println(F); } // write to the .wld file wldWriter.write(A + "\n"); wldWriter.write(D + "\n"); wldWriter.write(B + "\n"); wldWriter.write(E + "\n"); wldWriter.write(C + "\n"); wldWriter.write(F + "\n"); wldWriter.close(); }
From source file:com.mozilla.socorro.RawDumpSizeScan.java
public static void main(String[] args) throws ParseException { String startDateStr = args[0]; String endDateStr = args[1];/*from w ww . j ava2 s.c o m*/ // Set both start/end time and start/stop row Calendar startCal = Calendar.getInstance(); Calendar endCal = Calendar.getInstance(); SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd"); if (!StringUtils.isBlank(startDateStr)) { startCal.setTime(sdf.parse(startDateStr)); } if (!StringUtils.isBlank(endDateStr)) { endCal.setTime(sdf.parse(endDateStr)); } DescriptiveStatistics stats = new DescriptiveStatistics(); long numNullRawBytes = 0L; HTable table = null; Map<String, Integer> rowValueSizeMap = new HashMap<String, Integer>(); try { table = new HTable(TABLE_NAME_CRASH_REPORTS); Scan[] scans = generateScans(startCal, endCal); for (Scan s : scans) { ResultScanner rs = table.getScanner(s); Iterator<Result> iter = rs.iterator(); while (iter.hasNext()) { Result r = iter.next(); ImmutableBytesWritable rawBytes = r.getBytes(); //length = r.getValue(RAW_DATA_BYTES, DUMP_BYTES); if (rawBytes != null) { int length = rawBytes.getLength(); if (length > 20971520) { rowValueSizeMap.put(new String(r.getRow()), length); } stats.addValue(length); } else { numNullRawBytes++; } if (stats.getN() % 10000 == 0) { System.out.println("Processed " + stats.getN()); System.out.println(String.format("Min: %.02f Max: %.02f Mean: %.02f", stats.getMin(), stats.getMax(), stats.getMean())); System.out.println( String.format("1st Quartile: %.02f 2nd Quartile: %.02f 3rd Quartile: %.02f", stats.getPercentile(25.0d), stats.getPercentile(50.0d), stats.getPercentile(75.0d))); System.out.println("Number of large entries: " + rowValueSizeMap.size()); } } rs.close(); } System.out.println("Finished Processing!"); System.out.println(String.format("Min: %.02f Max: %.02f Mean: %.02f", stats.getMin(), stats.getMax(), stats.getMean())); System.out.println(String.format("1st Quartile: %.02f 2nd Quartile: %.02f 3rd Quartile: %.02f", stats.getPercentile(25.0d), stats.getPercentile(50.0d), stats.getPercentile(75.0d))); for (Map.Entry<String, Integer> entry : rowValueSizeMap.entrySet()) { System.out.println(String.format("RowId: %s => Length: %d", entry.getKey(), entry.getValue())); } } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } finally { if (table != null) { try { table.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } } }
From source file:com.graphhopper.jsprit.analysis.toolbox.ConcurrentBenchmarker.java
private String getString(DescriptiveStatistics stats) { return "[best=" + round(stats.getMin(), 2) + "][avg=" + round(stats.getMean(), 2) + "][worst=" + round(stats.getMax(), 2) + "][stdDev=" + round(stats.getStandardDeviation(), 2) + "]"; }
From source file:de.mpicbg.knime.hcs.base.utils.MutualInformation.java
private Double[] minmax(Double[] vect) { DescriptiveStatistics stats = new DescriptiveStatistics(); for (Double value : vect) { stats.addValue(value);/*ww w .j ava 2 s . c om*/ } return new Double[] { stats.getMin(), stats.getMax() }; }
From source file:com.google.caliper.runner.ConsoleResultProcessor.java
@Override public void processTrial(Trial trial) { ImmutableListMultimap<String, Measurement> measurementsIndex = new ImmutableListMultimap.Builder<String, Measurement>() .orderKeysBy(Ordering.natural()) .putAll(Multimaps.index(trial.measurements(), new Function<Measurement, String>() { @Override//from w w w .ja v a2 s. com public String apply(Measurement input) { return input.description(); } })).build(); for (Entry<String, Collection<Measurement>> entry : measurementsIndex.asMap().entrySet()) { Collection<Measurement> measurements = entry.getValue(); ImmutableSet<String> units = FluentIterable.from(measurements) .transform(new Function<Measurement, String>() { @Override public String apply(Measurement input) { return input.value().unit(); } }).toSet(); double[] weightedValues = new double[measurements.size()]; int i = 0; for (Measurement measurement : measurements) { weightedValues[i] = measurement.value().magnitude() / measurement.weight(); i++; } Percentile percentile = new Percentile(); percentile.setData(weightedValues); DescriptiveStatistics descriptiveStatistics = new DescriptiveStatistics(weightedValues); String unit = Iterables.getOnlyElement(units); stdout.printf(" %s%s: min=%.2f, 1st qu.=%.2f, median=%.2f, mean=%.2f, 3rd qu.=%.2f, max=%.2f%n", entry.getKey(), unit.isEmpty() ? "" : "(" + unit + ")", descriptiveStatistics.getMin(), percentile.evaluate(25), percentile.evaluate(50), descriptiveStatistics.getMean(), percentile.evaluate(75), descriptiveStatistics.getMax()); } instrumentSpecs.add(trial.instrumentSpec()); Scenario scenario = trial.scenario(); vmSpecs.add(scenario.vmSpec()); benchmarkSpecs.add(scenario.benchmarkSpec()); numMeasurements += trial.measurements().size(); }
From source file:dk.ilios.spanner.internal.ConsoleOutput.java
/** * Prints a summary of a successful trial result. */// ww w . j av a 2s. c om void processTrial(Trial.Result result) { Trial baseline = result.getExperiment().getBaseline(); trialsCompleted++; stdout.printf("Trial Report (%d of %d):%n Experiment %s%n", trialsCompleted, numberOfTrials, result.getExperiment()); if (!result.getTrialMessages().isEmpty()) { stdout.println(" Messages:"); for (String message : result.getTrialMessages()) { stdout.print(" "); stdout.println(message); } } Trial trial = result.getTrial(); // Group measurements by their description // TODO Why? All measurements for a single trial should have the same description ImmutableListMultimap<String, Measurement> measurementsIndex = new ImmutableListMultimap.Builder<String, Measurement>() .orderKeysBy(Ordering.natural()) .putAll(Multimaps.index(trial.measurements(), new Function<Measurement, String>() { @Override public String apply(Measurement input) { return input.description(); } })).build(); stdout.println(" Results:"); for (Map.Entry<String, Collection<Measurement>> entry : measurementsIndex.asMap().entrySet()) { Collection<Measurement> measurements = entry.getValue(); String unit = measurements.iterator().next().value().unit(); double[] weightedValues = new double[measurements.size()]; int i = 0; for (Measurement measurement : measurements) { weightedValues[i] = measurement.value().magnitude() / measurement.weight(); i++; } Percentile percentile = new Percentile(); percentile.setData(weightedValues); DescriptiveStatistics descriptiveStatistics = new DescriptiveStatistics(weightedValues); stdout.printf(" %s%s: min=%.2f, 1st qu.=%.2f, median=%.2f (%s), mean=%.2f, 3rd qu.=%.2f, max=%.2f%n", entry.getKey(), unit.isEmpty() ? "" : "(" + unit + ")", descriptiveStatistics.getMin(), percentile.evaluate(25), percentile.evaluate(50), calculateDiff(percentile.evaluate(50), baseline), descriptiveStatistics.getMean(), percentile.evaluate(75), descriptiveStatistics.getMax()); } instrumentSpecs.add(trial.instrumentSpec()); Scenario scenario = trial.scenario(); benchmarkSpecs.add(scenario.benchmarkSpec()); numMeasurements += trial.measurements().size(); }
From source file:com.google.caliper.runner.ConsoleOutput.java
/** * Prints a summary of a successful trial result. *///from ww w . jav a 2s .c o m void processTrial(TrialResult result) { trialsCompleted++; stdout.printf("Trial Report (%d of %d):%n Experiment %s%n", trialsCompleted, numberOfTrials, result.getExperiment()); if (!result.getTrialMessages().isEmpty()) { stdout.println(" Messages:"); for (String message : result.getTrialMessages()) { stdout.print(" "); stdout.println(message); } } Trial trial = result.getTrial(); ImmutableListMultimap<String, Measurement> measurementsIndex = new ImmutableListMultimap.Builder<String, Measurement>() .orderKeysBy(Ordering.natural()) .putAll(Multimaps.index(trial.measurements(), new Function<Measurement, String>() { @Override public String apply(Measurement input) { return input.description(); } })).build(); stdout.println(" Results:"); for (Entry<String, Collection<Measurement>> entry : measurementsIndex.asMap().entrySet()) { Collection<Measurement> measurements = entry.getValue(); ImmutableSet<String> units = FluentIterable.from(measurements) .transform(new Function<Measurement, String>() { @Override public String apply(Measurement input) { return input.value().unit(); } }).toSet(); double[] weightedValues = new double[measurements.size()]; int i = 0; for (Measurement measurement : measurements) { weightedValues[i] = measurement.value().magnitude() / measurement.weight(); i++; } Percentile percentile = new Percentile(); percentile.setData(weightedValues); DescriptiveStatistics descriptiveStatistics = new DescriptiveStatistics(weightedValues); String unit = Iterables.getOnlyElement(units); stdout.printf(" %s%s: min=%.2f, 1st qu.=%.2f, median=%.2f, mean=%.2f, 3rd qu.=%.2f, max=%.2f%n", entry.getKey(), unit.isEmpty() ? "" : "(" + unit + ")", descriptiveStatistics.getMin(), percentile.evaluate(25), percentile.evaluate(50), descriptiveStatistics.getMean(), percentile.evaluate(75), descriptiveStatistics.getMax()); } instrumentSpecs.add(trial.instrumentSpec()); Scenario scenario = trial.scenario(); vmSpecs.add(scenario.vmSpec()); benchmarkSpecs.add(scenario.benchmarkSpec()); numMeasurements += trial.measurements().size(); }
From source file:de.mpicbg.knime.hcs.base.utils.MutualInformation.java
private Double[] minmax(Double[] vect1, Double[] vect2) { DescriptiveStatistics stats = new DescriptiveStatistics(); for (Double value : vect1) { stats.addValue(value);//from w w w . j a v a 2 s .c o m } for (Double value : vect2) { stats.addValue(value); } return new Double[] { stats.getMin(), stats.getMax() }; }
From source file:de.mpicbg.knime.hcs.base.nodes.preproc.ParameterMutualInformation.java
@Override protected BufferedDataTable[] execute(BufferedDataTable[] inData, ExecutionContext exec) throws Exception { BufferedDataTable input = inData[0]; // Get the parameter and make sure there all double value columns List<Attribute> parameters = getParameterList(input); // Initialize int Np = parameters.size(); int iterations = Np * 2; double[][] mutmatrix = new double[Np][Np]; double[][] sigmatrix = new double[Np][Np]; ;//from w ww . jav a 2 s . c o m double[][] biamatrix = new double[Np][Np]; ; MutualInformation mutualinfo = new MutualInformation(); mutualinfo.set_base(logbase.getDoubleValue()); mutualinfo.set_method(method.getStringValue()); mutualinfo.set_axeslinking(linkaxes.getBooleanValue()); if (binning.getIntValue() > 0) mutualinfo.set_binning(binning.getIntValue()); // Load data. Double[][] table = new Double[Np][input.getRowCount()]; int j = 0; for (DataRow row : input) { int i = 0; for (Attribute param : parameters) { table[i][j] = param.getDoubleAttribute(row); i++; } j++; exec.checkCanceled(); } BufTableUtils.updateProgress(exec, Np, iterations); // Calculate mutual information for (int a = 0; a < Np; a++) { mutualinfo.set_xvector(table[a]); for (int b = a; b < Np; b++) { mutualinfo.set_yvector(table[b]); // Calculate the mutual info. Double[] res = mutualinfo.calculate(); // Put it into the output matrix mutmatrix[a][b] = res[0]; mutmatrix[b][a] = res[0]; sigmatrix[a][b] = res[1]; sigmatrix[b][a] = res[1]; biamatrix[a][b] = res[2]; biamatrix[b][a] = res[2]; } BufTableUtils.updateProgress(exec, Np + a, iterations); exec.checkCanceled(); } // Create Tables BufferedDataContainer matrixContainer = exec.createDataContainer(new DataTableSpec(getMatrixSpec())); BufferedDataContainer listContainer = exec.createDataContainer(new DataTableSpec(getListSpec())); double thresh = threshold.getDoubleValue(); int numListCol = listContainer.getTableSpec().getNumColumns(); for (int a = 0; a < Np; a++) { // Initialize DataCell[] matrixCells = new DataCell[Np]; DataCell[] listCells = new DataCell[numListCol]; String similars = ""; DescriptiveStatistics mutstats = new DescriptiveStatistics(); DescriptiveStatistics sigstats = new DescriptiveStatistics(); ; DescriptiveStatistics biastats = new DescriptiveStatistics(); ; // Create matrix rows and collect values for statistics. for (int b = 0; b < Np; b++) { matrixCells[b] = new DoubleCell(mutmatrix[a][b]); if (a != b) { mutstats.addValue(mutmatrix[a][b]); sigstats.addValue(sigmatrix[a][b]); biastats.addValue(biamatrix[a][b]); if (mutmatrix[a][b] > thresh) { similars += parameters.get(b).getName() + ","; } } } // Create matrix row DataRow matrixRow = new DefaultRow(parameters.get(a).getName(), matrixCells); matrixContainer.addRowToTable(matrixRow); // Create list row listCells[0] = new StringCell(parameters.get(a).getName()); listCells[1] = new DoubleCell(mutstats.getMin()); listCells[2] = new DoubleCell(mutstats.getMean()); listCells[3] = new DoubleCell(mutstats.getMax()); listCells[4] = new DoubleCell(sigstats.getGeometricMean()); listCells[5] = new DoubleCell(biastats.getMean()); listCells[6] = new StringCell(similars); DataRow listRow = new DefaultRow("row" + a, listCells); listContainer.addRowToTable(listRow); exec.checkCanceled(); } matrixContainer.close(); listContainer.close(); return new BufferedDataTable[] { listContainer.getTable(), matrixContainer.getTable() }; }
From source file:com.mozilla.socorro.hadoop.RawDumpSize.java
public int run(String[] args) throws Exception { if (args.length != 1) { return printUsage(); }/*from w ww.j a v a2 s . c o m*/ int rc = -1; Job job = initJob(args); job.waitForCompletion(true); if (job.isSuccessful()) { rc = 0; FileSystem hdfs = null; DescriptiveStatistics rawStats = new DescriptiveStatistics(); long rawTotal = 0L; DescriptiveStatistics processedStats = new DescriptiveStatistics(); long processedTotal = 0L; try { hdfs = FileSystem.get(job.getConfiguration()); Pattern tabPattern = Pattern.compile("\t"); for (FileStatus status : hdfs.listStatus(FileOutputFormat.getOutputPath(job))) { if (!status.isDir()) { BufferedReader reader = null; try { reader = new BufferedReader(new InputStreamReader(hdfs.open(status.getPath()))); String line = null; while ((line = reader.readLine()) != null) { String[] splits = tabPattern.split(line); int byteSize = Integer.parseInt(splits[2]); if ("raw".equals(splits[1])) { rawStats.addValue(byteSize); rawTotal += byteSize; } else if ("processed".equals(splits[1])) { processedStats.addValue(byteSize); processedTotal += byteSize; } } } finally { if (reader != null) { reader.close(); } } } } } finally { if (hdfs != null) { hdfs.close(); } } System.out.println("===== " + job.getConfiguration().get(START_DATE) + " raw_data:dump ====="); System.out.println(String.format("Min: %.02f Max: %.02f Mean: %.02f", rawStats.getMin(), rawStats.getMax(), rawStats.getMean())); System.out.println(String.format("1st Quartile: %.02f 2nd Quartile: %.02f 3rd Quartile: %.02f", rawStats.getPercentile(25.0d), rawStats.getPercentile(50.0d), rawStats.getPercentile(75.0d))); System.out.println("Total Bytes: " + rawTotal); System.out.println("===== " + job.getConfiguration().get(START_DATE) + " processed_data:json ====="); System.out.println(String.format("Min: %.02f Max: %.02f Mean: %.02f", processedStats.getMin(), processedStats.getMax(), processedStats.getMean())); System.out.println(String.format("1st Quartile: %.02f 2nd Quartile: %.02f 3rd Quartile: %.02f", processedStats.getPercentile(25.0d), processedStats.getPercentile(50.0d), processedStats.getPercentile(75.0d))); System.out.println("Total Bytes: " + processedTotal); } return rc; }