List of usage examples for org.apache.commons.math.stat.descriptive DescriptiveStatistics DescriptiveStatistics
public DescriptiveStatistics()
From source file:playground.johannes.studies.coopsim.SweepMerge1D.java
/** * @param args/*from ww w . j a va 2s . co m*/ * @throws IOException */ public static void main(String[] args) throws IOException { File root = new File("/Volumes/cluster.math.tu-berlin.de/net/ils2/jillenberger/leisure/runs/run272/"); // String property = "score_join_culture"; String property = "d_trip_culture"; int valIdx = 1; // String parameterKey = "beta_join"; // String parameterKey = "performing"; String parameterKey = "alterProba_culture"; int dumps = 1; File analysis = new File(String.format("%1$s/analysis/", root.getAbsolutePath())); analysis.mkdirs(); TDoubleDoubleHashMap values = new TDoubleDoubleHashMap(); File tasks = new File(String.format("%1$s/tasks/", root.getAbsolutePath())); for (File file : tasks.listFiles()) { if (file.isDirectory()) { if (!file.getName().equals("analysis")) { File output = new File(String.format("%1$s/output/", file.getAbsolutePath())); String[] dirs = output.list(); if (dirs.length > 0) { Arrays.sort(dirs, new Comparator<String>() { @Override public int compare(String o1, String o2) { return Double.compare(Double.parseDouble(o1), Double.parseDouble(o2)); } }); /* * get parameter value */ Config config = new Config(); ConfigReader creader = new ConfigReader(config); creader.readFile(String.format("%1$s/config.xml", file.getAbsolutePath())); double paramValue = Double.parseDouble(config.findParam("socialnets", parameterKey)); // double paramValue = Double.parseDouble(config.findParam("planCalcScore", parameterKey)); int start = dirs.length - dumps; start = Math.max(0, start); if (dirs.length < dumps) { logger.warn(String.format("Less than %1$s samples.", dumps)); } DescriptiveStatistics stat = new DescriptiveStatistics(); for (int i = start; i < dirs.length; i++) { // for(int i = 12; i < 13; i++) { File statsFile = new File( String.format("%1$s/%2$s/statistics.txt", output.getAbsolutePath(), dirs[i])); if (statsFile.exists()) { /* * get property value */ BufferedReader reader = new BufferedReader(new FileReader(statsFile)); String line = reader.readLine(); while ((line = reader.readLine()) != null) { String[] tokens = line.split("\t"); String key = tokens[0]; double val = Double.parseDouble(tokens[valIdx]); if (key.equals(property)) { stat.addValue(val); break; } } } } values.put(paramValue, stat.getMean()); } else { logger.warn("No samples."); } } } } TXTWriter.writeMap(values, parameterKey, property, String.format("%1$s/%2$s.txt", analysis.getAbsolutePath(), property)); }
From source file:playground.johannes.studies.coopsim.SweepMerge2D.java
/** * @param args// w ww . j a v a 2s . com * @throws IOException */ public static void main(String[] args) throws IOException { File root = new File("/Volumes/cluster.math.tu-berlin.de/net/ils2/jillenberger/leisure/runs/run205/"); String property = "d_trip_home"; int valIdx = 1; String parameterKey1 = "traveling"; String parameterKey2 = "performing"; File analysis = new File(String.format("%1$s/analysis/", root.getAbsolutePath())); analysis.mkdirs(); KeyMatrix<Double> matrix = new KeyMatrix<Double>(); File tasks = new File(String.format("%1$s/tasks/", root.getAbsolutePath())); for (File file : tasks.listFiles()) { if (file.isDirectory()) { if (!file.getName().equals("analysis")) { File output = new File(String.format("%1$s/output/", file.getAbsolutePath())); String[] dirs = output.list(); if (dirs.length > 0) { Arrays.sort(dirs, new Comparator<String>() { @Override public int compare(String o1, String o2) { return Double.compare(Double.parseDouble(o1), Double.parseDouble(o2)); } }); /* * get parameter value */ Config config = new Config(); ConfigReader creader = new ConfigReader(config); creader.readFile(String.format("%1$s/config.xml", file.getAbsolutePath())); double paramValue1 = Double.parseDouble(config.findParam("planCalcScore", parameterKey1)); double paramValue2 = Double.parseDouble(config.findParam("planCalcScore", parameterKey2)); int start = dirs.length - 10; start = Math.max(0, start); if (dirs.length < 10) { logger.warn("Less than 10 samples."); } DescriptiveStatistics stat = new DescriptiveStatistics(); for (int i = start; i < dirs.length; i++) { File statsFile = new File( String.format("%1$s/%2$s/statistics.txt", output.getAbsolutePath(), dirs[i])); if (statsFile.exists()) { /* * get property value */ BufferedReader reader = new BufferedReader(new FileReader(statsFile)); String line = reader.readLine(); while ((line = reader.readLine()) != null) { String[] tokens = line.split("\t"); String key = tokens[0]; double val = Double.parseDouble(tokens[valIdx]); if (key.equals(property)) { stat.addValue(val); break; } } } } matrix.putValue(stat.getMean(), Math.abs(paramValue1), Math.abs(paramValue2)); } else { logger.warn("No samples."); } } } } matrix.write(String.format("%1$s/%2$s.matrix.txt", analysis.getAbsolutePath(), property)); }
From source file:playground.johannes.studies.coopsim.TimeSamplerTest.java
/** * @param args//from w ww . j a v a 2 s.c om * @throws FunctionEvaluationException * @throws IOException */ public static void main(String[] args) throws FunctionEvaluationException, IOException { Random random = new Random(); AdditiveDistribution arrTimePDF = new AdditiveDistribution(); arrTimePDF.addComponent(new GaussDistribution(2151, 43030, 267)); arrTimePDF.addComponent(new GaussDistribution(14227, 58036, 1382)); TimeSampler arrivalSampler = new TimeSampler(arrTimePDF, 86400, random); AdditiveDistribution arrDurPDF = new AdditiveDistribution(); arrDurPDF.addComponent(new GaussDistribution(7210, 41513, 133759479)); arrDurPDF.addComponent(new GaussDistribution(15660, 73033, 277912890)); // TimeSampler arrDurSampler = new TimeSampler(arrDurPDF, 86400, random); // // LogNormalDistribution durPDF = new LogNormalDistribution(0.6883, 8.4954, 522.9869); // TimeSampler durSampler = new TimeSampler(durPDF, 86400, random); DescriptiveStatistics durations = new DescriptiveStatistics(); DescriptiveStatistics arrivals = new DescriptiveStatistics(); ProgressLogger.init(10000, 1, 5); double sigma = 0.6883; for (int i = 0; i < 10000; i++) { int t_arr = arrivalSampler.nextSample(); int dur_mean = (int) arrDurPDF.value(t_arr); if (dur_mean > 0) { double mu = Math.log(dur_mean) - Math.pow(sigma, 2) / 2.0; TimeSampler sampler = new TimeSampler(new LogNormalDistribution(sigma, mu, 522), 86400, random); double dur = sampler.nextSample(); durations.addValue(dur); arrivals.addValue(t_arr); ProgressLogger.step(); } } TDoubleDoubleHashMap hist = Histogram.createHistogram(durations, FixedSampleSizeDiscretizer.create(durations.getValues(), 1, 30), true); Histogram.normalize(hist); TXTWriter.writeMap(hist, "t", "n", "/Users/jillenberger/Work/socialnets/locationChoice/output/durations.txt"); TDoubleDoubleHashMap correl = Correlations.mean(arrivals.getValues(), durations.getValues(), FixedSampleSizeDiscretizer.create(arrivals.getValues(), 1, 24)); TXTWriter.writeMap(correl, "arr", "dur", "/Users/jillenberger/Work/socialnets/locationChoice/output/dur_arr.txt"); }
From source file:playground.johannes.studies.ivt.DensityPlotBiTree.java
public static void main(String[] args) throws IOException, FactoryException { SocialSampledGraphProjection<SocialSparseGraph, SocialSparseVertex, SocialSparseEdge> graph = GraphReaderFacade .read("/Users/jillenberger/Work/socialnets/data/ivt2009/11-2011/graph/graph.graphml"); SocialSampledGraphProjectionBuilder<SocialSparseGraph, SocialSparseVertex, SocialSparseEdge> builder = new SocialSampledGraphProjectionBuilder<SocialSparseGraph, SocialSparseVertex, SocialSparseEdge>(); SpatialSparseGraph popData = new Population2SpatialGraph(CRSUtils.getCRS(21781)) .read("/Users/jillenberger/Work/socialnets/data/schweiz/complete/plans/plans.0.10.xml"); SimpleFeature feature = FeatureSHP//from w w w . j a v a2 s.c o m .readFeatures("/Users/jillenberger/Work/socialnets/data/schweiz/complete/zones/G1L08.shp") .iterator().next(); Geometry chBorder = (Geometry) feature.getDefaultGeometry(); chBorder.setSRID(21781); graph.getDelegate().transformToCRS(CRSUtils.getCRS(21781)); logger.info("Applying spatial filter..."); SpatialFilter filter = new SpatialFilter((GraphBuilder) builder, chBorder); graph = (SocialSampledGraphProjection<SocialSparseGraph, SocialSparseVertex, SocialSparseEdge>) filter .apply(graph); Set<Point> points = new HashSet<Point>(); for (SpatialVertex v : graph.getVertices()) { points.add(v.getPoint()); } GeometryFactory factory = new GeometryFactory(); Point zrh = factory.createPoint(new Coordinate(8.55, 47.36)); zrh = CRSUtils.transformPoint(zrh, CRS.findMathTransform(DefaultGeographicCRS.WGS84, CRSUtils.getCRS(21781))); // graph.getDelegate().transformToCRS(DefaultGeographicCRS.WGS84); // graph2.transformToCRS(DefaultGeographicCRS.WGS84); logger.info("Segmenting tiles..."); // BiTreeGrid<Double> sampleGrid = BiTreeGridBuilder.createEqualCountGrid(points, 200, 1000); Envelope env = PointUtils.envelope(points); SpatialGrid<Double> sampleGrid = new SpatialGrid<Double>(env.getMinX(), env.getMinY(), env.getMaxX(), env.getMaxY(), 5000); DistanceCalculator calc = new CartesianDistanceCalculator(); Discretizer disc = new LinearDiscretizer(1000.0); logger.info("Creating survey grid..."); for (Point p : points) { // Tile<Double> tile = sampleGrid.getTile(p.getCoordinate()); Double data = sampleGrid.getValue(p); double val = 0; // if(tile.data != null) // val = tile.data; if (data != null) val = data.doubleValue(); double d = disc.discretize(calc.distance(p, zrh)); d = Math.max(1, d); double proba = Math.pow(d, -1.4); // val += 1/proba; val++; // tile.data = new Double(val); sampleGrid.setValue(val, p); } logger.info("Creating population grid..."); // BiTreeGrid<Integer> popGrid = new BiTreeGrid<Integer>(sampleGrid); SpatialGrid<Integer> popGrid = new SpatialGrid<Integer>(sampleGrid); for (SpatialVertex v : popData.getVertices()) { // Tile<Integer> tile = popGrid.getTile(v.getPoint().getCoordinate()); Integer data = popGrid.getValue(v.getPoint()); // if (tile != null) { // int val = 0; // if (tile.data != null) // val = tile.data; // val++; // tile.data = new Integer(val); // } int val = 0; if (data != null) { val = data.intValue(); } val++; popGrid.setValue(val, v.getPoint()); } logger.info("Creating density grid..."); // BiTreeGrid<Double> densityGrid = new BiTreeGrid<Double>(sampleGrid); SpatialGrid<Double> densityGrid = new SpatialGrid<Double>(sampleGrid); // Set<Tile<Double>> tiles = densityGrid.tiles(); // for(Tile<Double> tile : tiles) { // Tile<Double> surveyTile = sampleGrid.getTile(tile.envelope.centre()); // Tile<Integer> popTile = popGrid.getTile(tile.envelope.centre()); // // if(surveyTile != null && popTile != null) { // if(surveyTile.data != null && popTile.data != null) // tile.data = surveyTile.data/(double)popTile.data; // else // tile.data = new Double(0); // } else // tile.data = new Double(0); // } for (int row = 0; row < densityGrid.getNumRows(); row++) { for (int col = 0; col < densityGrid.getNumCols(row); col++) { Integer inhabitants = popGrid.getValue(row, col); Double samples = sampleGrid.getValue(row, col); if (inhabitants != null && samples != null) { double density = samples / (double) inhabitants; densityGrid.setValue(row, col, density); } else { densityGrid.setValue(row, col, 0.0); } } } ZoneLayer<Double> layer = ZoneUtils.createGridLayer(5000, chBorder); layer.overwriteCRS(CRSUtils.getCRS(21781)); DescriptiveStatistics stats = new DescriptiveStatistics(); for (int row = 0; row < densityGrid.getNumRows(); row++) { for (int col = 0; col < densityGrid.getNumCols(row); col++) { Point p = makeCoordinate(densityGrid, row, col); p.setSRID(21781); Zone<Double> z = layer.getZone(p); if (z != null) { double val = densityGrid.getValue(row, col); if (val > 0) { stats.addValue(val); } z.setAttribute(val); } } } // double min = stats.getMin(); // double max = stats.getPercentile(80); // double bins = 20; // double width = (max-min)/bins; // Discretizer discretizer = new BoundedLinearDiscretizer(width, min, max); // for(Zone<Double> z : layer.getZones()) { // Double val = z.getAttribute(); // if(val != null && val > 0) { // val = discretizer.index(val-min); // z.setAttribute(val); // } // } ZoneLayerSHP.write(layer, "/Users/jillenberger/Work/socialnets/data/ivt2009/11-2011/graph/density.grid.shp"); // logger.info("Writing KML..."); // SpatialGridKMLWriter writer = new SpatialGridKMLWriter(); // writer.write(densityGrid, CRSUtils.getCRS(21781), "/Users/jillenberger/Work/socialnets/data/ivt2009/11-2011/graph/density.grid.kml"); // writer.write(densityGrid, "/Users/jillenberger/Work/socialnets/data/ivt2009/11-2011/graph/density.grid.kml"); // GeometryFactory factory = new GeometryFactory(); // Set<Geometry> geometries = new HashSet<Geometry>(); // TObjectDoubleHashMap values = new TObjectDoubleHashMap(); //// tiles = sampleGrid.tiles(); // for(Tile<Double> n : tiles) { // Coordinate[] coords = new Coordinate[5]; // coords[0] = new Coordinate(n.envelope.getMinX(), n.envelope.getMinY()); // coords[1] = new Coordinate(n.envelope.getMinX(), n.envelope.getMaxY()); // coords[2] = new Coordinate(n.envelope.getMaxX(), n.envelope.getMaxY()); // coords[3] = new Coordinate(n.envelope.getMaxX(), n.envelope.getMinY()); // coords[4] = new Coordinate(n.envelope.getMinX(), n.envelope.getMinY()); // LinearRing shell = factory.createLinearRing(coords); // shell.setSRID(21781); // geometries.add(shell); // values.put(shell, n.data); // } // // NumericAttributeColorizer colorizer = new NumericAttributeColorizer(values); // colorizer.setLogscale(true); // FeatureKMLWriter writer = new FeatureKMLWriter(); // writer.setColorizable(colorizer); // writer.write(geometries, "/Users/jillenberger/Work/socialnets/data/ivt2009/11-2011/graph/density.kml"); }
From source file:playground.johannes.studies.ivt.DiaryAnalyzer.java
/** * @param args/* w w w . java 2 s . c o m*/ * @throws IOException */ public static void main(String[] args) throws IOException { BufferedReader reader = new BufferedReader( new FileReader("/Users/jillenberger/Work/socialnets/data/ivt2009/05-2011/raw/Diary.csv")); String header = reader.readLine(); String[] colNames = header.split(";"); int typeIdx = getIndex(colNames, "\"Activity\""); int accHhIdx = getIndex(colNames, "\"Activity_Acc_HH\""); int accOtherIdx = getIndex(colNames, "\"Activity_Acc_Others\""); DescriptiveStatistics accHhStats = new DescriptiveStatistics(); DescriptiveStatistics accOtherStats = new DescriptiveStatistics(); String line; while ((line = reader.readLine()) != null) { String[] tokens = line.split(";"); String type = tokens[typeIdx]; if (type.equals("\"Freizeit (offen)\"")) { String accHh = tokens[accHhIdx]; String accOther = tokens[accOtherIdx]; if (!accHh.equals("NA")) { int n = Integer.parseInt(accHh); if (n > 0) accHhStats.addValue(n); } if (!accOther.equals("NA")) { int n = Integer.parseInt(accOther); if (n > 0) accOtherStats.addValue(n); } } } System.out.println("Houshold members:"); System.out.println(accHhStats.toString()); System.out.println("Others:"); System.out.println(accOtherStats.toString()); }
From source file:playground.johannes.studies.ivt.FracDimOpportunities.java
/** * @param args// w w w .j a v a2s.c om * @throws IOException */ public static void main(String[] args) throws IOException { String targetsFile = "/Users/jillenberger/Work/socialnets/data/schweiz/complete/plans/plans.0.001.xml"; String chborderFile = "/Users/jillenberger/Work/socialnets/data/schweiz/complete/zones/G1L08.shp"; String graphFile = "/Users/jillenberger/Work/socialnets/data/ivt2009/11-2011/graph/graph.graphml"; String outFile = "/Users/jillenberger/Work/phd/doc/tex/ch3/fig/data/fdim.txt"; SpatialSparseGraph targetGraph = new Population2SpatialGraph(CRSUtils.getCRS(21781)).read(targetsFile); List<Point> targetPoints = new ArrayList<Point>(targetGraph.getVertices().size()); for (SpatialVertex v : targetGraph.getVertices()) { targetPoints.add(v.getPoint()); } SimpleFeature feature = FeatureSHP.readFeatures(chborderFile).iterator().next(); Geometry geometry = (Geometry) feature.getDefaultGeometry(); geometry.setSRID(21781); SocialSampledGraphProjection<SocialSparseGraph, SocialSparseVertex, SocialSparseEdge> graph = GraphReaderFacade .read(graphFile); graph.getDelegate().transformToCRS(CRSUtils.getCRS(21781)); Set<? extends SpatialVertex> egos = SnowballPartitions.createSampledPartition(graph.getVertices()); List<Point> startPoints = new ArrayList<Point>(egos.size()); for (SpatialVertex v : egos) { if (v.getPoint() != null) { if (geometry.contains(v.getPoint())) startPoints.add(v.getPoint()); } } DescriptiveStatistics stats = new DescriptiveStatistics(); DistanceCalculator dCalc = new CartesianDistanceCalculator(); int N = (int) (startPoints.size() * targetPoints.size()); ProgressLogger.init(N, 1, 5); for (int i = 0; i < startPoints.size(); i++) { Point p1 = startPoints.get(i); for (int j = 0; j < targetPoints.size(); j++) { Point p2 = targetPoints.get(j); double d = dCalc.distance(p1, p2); if (d > 0) stats.addValue(d); ProgressLogger.step(); } } System.out.println("Creating histograms..."); TDoubleDoubleHashMap hist = Histogram.createHistogram(stats, FixedSampleSizeDiscretizer.create(stats.getValues(), 100, 500), true); Histogram.normalize(hist); TXTWriter.writeMap(hist, "d", "p", outFile); }
From source file:playground.johannes.studies.netgeneration.CreatePowerLaw.java
/** * @param args// w ww . j av a 2 s . c o m * @throws IOException */ public static void main(String[] args) throws IOException { DescriptiveStatistics stats = new DescriptiveStatistics(); TDoubleDoubleHashMap hist = new TDoubleDoubleHashMap(); for (int i = 1; i < 1000; i++) // stats.addValue(Math.pow(i, -2)); hist.put(i, Math.pow(i, -2)); // Discretizer d = FixedSampleSizeDiscretizer.create(stats.getValues(), 10); // TDoubleDoubleHashMap hist = Histogram.createHistogram(stats, d, true); TXTWriter.writeMap(hist, "x", "p", "/Users/jillenberger/Desktop/powerlaw/distr.txt"); hist = Histogram.createCumulativeHistogram(hist); TXTWriter.writeMap(hist, "x", "p", "/Users/jillenberger/Desktop/powerlaw/distr.cum.txt"); }
From source file:pro.foundev.strategies.BenchmarkStrategy.java
private void exec(Runnable runnable, String name, int runs, BenchmarkReport report) { logger.info("Starting run of " + name); DescriptiveStatistics stats = new DescriptiveStatistics(); Stopwatch timer = new Stopwatch(); for (int i = 0; i < runs; i++) { timer.start();/*from w w w . j a v a2 s . co m*/ runnable.run(); timer.stop(); logger.info("Time to execute load run #" + i + " it took " + timer); stats.addValue(timer.elapsed(TimeUnit.MILLISECONDS)); timer.reset(); } logger.info("Finished run of " + name); report.addLine(name, stats.getMin(), stats.getMax(), stats.getPercentile(50), stats.getPercentile(90), stats.getMean()); }
From source file:ProcessData.GaussDistData.java
public GaussDistData() { mean = 0; sigma = 0; stats = new DescriptiveStatistics(); }
From source file:profiler.Result.java
public Result(Task task, double memory, double time) { this.task = task; this.memory = new DescriptiveStatistics(); this.memory.addValue(memory); this.time = new DescriptiveStatistics(); this.time.addValue(time); }