List of usage examples for org.apache.commons.math.stat.descriptive DescriptiveStatistics addValue
public void addValue(double v)
From source file:playground.johannes.gsv.matrices.analysis.CalcShares.java
/** * @param args/* www .java 2s . c o m*/ * @throws IOException */ public static void main(String[] args) throws IOException { KeyMatrixXMLReader reader = new KeyMatrixXMLReader(); reader.setValidating(false); reader.parse("/home/johannes/gsv/matrices/refmatrices/itp.xml"); KeyMatrix carItpMatrix = reader.getMatrix(); MatrixOperations.applyFactor(carItpMatrix, 1 / 365.0); reader.parse("/home/johannes/gsv/matrices/simmatrices/miv.819.2.xml"); KeyMatrix carSimMatrix = reader.getMatrix(); // MatrixOperations.symetrize(carSimMatrix); // MatrixOperations.applyFactor(carSimMatrix, 11.8); reader.parse("/home/johannes/gsv/matrices/refmatrices/tomtom.de.xml"); KeyMatrix carTomTomMatrix = reader.getMatrix(); reader.parse("/home/johannes/gsv/matrices/analysis/marketShares/rail.all.nuts3.xml"); KeyMatrix railSimMatrix = reader.getMatrix(); MatrixOperations.applyFactor(railSimMatrix, 1 / 365.0); reader.parse("/home/johannes/gsv/matrices/analysis/marketShares/car.share.xml"); KeyMatrix shareRefMatrix = reader.getMatrix(); ZoneCollection zones = new ZoneCollection(); String data = new String(Files.readAllBytes(Paths.get("/home/johannes/gsv/gis/nuts/de.nuts3.gk3.geojson"))); zones.addAll(Zone2GeoJSON.parseFeatureCollection(data)); data = null; zones.setPrimaryKey("gsvId"); ODUtils.cleanDistances(railSimMatrix, zones, 100000); ODUtils.cleanDistances(carSimMatrix, zones, 100000); ODUtils.cleanDistances(carTomTomMatrix, zones, 100000); // ODUtils.cleanVolumes(carTomTomMatrix, zones, 100); double c = ODUtils.calcNormalization(carTomTomMatrix, carSimMatrix); MatrixOperations.applyFactor(carTomTomMatrix, c); removeUnknownZones(railSimMatrix); List<Tuple<String, String>> relations = getRelations(railSimMatrix, 3000); // List<Tuple<String, String>> relations = getRelations(carTomTomMatrix, 3000); BufferedWriter writer = new BufferedWriter( new FileWriter("/home/johannes/gsv/matrices/analysis/marketShares/miv-shares.txt")); writer.write("from\tto\tshareRef\tshareSim\tcarVol\trailVol\tshareItp\tvolItp\tcarTomTom\tshareTomTom"); writer.newLine(); DescriptiveStatistics simStats = new DescriptiveStatistics(); DescriptiveStatistics itpStats = new DescriptiveStatistics(); DescriptiveStatistics simItpStats = new DescriptiveStatistics(); TDoubleArrayList railVolumes = new TDoubleArrayList(); TDoubleArrayList shareSimDiff = new TDoubleArrayList(); TDoubleArrayList shareItpDiff = new TDoubleArrayList(); TDoubleArrayList refShares = new TDoubleArrayList(); TDoubleArrayList simShares = new TDoubleArrayList(); TDoubleArrayList itpShares = new TDoubleArrayList(); TDoubleArrayList tomtomShares = new TDoubleArrayList(); for (Tuple<String, String> relation : relations) { String i = relation.getFirst(); String j = relation.getSecond(); Double carSimVal = carSimMatrix.get(i, j); if (carSimVal == null) carSimVal = 0.0; Double carItpVal = carItpMatrix.get(i, j); if (carItpVal == null) carItpVal = 0.0; Double carTomTomVal = carTomTomMatrix.get(i, j); if (carTomTomVal == null) carTomTomVal = 0.0; Double railVal = railSimMatrix.get(i, j); if (railVal == null) railVal = 0.0; Double shareVal = shareRefMatrix.get(i, j); if (shareVal == null) shareVal = 0.0; double simShare = carSimVal / (carSimVal + railVal); double itpShare = carItpVal / (carItpVal + railVal); double tomtomShare = carTomTomVal / (carTomTomVal + railVal); simStats.addValue(simShare - shareVal); itpStats.addValue(itpShare - shareVal); simItpStats.addValue(itpShare - simShare); refShares.add(shareVal); simShares.add(simShare); itpShares.add(itpShare); tomtomShares.add(tomtomShare); railVolumes.add(railVal); shareSimDiff.add(shareVal - simShare); shareItpDiff.add(shareVal - itpShare); writer.write(i); writer.write("\t"); writer.write(j); writer.write("\t"); writer.write(String.valueOf(shareVal)); writer.write("\t"); writer.write(String.valueOf(simShare)); writer.write("\t"); writer.write(String.valueOf(carSimVal)); writer.write("\t"); writer.write(String.valueOf(railVal)); writer.write("\t"); writer.write(String.valueOf(itpShare)); writer.write("\t"); writer.write(String.valueOf(carItpVal)); writer.write("\t"); writer.write(String.valueOf(carTomTomVal)); writer.write("\t"); writer.write(String.valueOf(tomtomShare)); writer.newLine(); } writer.close(); System.out.println("Avr sim share error: " + simStats.getMean()); System.out.println("Avr itp share error: " + itpStats.getMean()); System.out.println("Avr sim-itp share error: " + simItpStats.getMean()); writeCorrelation(refShares, simShares, "model", "sim", "/home/johannes/gsv/matrices/analysis/marketShares/modelSim.txt"); writeCorrelation(itpShares, simShares, "itp", "sim", "/home/johannes/gsv/matrices/analysis/marketShares/itpSim.txt"); writeCorrelation(tomtomShares, simShares, "tomtom", "sim", "/home/johannes/gsv/matrices/analysis/marketShares/tomtomSim.txt"); writeCorrelation(itpShares, tomtomShares, "itp", "tomtom", "/home/johannes/gsv/matrices/analysis/marketShares/itpTomtom.txt"); // double[] samples = railVolumes.toNativeArray(); // TDoubleDoubleHashMap hist = Correlations.mean(samples, shareSimDiff.toNativeArray(), FixedSampleSizeDiscretizer.create(samples, 50)); // TXTWriter.writeMap(hist, "rail volume", "share diff", "/home/johannes/gsv/matrices/analysis/marketShares/simVolDiff.txt"); // // hist = Correlations.mean(samples, shareItpDiff.toNativeArray(), FixedSampleSizeDiscretizer.create(samples, 50)); // TXTWriter.writeMap(hist, "rail volume", "share diff", "/home/johannes/gsv/matrices/analysis/marketShares/itpVolDiff.txt"); // // hist = Correlations.mean(refShares.toNativeArray(), simShares.toNativeArray()); // TXTWriter.writeMap(hist, "ref share", "sim share", "/home/johannes/gsv/matrices/analysis/marketShares/simShareCorrel.txt"); // // hist = Correlations.mean(refShares.toNativeArray(), itpShares.toNativeArray()); // TXTWriter.writeMap(hist, "ref share", "itp share", "/home/johannes/gsv/matrices/analysis/marketShares/itpShareCorrel.txt"); }
From source file:playground.johannes.gsv.matrices.analysis.DumpRelations.java
private static void writeRelations(KeyMatrix tomtom, KeyMatrix sim, KeyMatrix itp, KeyMatrix dist, List<Tuple<String, String>> relations, String file) throws IOException { BufferedWriter writer = new BufferedWriter(new FileWriter(file)); writer.write("from\tto\ttomtom\tsim\titp"); writer.newLine();//from ww w . ja va 2 s .co m DescriptiveStatistics simStats = new DescriptiveStatistics(); DescriptiveStatistics itpStats = new DescriptiveStatistics(); for (Tuple<String, String> tuple : relations) { writer.write(tuple.getFirst()); writer.write("\t"); writer.write(tuple.getSecond()); writer.write("\t"); Double tomtomVal = tomtom.get(tuple.getFirst(), tuple.getSecond()); if (tomtomVal == null) tomtomVal = 0.0; writer.write(String.valueOf(tomtomVal)); writer.write("\t"); Double val = sim.get(tuple.getFirst(), tuple.getSecond()); if (val == null) val = 0.0; writer.write(String.valueOf(val)); writer.write("\t"); simStats.addValue(Math.abs((val - tomtomVal) / tomtomVal)); val = itp.get(tuple.getFirst(), tuple.getSecond()); if (val == null) val = 0.0; writer.write(String.valueOf(val)); itpStats.addValue(Math.abs((val - tomtomVal) / tomtomVal)); writer.write("\t"); writer.write(String.valueOf(dist.get(tuple.getFirst(), tuple.getSecond()))); writer.newLine(); } writer.close(); System.out.println("sim: " + simStats.getMean()); System.out.println("itp: " + itpStats.getMean()); }
From source file:playground.johannes.gsv.matrices.analysis.DumpRelations.java
private static DescriptiveStatistics avrRankDiff(Map<Tuple<String, String>, Integer> ranks1, Map<Tuple<String, String>, Integer> ranks2) { DescriptiveStatistics stats = new DescriptiveStatistics(); for (Entry<Tuple<String, String>, Integer> e1 : ranks1.entrySet()) { Integer r1 = e1.getValue(); Integer r2 = ranks2.get(e1.getKey()); if (r2 != null) { stats.addValue(Math.abs(r1 - r2)); }/*from ww w . j a v a2s.co m*/ } return stats; }
From source file:playground.johannes.gsv.matrices.analysis.MatrixCompare.java
public static DescriptiveStatistics relErrorAll(Matrix m1, Matrix m2, boolean absolute, boolean ignoreZeros) { Set<String> zoneIds = new HashSet<>(); zoneIds.addAll(m1.getFromLocations().keySet()); zoneIds.addAll(m1.getToLocations().keySet()); zoneIds.addAll(m2.getFromLocations().keySet()); zoneIds.addAll(m2.getToLocations().keySet()); DescriptiveStatistics stats = new DescriptiveStatistics(); int zeros = 0; for (String id1 : zoneIds) { for (String id2 : zoneIds) { if (!id1.equals(id2)) { Entry e1 = m1.getEntry(id1, id2); Entry e2 = m2.getEntry(id1, id2); if (e1 != null && e2 != null) { Double val1 = e1.getValue(); Double val2 = e2.getValue(); if (!(ignoreZeros && val2 == 0)) { if (val1 > 0) { double err = calcRelError(val1, val2, absolute); stats.addValue(err); }/* w w w . j av a2s. co m*/ } } else zeros++; } } } if (zeros > 0) { System.out.println(String.format("%s cells where either old or new is zero.", zeros)); } return stats; }
From source file:playground.johannes.gsv.matrices.analysis.MatrixCompare.java
public static DescriptiveStatistics errorStats(TObjectDoubleHashMap<String> values) { DescriptiveStatistics stats = new DescriptiveStatistics(); for (double val : values.getValues()) { stats.addValue(val); }/*from w w w . j a va 2s. c o m*/ return stats; }
From source file:playground.johannes.gsv.matrices.analysis.NUTSCompare.java
private static DescriptiveStatistics absError(KeyMatrix m1, KeyMatrix m2) { KeyMatrix errMatrix = MatrixOperations.errorMatrix(m1, m2); // KeyMatrix errMatrix = MatrixOperations.diffMatrix(m1, m2); DescriptiveStatistics stats = new DescriptiveStatistics(); Set<String> keys = errMatrix.keys(); for (String i : keys) { for (String j : keys) { // if(i != j) { Double val = errMatrix.get(i, j); if (val != null) { stats.addValue(Math.abs(val)); // stats.addValue(val); // if(val >= 1) logger.info(String.format("%s - %s: %s", i, j, val)); // } }// w ww .j a v a 2 s . c o m } } return stats; }
From source file:playground.johannes.gsv.misc.MatrixCompareNorm.java
/** * @param args/*from w w w . j av a2s . c o m*/ */ public static void main(String[] args) { Matrix m1 = new Matrix("1", null); VisumMatrixReader reader = new VisumMatrixReader(m1); reader.readFile("/home/johannes/gsv/matrices/IV_gesamt.O.fma"); normMatrix(m1); Matrix m2 = new Matrix("2", null); reader = new VisumMatrixReader(m2); reader.readFile("/home/johannes/gsv/matrices/miv.277.fma"); normMatrix(m2); int notfound = 0; DescriptiveStatistics oRelErrs = new DescriptiveStatistics(); Set<String> origs = m1.getFromLocations().keySet(); for (String origId : origs) { List<Entry> entries1 = m1.getFromLocEntries(origId); List<Entry> entries2 = m2.getFromLocEntries(origId); double sum1 = 0; for (Entry entry : entries1) { sum1 += entry.getValue(); } if (entries2 == null) { oRelErrs.addValue(-1); } else if (entries2 != null && sum1 > 0) { double sum2 = 0; for (Entry entry : entries2) { sum2 += entry.getValue(); } oRelErrs.addValue((sum2 - sum1) / sum1); } else { notfound++; } } System.err.println(String.format("%s entries out of %s not found or with zero value.", notfound, notfound + oRelErrs.getN())); System.out.println(String.format("Rel err of origins: mean=%s, med=%s, min=%s, max=%s", oRelErrs.getMean(), oRelErrs.getPercentile(0.5), oRelErrs.getMin(), oRelErrs.getMax())); DescriptiveStatistics dRelErrs = new DescriptiveStatistics(); Set<String> dests = m1.getToLocations().keySet(); for (String destId : dests) { List<Entry> entries1 = m1.getToLocEntries(destId); List<Entry> entries2 = m2.getToLocEntries(destId); double sum1 = 0; for (Entry entry : entries1) { sum1 += entry.getValue(); } if (entries2 != null && sum1 > 0) { double sum2 = 0; for (Entry entry : entries2) { sum2 += entry.getValue(); } dRelErrs.addValue((sum2 - sum1) / sum1); } } System.out.println(String.format("Rel err of destinations: mean=%s, med=%s, min=%s, max=%s", dRelErrs.getMean(), dRelErrs.getPercentile(0.5), dRelErrs.getMin(), dRelErrs.getMax())); Map<String, String> ids = new HashMap<>(); ids.put("6412", "FRA"); ids.put("11000", "BER"); ids.put("2000", "HAM"); ids.put("3241", "HAN"); ids.put("5315", "KLN"); ids.put("9162", "MUN"); ids.put("8111", "STG"); Map<String, Double> errors = new HashMap<>(); for (String id1 : ids.keySet()) { for (String id2 : ids.keySet()) { if (!id1.equalsIgnoreCase(id2)) { Entry e1 = m1.getEntry(id1, id2); double val1 = e1.getValue(); Entry e2 = m2.getEntry(id1, id2); double val2 = e2.getValue(); double err = (val2 - val1) / val1; System.out.print(ids.get(id1)); System.out.print(" -> "); System.out.print(ids.get(id2)); System.out.print(": "); System.out.println(String.valueOf(err)); } } } }
From source file:playground.johannes.gsv.sim.cadyts.ODCountsAnalyzer.java
@Override public void notifyAfterMobsim(AfterMobsimEvent event) { Network network = event.getControler().getScenario().getNetwork(); DescriptiveStatistics diff = new DescriptiveStatistics(); DescriptiveStatistics absDiff = new DescriptiveStatistics(); DescriptiveStatistics error = new DescriptiveStatistics(); DescriptiveStatistics absError = new DescriptiveStatistics(); try {/* ww w . ja va 2s . c om*/ String file = event.getControler().getControlerIO().getIterationFilename(event.getIteration(), "odCounts.txt"); BufferedWriter writer = new BufferedWriter(new FileWriter(file)); writer.write("id\tobs\tsim"); writer.newLine(); for (Count count : counts.getCounts().values()) { if (count.getLocId().toString().startsWith(ODCalibrator.VIRTUAL_ID_PREFIX)) { Link link = network.getLinks().get(count.getLocId()); double refVal = count.getMaxVolume().getValue() * 24; double simVal = simResults.getSimValue(link, 0, 86400, TYPE.COUNT_VEH); double err = (simVal - refVal) / refVal; error.addValue(err); absError.addValue(Math.abs(err)); double delta = simVal - refVal; diff.addValue(delta); absDiff.addValue(Math.abs(delta)); writer.write(link.getId().toString()); writer.write("\t"); writer.write(String.valueOf(refVal)); writer.write("\t"); writer.write(String.valueOf(simVal)); writer.newLine(); } } writer.close(); logger.info(String.format("OD-relations diff: avr = %s, median = %s, var = %s, min = %s, max = %s", diff.getMean(), diff.getPercentile(50), diff.getVariance(), diff.getMin(), diff.getMax())); logger.info( String.format("OD-relations absolute diff: avr = %s, median = %s, var = %s, min = %s, max = %s", absDiff.getMean(), absDiff.getPercentile(50), absDiff.getVariance(), absDiff.getMin(), absDiff.getMax())); logger.info(String.format( "Relative OD-relations error: avr = %s, median = %s, var = %s, min = %s, max = %s", error.getMean(), error.getPercentile(50), error.getVariance(), error.getMin(), error.getMax())); logger.info(String.format( "Absolute relative OD-relations error: avr = %s, median = %s, var = %s, min = %s, max = %s", absError.getMean(), absError.getPercentile(50), absError.getVariance(), absError.getMin(), absError.getMax())); file = event.getControler().getControlerIO().getOutputFilename("odCountsDiff.txt"); writeStats(file, diff, event.getIteration()); file = event.getControler().getControlerIO().getOutputFilename("odCountsAbsDiff.txt"); writeStats(file, absDiff, event.getIteration()); file = event.getControler().getControlerIO().getOutputFilename("odCountsError.txt"); writeStats(file, error, event.getIteration()); file = event.getControler().getControlerIO().getOutputFilename("odCountsAbsError.txt"); writeStats(file, absError, event.getIteration()); } catch (IOException e) { e.printStackTrace(); } }
From source file:playground.johannes.gsv.synPop.analysis.ActivityChainTask.java
@Override public void analyze(Collection<ProxyPerson> persons, Map<String, DescriptiveStatistics> results) { TObjectDoubleHashMap<String> chains = new TObjectDoubleHashMap<String>(); TObjectIntHashMap<String> typeCount = new TObjectIntHashMap<String>(); TDoubleDoubleHashMap tripCounts = new TDoubleDoubleHashMap(); for (ProxyPerson person : persons) { ProxyPlan plan = person.getPlan(); StringBuilder builder = new StringBuilder(); for (int i = 0; i < plan.getActivities().size(); i++) { String type = (String) plan.getActivities().get(i).getAttribute(CommonKeys.ACTIVITY_TYPE); typeCount.adjustOrPutValue(type, 1, 1); builder.append(type);/*from w w w. j ava 2 s. co m*/ builder.append("-"); } String chain = builder.toString(); chains.adjustOrPutValue(chain, 1, 1); tripCounts.adjustOrPutValue(plan.getLegs().size(), 1, 1); } for (Object key : typeCount.keys()) { DescriptiveStatistics stats = new DescriptiveStatistics(); stats.addValue(typeCount.get((String) key)); results.put(String.format("%s.%s", KEY, key), stats); } if (outputDirectoryNotNull()) { try { TXTWriter.writeMap(chains, "chain", "n", getOutputDirectory() + "/actchains.txt", true); TXTWriter.writeMap(tripCounts, "nTrips", "n", getOutputDirectory() + "/tripcounts.txt", true); } catch (IOException e) { e.printStackTrace(); } } }
From source file:playground.johannes.gsv.synPop.analysis.ActivityDistanceTask.java
protected DescriptiveStatistics statistics(Collection<ProxyPerson> persons, String purpose, String mode) { DescriptiveStatistics stats = new DescriptiveStatistics(); for (ProxyPerson person : persons) { ProxyPlan plan = person.getPlan(); for (int i = 1; i < plan.getActivities().size(); i++) { ProxyObject thisAct = plan.getActivities().get(i); ProxyObject leg = plan.getLegs().get(i - 1); if (mode.equalsIgnoreCase(leg.getAttribute(CommonKeys.LEG_MODE))) { if (purpose == null || purpose.equalsIgnoreCase(thisAct.getAttribute(CommonKeys.ACTIVITY_TYPE))) { ProxyObject prevAct = plan.getActivities().get(i - 1); Id<ActivityFacility> prevId = Id.create(prevAct.getAttribute(CommonKeys.ACTIVITY_FACILITY), ActivityFacility.class); ActivityFacility prevFac = facilities.getFacilities().get(prevId); Id<ActivityFacility> thisId = Id.create(thisAct.getAttribute(CommonKeys.ACTIVITY_FACILITY), ActivityFacility.class); ActivityFacility thisFac = facilities.getFacilities().get(thisId); Point p1 = MatsimCoordUtils.coordToPoint(prevFac.getCoord()); Point p2 = MatsimCoordUtils.coordToPoint(thisFac.getCoord()); double d = calc.distance(p1, p2); stats.addValue(d); }/*ww w. j av a 2s .c om*/ } } } return stats; }