Example usage for org.apache.commons.math.stat.descriptive DescriptiveStatistics DescriptiveStatistics

List of usage examples for org.apache.commons.math.stat.descriptive DescriptiveStatistics DescriptiveStatistics

  1. HOME
  2. Java
  3. org.apache.commons
  4. org.apache.commons.math.stat.descriptive.*
  5. DescriptiveStatistics
  6. DescriptiveStatistics

Introduction

In this page you can find the example usage for org.apache.commons.math.stat.descriptive DescriptiveStatistics DescriptiveStatistics.

Prototype

public DescriptiveStatistics()

Source Link

Document

Construct a DescriptiveStatistics instance with an infinite window

Usage

From source file:playground.johannes.snowball2.Clustering.java

@SuppressWarnings("unchecked")
@Override/*from   ww  w.  j a va 2  s. c  o m*/
public DescriptiveStatistics calculate(Graph g, int iteration, DescriptiveStatistics reference) {
    Map<Vertex, Double> values = GraphStatistics.clusteringCoefficients(g);
    DescriptiveStatistics stats = new DescriptiveStatistics();

    TIntDoubleHashMap degreeClustering = new TIntDoubleHashMap();
    TIntIntHashMap numDegree = new TIntIntHashMap();

    double sum = 0;
    double wsum = 0;

    if (g instanceof SampledGraph) {
        for (Vertex v : values.keySet()) {
            int k = v.degree();

            if (!((SampledVertex) v).isAnonymous()) {
                double cc = degreeClustering.get(k);

                if (v.degree() == 1) {
                    stats.addValue(0.0);
                    //                  sum += (cc / ((SampledVertex)v).getSampleProbability());
                } else {
                    double C = values.get(v);
                    stats.addValue(C);
                    cc += C;
                    sum += (C / ((SampledVertex) v).getSampleProbability());
                }
                degreeClustering.put(k, cc);
                numDegree.put(k, numDegree.get(k) + 1);

                wsum += (1 / ((SampledVertex) v).getSampleProbability());
            }
        }
    } else {
        for (Vertex v : values.keySet()) {
            int k = v.degree();
            double cc = degreeClustering.get(k);

            wsum++;

            if (v.degree() == 1)
                stats.addValue(0.0);
            else {
                double C = values.get(v);
                stats.addValue(C);
                cc += C;
                sum += C;
            }

            degreeClustering.put(k, cc);
            numDegree.put(k, numDegree.get(k) + 1);
        }
    }

    wMean = sum / wsum;

    try {
        BufferedWriter writer = IOUtils
                .getBufferedWriter(String.format("%1$s/%2$s.degreeDependency.txt", outputDir, iteration));
        int[] keys = numDegree.keys();
        Arrays.sort(keys);
        for (int k : keys) {
            double bc = degreeClustering.get(k);
            int numV = numDegree.get(k);

            writer.write(String.valueOf(k));
            writer.write("\t");
            writer.write(String.valueOf(bc / (double) numV));
            writer.newLine();
        }
        writer.close();
    } catch (Exception e) {
        e.printStackTrace();
    }

    dumpStatistics(getStatisticsMap(stats), iteration);

    if (reference != null) {
        Histogram hist = new Histogram(100, reference.getMin(), reference.getMax());
        plotHistogram(stats.getValues(), hist, iteration);
    } else {
        plotHistogram(stats.getValues(), new Histogram(100), iteration);
    }

    return stats;
}

From source file:playground.johannes.snowball2.ComponentsStats.java

public DescriptiveStatistics calculate(Graph g, int iteration, DescriptiveStatistics reference) {
    if (g instanceof SampledGraph) {
        Set<Collection<SampledVertex>> clusters = SampledGraphStatistics
                .getDisconnectedComponents((SampledGraph) g);
        Collection<SampledGraph> subGraphs = new LinkedList<SampledGraph>();
        for (Collection<SampledVertex> cluster : clusters) {
            subGraphs.add(SampledGraphStatistics.extractGraphFromCluster(cluster));
        }/*from w w w. ja v  a  2  s  .  c o  m*/

        for (int i = 0; i < seeds.size(); i++) {
            SampledGraph subGraph = graphForSeed(seeds.get(i), subGraphs);
            BufferedWriter writer = writers.get(i);

            int numSampledVertices = 0;
            Set<SampledVertex> vertices = subGraph.getVertices();
            for (SampledVertex v : vertices) {
                if (!v.isAnonymous())
                    numSampledVertices++;
            }

            int numVisitedVertices = 0;
            for (SampledVertex v : vertices) {
                numVisitedVertices += v.getVisited();
            }

            int deltaSampled = numSampledVertices - lastNumSampledVertices.get(seeds.get(i));
            int deltaVisited = numVisitedVertices - lastNumVisitedVertices.get(seeds.get(i));
            double efficiency = deltaSampled / (double) deltaLastVisited.get(seeds.get(i));

            try {
                writer.write(String.valueOf(iteration));
                writer.write("\t");
                writer.write(String.valueOf(numSampledVertices));
                writer.write("\t");
                writer.write(String.valueOf(subGraph.numEdges()));
                writer.write("\t");
                writer.write(String.format(Locale.US, "%1.4f",
                        numSampledVertices / (double) lastNumSampledVertices.get(seeds.get(i))));
                writer.write("\t");
                writer.write(String.format(Locale.US, "%1.4f", efficiency));
                writer.write("\t");
                writer.write(String.format(Locale.US, "%1.4f",
                        SampledGraphStatistics.getDegreeStatistics(subGraph).getMean()));
                writer.write("\t");
                writer.write(String.format(Locale.US, "%1.4f",
                        SampledGraphStatistics.getClusteringStatistics(subGraph).getMean()));
                writer.newLine();
                writer.flush();
            } catch (IOException e) {
                e.printStackTrace();
            }

            lastNumSampledVertices.put(seeds.get(i), numSampledVertices);
            lastNumVisitedVertices.put(seeds.get(i), numVisitedVertices);
            deltaLastVisited.put(seeds.get(i), deltaVisited);
        }
    }

    return new DescriptiveStatistics();
}

From source file:playground.johannes.snowball2.CountComponents.java

public DescriptiveStatistics calculate(Graph g, int iteration, DescriptiveStatistics reference) {
    DescriptiveStatistics stats = new DescriptiveStatistics();
    stats.addValue(GraphStatistics.getDisconnectedComponents(g).size());
    return stats;
}

From source file:playground.johannes.snowball2.CountIsolates.java

@SuppressWarnings("unchecked")
@Override/*ww  w .  j  av a 2 s  . c  om*/
public DescriptiveStatistics calculate(Graph g, int iteration, DescriptiveStatistics reference) {
    DescriptiveStatistics stats = new DescriptiveStatistics();
    Set<Vertex> vertices = g.getVertices();
    int count = 0;
    for (Vertex v : vertices)
        if (v.degree() == 0)
            count++;
    stats.addValue(count);
    return stats;
}

From source file:playground.johannes.snowball2.Degree.java

@SuppressWarnings("unchecked")
@Override//from   w  w  w .ja  v a  2s.  co  m
public DescriptiveStatistics calculate(Graph g, int iteration, DescriptiveStatistics reference) {
    DescriptiveStatistics stats = new DescriptiveStatistics();
    TDoubleArrayList values = new TDoubleArrayList(g.numVertices());
    TDoubleArrayList weights = new TDoubleArrayList(g.numVertices());
    TDoubleArrayList normWeights = new TDoubleArrayList(g.numVertices());

    if (g instanceof SampledGraph) {
        Set<SampledVertex> vertices = g.getVertices();

        double wsum = 0.0;
        for (SampledVertex v : vertices) {
            if (!v.isAnonymous()) {
                values.add(v.degree());
                if (biasCorrection) {
                    weights.add(1 / v.getSampleProbability());
                    wsum += 1 / v.getSampleProbability();
                } else {
                    weights.add(1.0);
                    wsum++;
                }
            }
        }
        double k = values.size() / wsum;
        for (int i = 0; i < weights.size(); i++) {
            normWeights.add(weights.getQuick(i) * k);
            stats.addValue(values.getQuick(i) * normWeights.getQuick(i));
        }

    } else {
        Set<Vertex> vertices = g.getVertices();
        for (Vertex v : vertices) {
            stats.addValue(v.degree());
            values.add(v.degree());
            weights.add(1.0);
        }
    }

    gamma = calcGammaExponent(values.toNativeArray(), weights.toNativeArray(), 1.0, 0);

    dumpStatistics(getStatisticsMap(stats), iteration);

    if (reference != null) {
        Histogram hist = new Histogram(1.0, reference.getMin(), reference.getMax());
        plotHistogram(values.toNativeArray(), weights.toNativeArray(), hist, iteration);
    } else {
        plotHistogram(values.toNativeArray(), weights.toNativeArray(), new Histogram(1.0), iteration);
    }

    return stats;
}

From source file:playground.johannes.snowball2.DegreeCorrelation.java

public DescriptiveStatistics calculate(Graph g, int iteration, DescriptiveStatistics reference) {
    double product = 0;
    double sum = 0;
    double wdegreeSum = 0;
    double squareSum = 0;
    //      double wsquareSum = 0;
    double edges = 0;

    boolean sampled = false;
    if (g instanceof SampledGraph)
        sampled = true;// www  .ja v  a  2s  .c o m

    /*
     * Calculate normalization constant
     */
    double wconst = 1;
    if (sampled) {
        double wsum = 0;
        int count = 0;
        for (Object v : g.getVertices()) {
            if (!((SampledVertex) v).isAnonymous()) {
                wsum += 1 / ((SampledVertex) v).getSampleProbability();
                count++;
            }
        }
        wconst = count / wsum;
    }

    double probaSum = 0;
    double probaSquareSum = 0;
    double probaProductSum = 0;
    for (Object e : g.getEdges()) {
        Pair p = ((Edge) e).getEndpoints();
        Vertex v1 = (Vertex) p.getFirst();
        Vertex v2 = (Vertex) p.getSecond();

        if (sampled) {
            //            if (!((SampledVertex)v1).isAnonymous() && !((SampledVertex)v2).isAnonymous()) {
            //               int d_v1 = v1.degree();
            //               int d_v2 = v2.degree();
            ////               double d_v1 = v1.degree() * 1/((SampledVertex)v1).getSampleProbability();// * wconst;
            ////               double d_v2 = v2.degree() * 1/((SampledVertex)v2).getSampleProbability();// * wconst;
            //
            //               sum += d_v1 + d_v2;
            ////               wdegreeSum += (d_v1 * 1/((SampledVertex)v1).getSampleProbability() * wconst)
            ////                        + (d_v2 * 1/((SampledVertex)v2).getSampleProbability() * wconst);
            //               squareSum += Math.pow(d_v1, 2) + Math.pow(d_v2, 2);
            ////               wsquareSum += Math.pow(d_v1 * 1/((SampledVertex)v1).getSampleProbability() * wconst, 2)
            ////                        + Math.pow(d_v2 * 1/((SampledVertex)v2).getSampleProbability() * wconst, 2);
            ////               product += d_v1 * 1/((SampledVertex)v1).getSampleProbability() * wconst
            ////                      * d_v2 * 1/((SampledVertex)v2).getSampleProbability() * wconst;
            //               product += d_v1 * d_v2;
            //               
            //               edges++;
            //            }

            if (!((SampledVertex) v1).isAnonymous() && !((SampledVertex) v2).isAnonymous()) {
                int d_v1 = v1.degree();
                int d_v2 = v2.degree();
                double p_v1 = ((SampledVertex) v1).getSampleProbability();// * wconst;
                double p_v2 = ((SampledVertex) v2).getSampleProbability();// * wconst;
                double proba = p_v1 * p_v2;

                sum += 0.5 * (d_v1 + d_v2) / proba;
                probaSum += 1 / proba;

                squareSum += 0.5 * (Math.pow(d_v1, 2) + Math.pow(d_v2, 2)) / proba;
                probaSquareSum = 1 / proba;

                product += (d_v1 * d_v2) / proba;
                probaProductSum += 1 / proba;

                edges++;
            }
        } else {
            int d_v1 = v1.degree();
            int d_v2 = v2.degree();

            sum += 0.5 * (d_v1 + d_v2);
            wdegreeSum = sum;
            squareSum += 0.5 * (Math.pow(d_v1, 2) + Math.pow(d_v2, 2));
            product += d_v1 * d_v2;

            probaSum++;
            probaSquareSum++;
            probaProductSum++;
            edges++;
        }
    }
    double M_minus1 = 1 / (double) edges;
    double normSumSquare = Math.pow((1 / probaSum * sum), 2);
    //      double wSumSquare = Math.pow((M_minus1 * 0.5 * wdegreeSum), 2);
    double numerator = (1 / probaProductSum * product) - normSumSquare;
    double denumerator = (1 / probaSquareSum * squareSum) - normSumSquare;

    double result = numerator / denumerator;

    DescriptiveStatistics stats = new DescriptiveStatistics();
    stats.addValue(result);
    return stats;
}

From source file:playground.johannes.snowball2.Mutuality.java

@SuppressWarnings("unchecked")
public DescriptiveStatistics calculate(Graph g, int iteration, DescriptiveStatistics reference) {
    Map<Vertex, Double> clustering = GraphStatistics.clusteringCoefficients(g);
    Set<Vertex> vertices = g.getVertices();
    double z = 0;
    double m = 0;

    boolean isSampled = false;
    if (g instanceof SampledGraph)
        isSampled = true;/* w  ww .ja va2s . co m*/

    for (Vertex v : vertices) {
        if (isSampled) {
            if (!((SampledVertex) v).isAnonymous()) {
                z += v.degree();
                double c = clustering.get(v);
                m += v.degree() / (double) (1 + Math.pow(c, 2) * (v.degree() - 1));
            }
        } else {
            z += v.degree();
            double c = clustering.get(v);
            m += v.degree() / (double) (1 + Math.pow(c, 2) * (v.degree() - 1));
        }
    }
    z = z / (double) vertices.size();
    m = m / (double) vertices.size();

    double result = m / z;

    DescriptiveStatistics stats = new DescriptiveStatistics();
    stats.addValue(result);
    return stats;
}

From source file:playground.johannes.snowball2.TypeClusters.java

public DescriptiveStatistics calculate(Graph g, int iteration, DescriptiveStatistics reference) {

    Set<Vertex> vertices = g.getVertices();
    Map<String, Collection<Vertex>> clusters = new HashMap<String, Collection<Vertex>>();
    for (Vertex v : vertices) {
        String type = (String) v.getUserDatum(UserDataKeys.TYPE_KEY);
        if (type != null) {
            Collection<Vertex> cluster = clusters.get(type);
            if (cluster == null) {
                cluster = new LinkedList<Vertex>();
                clusters.put(type, cluster);
            }//from  w ww .  j  ava 2 s . com
            cluster.add(v);
        }
    }

    Map<String, Graph> subGraphs = new HashMap<String, Graph>();
    for (String type : clusters.keySet()) {
        subGraphs.put(type, extractCluster(clusters.get(type)));
    }

    DescriptiveStatistics degree;
    DescriptiveStatistics clustering;
    if (g instanceof SampledGraph) {
        degree = SampledGraphStatistics.getDegreeStatistics((SampledGraph) g);
        clustering = SampledGraphStatistics.getClusteringStatistics((SampledGraph) g);
    } else {
        degree = GraphStatistics.getDegreeStatistics(g);
        clustering = GraphStatistics.getClusteringStatistics(g);
    }
    //      CountComponents cc = new CountComponents(null, null);
    for (String type : clusters.keySet()) {
        Graph subGraph = subGraphs.get(type);
        float z = (float) degree.getMean();
        float c = (float) clustering.getMean();
        int components = GraphStatistics.getDisconnectedComponents(subGraph).size();

        System.out.println(String.format(
                "Subgraph of type %1$s has %2$s vertices, %3$s components, mean degree %4$s and mean clustering %5$s.",
                type, subGraph.numVertices(), components, z, c));
    }

    return new DescriptiveStatistics();
}

From source file:playground.johannes.socialnetworks.graph.social.analysis.AgeTask.java

@SuppressWarnings("unchecked")
@Override/*w  w  w .  j ava2  s .c o m*/
public void analyze(Graph graph, Map<String, DescriptiveStatistics> statsMap) {
    DescriptiveStatistics stats = module.statistics(graph.getVertices());
    statsMap.put(key, stats);
    printStats(stats, key);

    if (outputDirectoryNotNull()) {
        try {
            writeHistograms(stats, new LinearDiscretizer(1.0), key, false);

            TXTWriter.writeMap(module.correlation((Set<? extends SocialVertex>) graph.getVertices()), "age",
                    "age_mean", getOutputDirectory() + "/age_age.mean.txt");

            TDoubleObjectHashMap<DescriptiveStatistics> stat = module
                    .boxplot((Set<? extends SocialVertex>) graph.getVertices());
            TXTWriter.writeBoxplotStats(stat, getOutputDirectory() + "age_age.table.txt");
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    stats = new DescriptiveStatistics();
    stats.addValue(module.correlationCoefficient((Set<? extends SocialEdge>) graph.getEdges()));
    statsMap.put("r_" + key, stats);
    printStats(stats, "r_" + key);
}

From source file:playground.johannes.socialnetworks.graph.social.analysis.GenderTask.java

@Override
public void analyze(Graph g, Map<String, DescriptiveStatistics> statsMap) {
    SocialGraph graph = (SocialGraph) g;

    Map<SocialVertex, String> values = module.values(graph.getVertices());
    TObjectDoubleHashMap<String> hist = LinguisticHistogram.create(values.values());

    DescriptiveStatistics male = new DescriptiveStatistics();
    male.addValue(hist.get(Gender.MALE));
    String key = "n_male";
    statsMap.put(key, male);//  ww  w.ja  v a2s.co  m
    printStats(male, key);

    DescriptiveStatistics female = new DescriptiveStatistics();
    female.addValue(hist.get(Gender.FEMALE));
    key = "n_female";
    statsMap.put(key, female);
    printStats(female, key);

    DescriptiveStatistics r = new DescriptiveStatistics();
    r.addValue(module.correlation(graph.getEdges()));
    key = "r_gender";
    statsMap.put(key, r);
    printStats(r, key);

    if (outputDirectoryNotNull()) {
        try {
            TXTWriter.writeMap(hist, "gender", "n", String.format("%1$s/gender.txt", getOutputDirectory()));

            SocioMatrix<String> m = module.countsMatrix(graph.getVertices());
            m.toFile(String.format("%1$s/gender.countsMatrix.txt", getOutputDirectory()));

            SocioMatrixBuilder.normalizeTotalSum(m);
            m.toFile(String.format("%1$s/gender.countsMatrix.normTotal.txt", getOutputDirectory()));

            m = module.countsMatrix(graph.getVertices());
            SocioMatrixBuilder.normalizeRowSum(m);
            m.toFile(String.format("%1$s/gender.countsMatrix.normRow.txt", getOutputDirectory()));

            m = module.probaMatrix(graph.getVertices());
            m.toFile(String.format("%1$s/gender.probaMatrix.txt", getOutputDirectory()));
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}