Example usage for org.apache.commons.math3.stat.correlation SpearmansCorrelation SpearmansCorrelation

Introduction

In this page you can find the example usage for org.apache.commons.math3.stat.correlation SpearmansCorrelation SpearmansCorrelation.

Prototype

public SpearmansCorrelation(final RealMatrix dataMatrix) 

Source Link

Document

Create a SpearmansCorrelation from the given data matrix.

Usage

From source file:de.tudarmstadt.ukp.dkpro.c4corpus.hadoop.statistics.vocabulary.TopNWordsCorrelation.java

/**
 * Computes Spearman correlation by comparing order of two corpora vocabularies
 *
 * @param goldCorpus  gold corpus/*from  ww  w  .j  a  va2  s  .  c om*/
 * @param otherCorpus other corpus
 * @param topN        how many entries from the gold corpus should be taken
 * @throws IOException I/O exception
 */
public static void spearmanCorrelation(File goldCorpus, File otherCorpus, int topN) throws IOException {
    LinkedHashMap<String, Integer> gold = loadCorpusToRankedVocabulary(new FileInputStream(goldCorpus));
    LinkedHashMap<String, Integer> other = loadCorpusToRankedVocabulary(new FileInputStream(otherCorpus));

    double[][] matrix = new double[topN][];

    if (gold.size() < topN) {
        throw new IllegalArgumentException(
                "topN (" + topN + ") cannot be greater than vocabulary size (" + gold.size() + ")");
    }

    Iterator<Map.Entry<String, Integer>> iterator = gold.entrySet().iterator();
    int counter = 0;
    while (counter < topN) {
        Map.Entry<String, Integer> next = iterator.next();
        String goldWord = next.getKey();
        Integer goldValue = next.getValue();

        // look-up position in other corpus
        Integer otherValue = other.get(goldWord);
        if (otherValue == null) {
            //                System.err.println("Word " + goldWord + " not found in the other corpus");
            otherValue = Integer.MAX_VALUE;
        }

        matrix[counter] = new double[2];
        matrix[counter][0] = goldValue;
        matrix[counter][1] = otherValue;

        counter++;
    }

    RealMatrix realMatrix = new Array2DRowRealMatrix(matrix);

    SpearmansCorrelation spearmansCorrelation = new SpearmansCorrelation(realMatrix);
    double pValue = spearmansCorrelation.getRankCorrelation().getCorrelationPValues().getEntry(0, 1);
    double correlation = spearmansCorrelation.getRankCorrelation().getCorrelationMatrix().getEntry(0, 1);

    System.out.println("Gold: " + goldCorpus.getName());
    System.out.println("Other: " + otherCorpus.getName());
    System.out.printf(Locale.ENGLISH, "Top N:\n%d\nCorrelation\n%.3f\np-value\n%.3f\n", topN, correlation,
            pValue);
}

---

From source file:org.meteoinfo.math.stats.StatsUtil.java

/**
 * Computes Spearman's rank correlation for pairs of arrays or columns of a matrix.
 *
 * @param x X data/*from  w w w  .  j  a  va2 s .  c  o m*/
 * @param y Y data
 * @return Spearman's rank correlation
 */
public static Array spearmanr(Array x, Array y) {
    int m = x.getShape()[0];
    int n = 1;
    if (x.getRank() == 2)
        n = x.getShape()[1];
    double[][] aa = new double[m][n * 2];
    for (int i = 0; i < m; i++) {
        for (int j = 0; j < n * 2; j++) {
            if (j < n) {
                aa[i][j] = x.getDouble(i * n + j);
            } else {
                aa[i][j] = y.getDouble(i * n + j - n);
            }
        }
    }
    RealMatrix matrix = new Array2DRowRealMatrix(aa, false);
    SpearmansCorrelation cov = new SpearmansCorrelation(matrix);
    RealMatrix mcov = cov.getCorrelationMatrix();
    m = mcov.getColumnDimension();
    n = mcov.getRowDimension();
    Array r = Array.factory(DataType.DOUBLE, new int[] { m, n });
    for (int i = 0; i < m; i++) {
        for (int j = 0; j < n; j++) {
            r.setDouble(i * n + j, mcov.getEntry(i, j));
        }
    }

    return r;
}

---

From source file:org.meteoinfo.math.stats.StatsUtil.java

/**
 * Computes Spearman's rank correlation for columns of a matrix.
 * @param a Matrix data//from   www .j  a v a  2 s  .c  o m
 * @return Spearman's rank correlation
 */
public static Object spearmanr(Array a) {
    if (a.getRank() == 1) {
        double[] ad = (double[]) ArrayUtil.copyToNDJavaArray(a);
        Covariance cov = new Covariance();
        return cov.covariance(ad, ad);
    } else {
        double[][] aa = (double[][]) ArrayUtil.copyToNDJavaArray(a);
        RealMatrix matrix = new Array2DRowRealMatrix(aa, false);
        SpearmansCorrelation cov = new SpearmansCorrelation(matrix);
        RealMatrix mcov = cov.getCorrelationMatrix();
        int m = mcov.getColumnDimension();
        int n = mcov.getRowDimension();
        Array r = Array.factory(DataType.DOUBLE, new int[] { m, n });
        for (int i = 0; i < m; i++) {
            for (int j = 0; j < n; j++) {
                r.setDouble(i * n + j, mcov.getEntry(i, j));
            }
        }

        return r;
    }
}

---