Example usage for org.apache.commons.math3.stat.correlation SpearmansCorrelation getRankCorrelation

Introduction

In this page you can find the example usage for org.apache.commons.math3.stat.correlation SpearmansCorrelation getRankCorrelation.

Prototype

public PearsonsCorrelation getRankCorrelation() 

Source Link

Document

Returns a PearsonsCorrelation instance constructed from the ranked input data.

Usage

From source file:de.tudarmstadt.ukp.dkpro.c4corpus.hadoop.statistics.vocabulary.TopNWordsCorrelation.java

/**
 * Computes Spearman correlation by comparing order of two corpora vocabularies
 *
 * @param goldCorpus  gold corpus//from  ww  w .  j  a  v a2  s  . c o  m
 * @param otherCorpus other corpus
 * @param topN        how many entries from the gold corpus should be taken
 * @throws IOException I/O exception
 */
public static void spearmanCorrelation(File goldCorpus, File otherCorpus, int topN) throws IOException {
    LinkedHashMap<String, Integer> gold = loadCorpusToRankedVocabulary(new FileInputStream(goldCorpus));
    LinkedHashMap<String, Integer> other = loadCorpusToRankedVocabulary(new FileInputStream(otherCorpus));

    double[][] matrix = new double[topN][];

    if (gold.size() < topN) {
        throw new IllegalArgumentException(
                "topN (" + topN + ") cannot be greater than vocabulary size (" + gold.size() + ")");
    }

    Iterator<Map.Entry<String, Integer>> iterator = gold.entrySet().iterator();
    int counter = 0;
    while (counter < topN) {
        Map.Entry<String, Integer> next = iterator.next();
        String goldWord = next.getKey();
        Integer goldValue = next.getValue();

        // look-up position in other corpus
        Integer otherValue = other.get(goldWord);
        if (otherValue == null) {
            //                System.err.println("Word " + goldWord + " not found in the other corpus");
            otherValue = Integer.MAX_VALUE;
        }

        matrix[counter] = new double[2];
        matrix[counter][0] = goldValue;
        matrix[counter][1] = otherValue;

        counter++;
    }

    RealMatrix realMatrix = new Array2DRowRealMatrix(matrix);

    SpearmansCorrelation spearmansCorrelation = new SpearmansCorrelation(realMatrix);
    double pValue = spearmansCorrelation.getRankCorrelation().getCorrelationPValues().getEntry(0, 1);
    double correlation = spearmansCorrelation.getRankCorrelation().getCorrelationMatrix().getEntry(0, 1);

    System.out.println("Gold: " + goldCorpus.getName());
    System.out.println("Other: " + otherCorpus.getName());
    System.out.printf(Locale.ENGLISH, "Top N:\n%d\nCorrelation\n%.3f\np-value\n%.3f\n", topN, correlation,
            pValue);
}

---

From source file:org.apache.solr.client.solrj.io.eval.CorrelationEvaluator.java

@Override
public Object doWork(Object... values) throws IOException {

    if (values.length == 2) {
        Object first = values[0];
        Object second = values[1];

        if (null == first) {
            throw new IOException(
                    String.format(Locale.ROOT, "Invalid expression %s - null found for the first value",
                            toExpression(constructingFactory)));
        }/*from   ww w  .  j  a  v a2s  .c  om*/
        if (null == second) {
            throw new IOException(
                    String.format(Locale.ROOT, "Invalid expression %s - null found for the second value",
                            toExpression(constructingFactory)));
        }
        if (!(first instanceof List<?>)) {
            throw new IOException(String.format(Locale.ROOT,
                    "Invalid expression %s - found type %s for the first value, expecting a list of numbers",
                    toExpression(constructingFactory), first.getClass().getSimpleName()));
        }
        if (!(second instanceof List<?>)) {
            throw new IOException(String.format(Locale.ROOT,
                    "Invalid expression %s - found type %s for the second value, expecting a list of numbers",
                    toExpression(constructingFactory), first.getClass().getSimpleName()));
        }

        if (type.equals(CorrelationType.pearsons)) {
            PearsonsCorrelation pearsonsCorrelation = new PearsonsCorrelation();
            return pearsonsCorrelation.correlation(
                    ((List) first).stream().mapToDouble(value -> ((BigDecimal) value).doubleValue()).toArray(),
                    ((List) second).stream().mapToDouble(value -> ((BigDecimal) value).doubleValue())
                            .toArray());
        } else if (type.equals(CorrelationType.kendalls)) {
            KendallsCorrelation kendallsCorrelation = new KendallsCorrelation();
            return kendallsCorrelation.correlation(
                    ((List) first).stream().mapToDouble(value -> ((BigDecimal) value).doubleValue()).toArray(),
                    ((List) second).stream().mapToDouble(value -> ((BigDecimal) value).doubleValue())
                            .toArray());

        } else if (type.equals(CorrelationType.spearmans)) {
            SpearmansCorrelation spearmansCorrelation = new SpearmansCorrelation();
            return spearmansCorrelation.correlation(
                    ((List) first).stream().mapToDouble(value -> ((BigDecimal) value).doubleValue()).toArray(),
                    ((List) second).stream().mapToDouble(value -> ((BigDecimal) value).doubleValue())
                            .toArray());
        } else {
            return null;
        }
    } else if (values.length == 1) {
        if (values[0] instanceof Matrix) {
            Matrix matrix = (Matrix) values[0];
            double[][] data = matrix.getData();
            if (type.equals(CorrelationType.pearsons)) {
                PearsonsCorrelation pearsonsCorrelation = new PearsonsCorrelation(data);
                RealMatrix corrMatrix = pearsonsCorrelation.getCorrelationMatrix();
                double[][] corrMatrixData = corrMatrix.getData();
                Matrix realMatrix = new Matrix(corrMatrixData);
                realMatrix.addToContext("corr", pearsonsCorrelation);
                return realMatrix;
            } else if (type.equals(CorrelationType.kendalls)) {
                KendallsCorrelation kendallsCorrelation = new KendallsCorrelation(data);
                RealMatrix corrMatrix = kendallsCorrelation.getCorrelationMatrix();
                double[][] corrMatrixData = corrMatrix.getData();
                Matrix realMatrix = new Matrix(corrMatrixData);
                realMatrix.addToContext("corr", kendallsCorrelation);
                return realMatrix;
            } else if (type.equals(CorrelationType.spearmans)) {
                SpearmansCorrelation spearmansCorrelation = new SpearmansCorrelation(
                        new Array2DRowRealMatrix(data));
                RealMatrix corrMatrix = spearmansCorrelation.getCorrelationMatrix();
                double[][] corrMatrixData = corrMatrix.getData();
                Matrix realMatrix = new Matrix(corrMatrixData);
                realMatrix.addToContext("corr", spearmansCorrelation.getRankCorrelation());
                return realMatrix;
            } else {
                return null;
            }
        } else {
            throw new IOException(
                    "corr function operates on either two numeric arrays or a single matrix as parameters.");
        }
    } else {
        throw new IOException(
                "corr function operates on either two numeric arrays or a single matrix as parameters.");
    }
}

---