List of usage examples for org.apache.commons.math3.stat.descriptive.moment Skewness Skewness
public Skewness()
From source file:com.facebook.presto.operator.aggregation.TestDoubleSkewnessAggregation.java
@Override public Number getExpectedValue(int start, int length) { if (length < 3) { return null; }//from ww w . j ava 2 s . com double[] values = new double[length]; for (int i = 0; i < length; i++) { values[i] = start + i; } Skewness skewness = new Skewness(); return skewness.evaluate(values); }
From source file:feature.lowLevel.audio.FeatureExtractor.java
private double[] getStatisticalSpectrumDescriptor(double[] dataRow) { int N = dataRow.length; double ssd[] = new double[N_STATISTICAL_DESCRIPTORS]; ssd[0] = new Mean().evaluate(dataRow); ssd[1] = new Variance().evaluate(dataRow); ssd[2] = new Skewness().evaluate(dataRow); ssd[3] = new Kurtosis().evaluate(dataRow); // NOTE: be careful, sort changes data! // (as sonogram.getRow() above copies the data anyway, in this case there is no problem) // (otherwise, use dataRow.clone(); ) Arrays.sort(dataRow);//from ww w . j av a 2 s . c o m // median if (N % 2 == 0) { ssd[4] = (dataRow[(N / 2) - 1] + dataRow[(N / 2)]) / 2; } else { ssd[4] = dataRow[(N - 1) / 2]; } ssd[5] = dataRow[0]; // min value ssd[6] = dataRow[N - 1]; // max value return ssd; }
From source file:msi.gaml.operators.Stats.java
@operator(value = "skewness", can_be_const = false, // type = IType.LIST, category = { IOperatorCategory.STATISTICAL }, concept = { IConcept.STATISTIC, IConcept.CLUSTERING }) @doc(value = "returns skewness value computed from the operand list of values", special_cases = "if the length of the list is lower than 3, returns NaN", examples = { @example(value = "skewness ([1,2,3,4,5])", equals = "0.0") }) public static Double skewness(final IScope scope, final GamaList data) throws GamaRuntimeException { final Skewness sk = new Skewness(); final double[] values = new double[data.length(scope)]; for (int i = 0; i < values.length; i++) { values[i] = Cast.asFloat(scope, data.get(i)); }// w w w . j av a 2s . co m return sk.evaluate(values); }
From source file:nl.systemsgenetics.genenetworkbackend.div.CalculateGenePredictability.java
/** * @param args the command line arguments *//*w w w.j ava 2 s.c om*/ public static void main(String[] args) throws IOException { File predictionMatrixFile = new File( "C:\\UMCG\\Genetica\\Projects\\GeneNetwork\\Data31995Genes05-12-2017\\PCA_01_02_2018\\predictions\\reactome_predictions.txt.gz"); File annotationMatrixFile = new File( "C:\\UMCG\\Genetica\\Projects\\GeneNetwork\\Data31995Genes05-12-2017\\PCA_01_02_2018\\PathwayMatrix\\Ensembl2Reactome_All_Levels.txt_matrix.txt.gz"); File significantTermsFile = new File( "C:\\UMCG\\Genetica\\Projects\\GeneNetwork\\Data31995Genes05-12-2017\\PCA_01_02_2018\\predictions\\reactome_predictions_bonSigTerms_alsoInGoP.txt"); File outputFile = new File( "C:\\UMCG\\Genetica\\Projects\\GeneNetwork\\Data31995Genes05-12-2017\\PCA_01_02_2018\\predictions\\reactome_predictions_genePredictability_alsoInGoP.txt"); // File predictionMatrixFile = new File("C:\\UMCG\\Genetica\\Projects\\GeneNetwork\\Data31995Genes05-12-2017\\PCA_01_02_2018\\predictions\\hpo_predictions.txt.gz"); // File annotationMatrixFile = new File("C:\\UMCG\\Genetica\\Projects\\GeneNetwork\\Data31995Genes05-12-2017\\PCA_01_02_2018\\PathwayMatrix\\ALL_SOURCES_ALL_FREQUENCIES_phenotype_to_genes.txt_matrix.txt.gz"); // File significantTermsFile = new File("C:\\UMCG\\Genetica\\Projects\\GeneNetwork\\Data31995Genes05-12-2017\\PCA_01_02_2018\\predictions\\hpo_predictions_bonSigTerms.txt"); // File outputFile = new File("C:\\UMCG\\Genetica\\Projects\\GeneNetwork\\Data31995Genes05-12-2017\\PCA_01_02_2018\\predictions\\hpo_predictions_genePredictability.txt"); // // File predictionMatrixFile = new File("C:\\UMCG\\Genetica\\Projects\\GeneNetwork\\Data31995Genes05-12-2017\\PCA_01_02_2018\\predictions\\go_F_predictions.txt.gz"); // File annotationMatrixFile = new File("C:\\UMCG\\Genetica\\Projects\\GeneNetwork\\Data31995Genes05-12-2017\\PCA_01_02_2018\\PathwayMatrix\\goa_human.gaf_F_matrix.txt.gz"); // File significantTermsFile = new File("C:\\UMCG\\Genetica\\Projects\\GeneNetwork\\Data31995Genes05-12-2017\\PCA_01_02_2018\\predictions\\go_F_predictions_bonSigTerms.txt"); // File outputFile = new File("C:\\UMCG\\Genetica\\Projects\\GeneNetwork\\Data31995Genes05-12-2017\\PCA_01_02_2018\\predictions\\go_F_predictions_genePredictability.txt"); // // File predictionMatrixFile = new File("C:\\UMCG\\Genetica\\Projects\\GeneNetwork\\Data31995Genes05-12-2017\\PCA_01_02_2018\\predictions\\kegg_predictions.txt.gz"); // File annotationMatrixFile = new File("C:\\UMCG\\Genetica\\Projects\\GeneNetwork\\Data31995Genes05-12-2017\\PCA_01_02_2018\\PathwayMatrix\\c2.cp.kegg.v6.1.entrez.gmt_matrix.txt.gz"); // File significantTermsFile = new File("C:\\UMCG\\Genetica\\Projects\\GeneNetwork\\Data31995Genes05-12-2017\\PCA_01_02_2018\\predictions\\kegg_predictions_bonSigTerms.txt"); // File outputFile = new File("C:\\UMCG\\Genetica\\Projects\\GeneNetwork\\Data31995Genes05-12-2017\\PCA_01_02_2018\\predictions\\kegg_predictions_genePredictability.txt"); // // File predictionMatrixFile = new File("/groups/umcg-wijmenga/tmp04/umcg-svandam/GeneNetwork/Data31995Genes05-12-2017/GeneNetwork_V2_01-02-2018/Covariates/PCA/predictions/go_P_predictions.txt.gz"); // File annotationMatrixFile = new File("/groups/umcg-wijmenga/tmp04/umcg-svandam/GeneNetwork/Data31995Genes05-12-2017/GeneNetwork_V2_01-02-2018/Covariates/PCA/PathwayMatrix/goa_human.gaf_P_matrix.txt.gz"); // File significantTermsFile = new File("/groups/umcg-wijmenga/tmp04/umcg-svandam/GeneNetwork/Data31995Genes05-12-2017/GeneNetwork_V2_01-02-2018/Covariates/PCA/predictions/go_P_predictions_bonSigTerms_alsoInReactome.txt"); // File outputFile = new File("/groups/umcg-wijmenga/tmp04/umcg-svandam/GeneNetwork/Data31995Genes05-12-2017/GeneNetwork_V2_01-02-2018/Covariates/PCA/predictions/go_P_predictions_genePredictability_alsoInReactome.txt"); LinkedHashSet<String> significantTerms = loadSignificantTerms(significantTermsFile); DoubleMatrixDataset<String, String> predictionMatrix = DoubleMatrixDataset .loadDoubleData(predictionMatrixFile.getAbsolutePath()); DoubleMatrixDataset<String, String> annotationMatrix = DoubleMatrixDataset .loadDoubleData(annotationMatrixFile.getAbsolutePath()); DoubleMatrixDataset<String, String> predictionMatrixSignificant = predictionMatrix .viewColSelection(significantTerms); DoubleMatrixDataset<String, String> annotationMatrixSignificant = annotationMatrix .viewColSelection(significantTerms); if (!predictionMatrixSignificant.getColObjects().equals(annotationMatrixSignificant.getColObjects())) { System.err.println("Differnce in terms"); return; } if (!predictionMatrixSignificant.getRowObjects().equals(annotationMatrixSignificant.getRowObjects())) { System.err.println("Differnce in genes"); return; } MannWhitneyUTest2 uTest = new MannWhitneyUTest2(); Kurtosis kurtosisCalculator = new Kurtosis(); Skewness skewnessCalculator = new Skewness(); Mean annotatedMeanCalculator = new Mean(); Mean notAnnotatedMeanCalculator = new Mean(); double[] genePredictabilityZscores = new double[predictionMatrixSignificant.rows()]; int[] pathwayCount = new int[predictionMatrixSignificant.rows()]; double[] geneKurtosis = new double[predictionMatrixSignificant.rows()]; double[] geneSkewness = new double[predictionMatrixSignificant.rows()]; double[] geneAnnotatedMean = new double[predictionMatrixSignificant.rows()]; double[] geneNotAnnotatedMean = new double[predictionMatrixSignificant.rows()]; for (int g = 0; g < predictionMatrixSignificant.rows(); g++) { kurtosisCalculator.clear(); skewnessCalculator.clear(); annotatedMeanCalculator.clear(); notAnnotatedMeanCalculator.clear(); DoubleMatrix1D geneAnnotations = annotationMatrixSignificant.getRow(g); int geneAnnotationCount = geneAnnotations.cardinality(); pathwayCount[g] = geneAnnotationCount; double[] zScoresAnnotatedPathways = new double[geneAnnotationCount]; double[] zScoresOtherPathways = new double[annotationMatrixSignificant.columns() - geneAnnotationCount]; int x = 0; int y = 0; for (int p = 0; p < geneAnnotations.size(); p++) { double z = predictionMatrixSignificant.getElementQuick(g, p); if (geneAnnotations.getQuick(p) != 0) { annotatedMeanCalculator.increment(z); zScoresAnnotatedPathways[x++] = z; } else { notAnnotatedMeanCalculator.increment(z); zScoresOtherPathways[y++] = z; } kurtosisCalculator.increment(z); skewnessCalculator.increment(z); } if (geneAnnotationCount >= 10) { uTest.setData(zScoresOtherPathways, zScoresAnnotatedPathways); genePredictabilityZscores[g] = uTest.getZ(); } else { genePredictabilityZscores[g] = Double.NaN; } geneKurtosis[g] = kurtosisCalculator.getResult(); geneSkewness[g] = skewnessCalculator.getResult(); geneAnnotatedMean[g] = annotatedMeanCalculator.getResult(); geneNotAnnotatedMean[g] = notAnnotatedMeanCalculator.getResult(); } CSVWriter writer = new CSVWriter(new FileWriter(outputFile), '\t', '\0', '\0', "\n"); String[] outputLine = new String[7]; int c = 0; outputLine[c++] = "Gene"; outputLine[c++] = "Z-score"; outputLine[c++] = "Skewness"; outputLine[c++] = "Kurtosis"; outputLine[c++] = "MeanNotAnnotated"; outputLine[c++] = "MeanAnnotated"; outputLine[c++] = "Annoted_pathways"; writer.writeNext(outputLine); ArrayList<String> geneNames = predictionMatrixSignificant.getRowObjects(); for (int g = 0; g < predictionMatrixSignificant.rows(); g++) { c = 0; outputLine[c++] = geneNames.get(g); outputLine[c++] = String.valueOf(genePredictabilityZscores[g]); outputLine[c++] = String.valueOf(geneSkewness[g]); outputLine[c++] = String.valueOf(geneKurtosis[g]); outputLine[c++] = String.valueOf(geneNotAnnotatedMean[g]); outputLine[c++] = String.valueOf(geneAnnotatedMean[g]); outputLine[c++] = String.valueOf(pathwayCount[g]); writer.writeNext(outputLine); } writer.close(); }
From source file:org.eclipse.dataset.Stats.java
static private void calculateHigherMoments(final AbstractDataset a, final boolean ignoreNaNs) { final int is = a.getElementsPerItem(); final IndexIterator iter = a.getIterator(); if (is == 1) { Skewness s = new Skewness(); Kurtosis k = new Kurtosis(); if (ignoreNaNs) { while (iter.hasNext()) { final double x = a.getElementDoubleAbs(iter.index); if (Double.isNaN(x)) continue; s.increment(x);// ww w . jav a 2 s . co m k.increment(x); } } else { while (iter.hasNext()) { final double x = a.getElementDoubleAbs(iter.index); s.increment(x); k.increment(x); } } a.setStoredValue(AbstractDataset.storeName(ignoreNaNs, STORE_SKEWNESS), s.getResult()); a.setStoredValue(AbstractDataset.storeName(ignoreNaNs, STORE_KURTOSIS), k.getResult()); } else { final Skewness[] s = new Skewness[is]; final Kurtosis[] k = new Kurtosis[is]; for (int j = 0; j < is; j++) { s[j] = new Skewness(); k[j] = new Kurtosis(); } if (ignoreNaNs) { while (iter.hasNext()) { boolean skip = false; for (int j = 0; j < is; j++) { if (Double.isNaN(a.getElementDoubleAbs(iter.index + j))) { skip = true; break; } } if (!skip) for (int j = 0; j < is; j++) { final double val = a.getElementDoubleAbs(iter.index + j); s[j].increment(val); k[j].increment(val); } } } else { while (iter.hasNext()) { for (int j = 0; j < is; j++) { final double val = a.getElementDoubleAbs(iter.index + j); s[j].increment(val); k[j].increment(val); } } } final double[] ts = new double[is]; final double[] tk = new double[is]; for (int j = 0; j < is; j++) { ts[j] = s[j].getResult(); tk[j] = k[j].getResult(); } a.setStoredValue(AbstractDataset.storeName(ignoreNaNs, STORE_SKEWNESS), ts); a.setStoredValue(AbstractDataset.storeName(ignoreNaNs, STORE_KURTOSIS), tk); } }
From source file:org.eclipse.dataset.Stats.java
static private void calculateHigherMoments(final AbstractDataset a, final boolean ignoreNaNs, final int axis) { final int rank = a.getRank(); final int is = a.getElementsPerItem(); final int[] oshape = a.getShape(); final int alen = oshape[axis]; oshape[axis] = 1;/*from www . j a v a 2s . c o m*/ final int[] nshape = AbstractDataset.squeezeShape(oshape, false); final Dataset sk; final Dataset ku; if (is == 1) { sk = new DoubleDataset(nshape); ku = new DoubleDataset(nshape); final IndexIterator qiter = sk.getIterator(true); final int[] qpos = qiter.getPos(); final int[] spos = oshape; while (qiter.hasNext()) { int i = 0; for (; i < axis; i++) { spos[i] = qpos[i]; } spos[i++] = 0; for (; i < rank; i++) { spos[i] = qpos[i - 1]; } Skewness s = new Skewness(); Kurtosis k = new Kurtosis(); if (ignoreNaNs) { for (int j = 0; j < alen; j++) { spos[axis] = j; final double val = a.getDouble(spos); if (Double.isNaN(val)) continue; s.increment(val); k.increment(val); } } else { for (int j = 0; j < alen; j++) { spos[axis] = j; final double val = a.getDouble(spos); s.increment(val); k.increment(val); } } sk.set(s.getResult(), spos); ku.set(k.getResult(), spos); } } else { sk = new CompoundDoubleDataset(is, nshape); ku = new CompoundDoubleDataset(is, nshape); final IndexIterator qiter = sk.getIterator(true); final int[] qpos = qiter.getPos(); final int[] spos = oshape; final Skewness[] s = new Skewness[is]; final Kurtosis[] k = new Kurtosis[is]; final double[] ts = new double[is]; final double[] tk = new double[is]; for (int j = 0; j < is; j++) { s[j] = new Skewness(); k[j] = new Kurtosis(); } while (qiter.hasNext()) { int i = 0; for (; i < axis; i++) { spos[i] = qpos[i]; } spos[i++] = 0; for (; i < rank; i++) { spos[i] = qpos[i - 1]; } for (int j = 0; j < is; j++) { s[j].clear(); k[j].clear(); } int index = a.get1DIndex(spos); if (ignoreNaNs) { boolean skip = false; for (int j = 0; j < is; j++) { if (Double.isNaN(a.getElementDoubleAbs(index + j))) { skip = true; break; } } if (!skip) for (int j = 0; j < is; j++) { final double val = a.getElementDoubleAbs(index + j); s[j].increment(val); k[j].increment(val); } } else { for (int j = 0; j < is; j++) { final double val = a.getElementDoubleAbs(index + j); s[j].increment(val); k[j].increment(val); } } for (int j = 0; j < is; j++) { ts[j] = s[j].getResult(); tk[j] = k[j].getResult(); } sk.set(ts, spos); ku.set(tk, spos); } } a.setStoredValue(AbstractDataset.storeName(ignoreNaNs, STORE_SKEWNESS + "-" + axis), sk); a.setStoredValue(AbstractDataset.storeName(ignoreNaNs, STORE_KURTOSIS + "-" + axis), ku); }
From source file:org.eclipse.january.dataset.Stats.java
static private HigherStatisticsImpl<?> calculateHigherMoments(final Dataset a, final boolean ignoreNaNs, final boolean ignoreInfs) { final int is = a.getElementsPerItem(); final IndexIterator iter = a.getIterator(); if (is == 1) { Skewness s = new Skewness(); Kurtosis k = new Kurtosis(); if (ignoreNaNs) { while (iter.hasNext()) { final double x = a.getElementDoubleAbs(iter.index); if (Double.isNaN(x)) continue; s.increment(x);//w w w . ja va 2 s .co m k.increment(x); } } else { while (iter.hasNext()) { final double x = a.getElementDoubleAbs(iter.index); s.increment(x); k.increment(x); } } HigherStatisticsImpl<Double> stats = new HigherStatisticsImpl<Double>(); stats.skewness = s.getResult(); stats.kurtosis = k.getResult(); return stats; } final Skewness[] s = new Skewness[is]; final Kurtosis[] k = new Kurtosis[is]; for (int j = 0; j < is; j++) { s[j] = new Skewness(); k[j] = new Kurtosis(); } if (ignoreNaNs) { while (iter.hasNext()) { boolean skip = false; for (int j = 0; j < is; j++) { if (Double.isNaN(a.getElementDoubleAbs(iter.index + j))) { skip = true; break; } } if (!skip) for (int j = 0; j < is; j++) { final double val = a.getElementDoubleAbs(iter.index + j); s[j].increment(val); k[j].increment(val); } } } else { while (iter.hasNext()) { for (int j = 0; j < is; j++) { final double val = a.getElementDoubleAbs(iter.index + j); s[j].increment(val); k[j].increment(val); } } } final double[] ts = new double[is]; final double[] tk = new double[is]; for (int j = 0; j < is; j++) { ts[j] = s[j].getResult(); tk[j] = k[j].getResult(); } HigherStatisticsImpl<double[]> stats = new HigherStatisticsImpl<double[]>(); stats.skewness = ts; stats.kurtosis = tk; return stats; }
From source file:org.eclipse.january.dataset.Stats.java
static private HigherStatisticsImpl<?> calculateHigherMoments(final Dataset a, final boolean ignoreNaNs, final boolean ignoreInfs, final int axis) { final int rank = a.getRank(); final int is = a.getElementsPerItem(); final int[] oshape = a.getShape(); final int alen = oshape[axis]; oshape[axis] = 1;/*from w w w .j a va 2 s. c o m*/ final int[] nshape = ShapeUtils.squeezeShape(oshape, false); final Dataset sk; final Dataset ku; HigherStatisticsImpl<?> stats = null; if (is == 1) { if (stats == null) { stats = new HigherStatisticsImpl<Double>(); a.setMetadata(stats); } sk = DatasetFactory.zeros(DoubleDataset.class, nshape); ku = DatasetFactory.zeros(DoubleDataset.class, nshape); final IndexIterator qiter = sk.getIterator(true); final int[] qpos = qiter.getPos(); final int[] spos = oshape; while (qiter.hasNext()) { int i = 0; for (; i < axis; i++) { spos[i] = qpos[i]; } spos[i++] = 0; for (; i < rank; i++) { spos[i] = qpos[i - 1]; } Skewness s = new Skewness(); Kurtosis k = new Kurtosis(); if (ignoreNaNs) { for (int j = 0; j < alen; j++) { spos[axis] = j; final double val = a.getDouble(spos); if (Double.isNaN(val)) continue; s.increment(val); k.increment(val); } } else { for (int j = 0; j < alen; j++) { spos[axis] = j; final double val = a.getDouble(spos); s.increment(val); k.increment(val); } } sk.set(s.getResult(), qpos); ku.set(k.getResult(), qpos); } } else { if (stats == null) { stats = new HigherStatisticsImpl<double[]>(); a.setMetadata(stats); } sk = DatasetFactory.zeros(is, CompoundDoubleDataset.class, nshape); ku = DatasetFactory.zeros(is, CompoundDoubleDataset.class, nshape); final IndexIterator qiter = sk.getIterator(true); final int[] qpos = qiter.getPos(); final int[] spos = oshape; final Skewness[] s = new Skewness[is]; final Kurtosis[] k = new Kurtosis[is]; final double[] ts = new double[is]; final double[] tk = new double[is]; for (int j = 0; j < is; j++) { s[j] = new Skewness(); k[j] = new Kurtosis(); } while (qiter.hasNext()) { int i = 0; for (; i < axis; i++) { spos[i] = qpos[i]; } spos[i++] = 0; for (; i < rank; i++) { spos[i] = qpos[i - 1]; } for (int j = 0; j < is; j++) { s[j].clear(); k[j].clear(); } int index = a.get1DIndex(spos); if (ignoreNaNs) { boolean skip = false; for (int j = 0; j < is; j++) { if (Double.isNaN(a.getElementDoubleAbs(index + j))) { skip = true; break; } } if (!skip) for (int j = 0; j < is; j++) { final double val = a.getElementDoubleAbs(index + j); s[j].increment(val); k[j].increment(val); } } else { for (int j = 0; j < is; j++) { final double val = a.getElementDoubleAbs(index + j); s[j].increment(val); k[j].increment(val); } } for (int j = 0; j < is; j++) { ts[j] = s[j].getResult(); tk[j] = k[j].getResult(); } sk.set(ts, qpos); ku.set(tk, qpos); } } stats.setSkewness(axis, sk); stats.setKurtosis(axis, ku); return stats; }