List of usage examples for org.apache.commons.math3.stat.correlation Covariance Covariance
public Covariance(RealMatrix matrix) throws MathIllegalArgumentException
From source file:imagingbook.lib.math.Statistics.java
public static double[][] covarianceMatrix(double[][] samples) { Covariance sm = new Covariance(samples); return sm.getCovarianceMatrix().getData(); }
From source file:com.idylwood.Statistics.java
public static double[][] covariance(List<YahooFinance.Pair> pairs) { Covariance cov = new Covariance(data(pairs)); return cov.getCovarianceMatrix().getData(); }
From source file:imagingbook.lib.math.MahalanobisDistance.java
public MahalanobisDistance(double[][] samples) { N = samples.length;/* w ww . java 2s. c o m*/ K = samples[0].length; Covariance cov = new Covariance(samples); RealMatrix S = cov.getCovarianceMatrix(); // condition the covariance matrix to avoid singularity // (add a small quantity to the diagonal) for (int i = 0; i < K; i++) { S.addToEntry(i, i, 0.0001); } // get the inverse covariance matrix RealMatrix iSM = new LUDecomposition(S).getSolver().getInverse(); iS = iSM.getData(); }
From source file:com.itemanalysis.psychometrics.mixture.MvNormalComponentDistributionTest.java
/** * This test uses the iris data from R. It is a three group mixture model. * Results confirmed with the following R code: * * library(mixtools)//from w w w.ja va2s .com * mvnormalmixEM(iris[,-5], arbvar=TRUE, k=3) * */ // @Test public void mixModel1Test() { System.out.println("Mixture model test with Iris Data"); MixtureModelTestData testData = new MixtureModelTestData(); BlockRealMatrix allData = new BlockRealMatrix(testData.getMixtureData3()); BlockRealMatrix group1Data = allData.getSubMatrix(0, 49, 0, 3); BlockRealMatrix group2Data = allData.getSubMatrix(50, 99, 0, 3); BlockRealMatrix group3Data = allData.getSubMatrix(100, 149, 0, 3); BlockRealMatrix data = allData.getSubMatrix(0, 149, 0, 3); // System.out.println(Arrays.toString(allData.getRow(0))); VectorialMean trueMean1 = new VectorialMean(4); VectorialMean trueMean2 = new VectorialMean(4); VectorialMean trueMean3 = new VectorialMean(4); Covariance trueCov1 = new Covariance(group1Data); Covariance trueCov2 = new Covariance(group2Data); Covariance trueCov3 = new Covariance(group3Data); for (int i = 0; i < group1Data.getRowDimension(); i++) { trueMean1.increment(group1Data.getRow(i)); } for (int i = 0; i < group2Data.getRowDimension(); i++) { trueMean2.increment(group2Data.getRow(i)); } for (int i = 0; i < group3Data.getRowDimension(); i++) { trueMean3.increment(group3Data.getRow(i)); } double[] trueMeanGroup1 = trueMean1.getResult(); double[] trueMeanGroup2 = trueMean2.getResult(); double[] trueMeanGroup3 = trueMean3.getResult(); int startGroup = 3; int endGroup = 3; int maxIter = 1000; double converge = 1e-8; int starts = 100; MvNormalMixtureModel model = null; model = new MvNormalMixtureModel(data, endGroup); model.setModelConstraints(false, false, false, false); model.setEmOptions(maxIter, converge, starts); model.call(); System.out.println(" Testing convergence..."); assertTrue("Convergence test", model.converged()); double[] pi = new double[3]; pi[0] = model.getMixingProportion(0); pi[1] = model.getMixingProportion(1); pi[2] = model.getMixingProportion(2); //check mixing proportions // System.out.println(" Testing proportions..."); // assertEquals("Proportion 1 test: ", 0.40, pi[index[0]], 5e-2); // assertEquals("Proportion 2 test: ", 0.60, pi[index[1]], 5e-2); double[] estMean1 = model.getMean(0); double[] estMean2 = model.getMean(1); double[] estMean3 = model.getMean(2); double[][] estCov1 = model.getCov(0); double[][] estCov2 = model.getCov(1); double[][] estCov3 = model.getCov(2); double[][] trueCov1Array = trueCov1.getCovarianceMatrix().getData(); double[][] trueCov2Array = trueCov2.getCovarianceMatrix().getData(); double[][] trueCov3Array = trueCov3.getCovarianceMatrix().getData(); System.out.println(model.printResults()); /** * Test means */ // System.out.println(" Testing means..."); // for(int i=0;i<3;i++){ // assertEquals("Group 1 mean test", trueMeanGroup1[i], estMean1[i], 5e-2); // assertEquals("Group 2 mean test", trueMeanGroup2[i], estMean2[i], 5e-2); // assertEquals("Group 3 mean test", trueMeanGroup3[i], estMean3[i], 5e-2); // } // // /** // * Test Covariance matrix // */ // System.out.println(" Testing covariances..."); // for(int i=0;i<3;i++){ // for(int j=0;j<3;j++){ // assertEquals("Cov[" +i + "," + j+"] test:", trueCov1Array[i][j], estCov1[i][j], 5e-2); // } // } // // for(int i=0;i<3;i++){ // for(int j=0;j<3;j++){ // assertEquals("Cov[" +i + "," + j+"] test:", trueCov2Array[i][j], estCov2[i][j], 5e-2); // } // } // // for(int i=0;i<3;i++){ // for(int j=0;j<3;j++){ // assertEquals("Cov[" +i + "," + j+"] test:", trueCov2Array[i][j], estCov2[i][j], 5e-2); // } // } }
From source file:com.itemanalysis.psychometrics.mixture.MvNormalMixtureModel.java
public void estimateCov(RealMatrix x) { Covariance cv = new Covariance(x); initialCovariance = cv.getCovarianceMatrix(); }
From source file:iDynoOptimizer.MOEAFramework26.src.org.moeaframework.core.operator.real.AdaptiveMetropolis.java
@Override public Solution[] evolve(Solution[] parents) { int k = parents.length; int n = parents[0].getNumberOfVariables(); RealMatrix x = new Array2DRowRealMatrix(k, n); for (int i = 0; i < k; i++) { x.setRow(i, EncodingUtils.getReal(parents[i])); }//w w w .j a va2s. co m try { //perform Cholesky factorization and get the upper triangular matrix double jumpRate = Math.pow(jumpRateCoefficient / Math.sqrt(n), 2.0); RealMatrix chol = new CholeskyDecomposition( new Covariance(x.scalarMultiply(jumpRate)).getCovarianceMatrix()).getLT(); //produce the offspring Solution[] offspring = new Solution[numberOfOffspring]; for (int i = 0; i < numberOfOffspring; i++) { Solution child = parents[PRNG.nextInt(parents.length)].copy(); //apply adaptive metropolis step to solution RealVector muC = new ArrayRealVector(EncodingUtils.getReal(child)); RealVector ru = new ArrayRealVector(n); for (int j = 0; j < n; j++) { ru.setEntry(j, PRNG.nextGaussian()); } double[] variables = muC.add(chol.preMultiply(ru)).toArray(); //assign variables back to solution for (int j = 0; j < n; j++) { RealVariable variable = (RealVariable) child.getVariable(j); double value = variables[j]; if (value < variable.getLowerBound()) { value = variable.getLowerBound(); } else if (value > variable.getUpperBound()) { value = variable.getUpperBound(); } variable.setValue(value); } offspring[i] = child; } return offspring; } catch (Exception e) { return new Solution[0]; } }
From source file:edu.cmu.sphinx.speakerid.SpeakerIdentification.java
/** * @param mat/* w w w . j a va 2 s .c o m*/ * A matrix with feature vectors as rows. * @return Returns the BICValue of the Gaussian model that approximates the * the feature vectors data samples */ public static double getBICValue(Array2DRowRealMatrix mat) { double ret = 0; EigenDecomposition ed = new EigenDecomposition(new Covariance(mat).getCovarianceMatrix()); double[] re = ed.getRealEigenvalues(); for (int i = 0; i < re.length; i++) ret += Math.log(re[i]); return ret * (mat.getRowDimension() / 2); }
From source file:eagle.security.userprofile.model.eigen.UserProfileEigenModeler.java
private void computeCovarianceAndSVD(RealMatrix inputMat, int containsLowVariantCol) { int finalMatrixRow = 0; int finalMatrixCol = 0; LOG.info("containsLowVariantCol size: " + containsLowVariantCol); int colDimension = (inputMat.getColumnDimension() - containsLowVariantCol); try {/*from w w w . j a v a2s.c o m*/ finalMatrixWithoutLowVariantCmds = new Array2DRowRealMatrix(inputMat.getRowDimension(), colDimension); } catch (NotStrictlyPositiveException e) { LOG.error(String.format("Failed to build matrix [rowDimension:%s, columnDimension: %s]", inputMat.getRowDimension(), colDimension), e); throw e; } for (int i = 0; i < inputMat.getRowDimension(); i++) { for (int j = 0; j < inputMat.getColumnDimension(); j++) { if (!statistics[j].isLowVariant()) { finalMatrixWithoutLowVariantCmds.setEntry(finalMatrixRow, finalMatrixCol, inputMat.getEntry(i, j)); finalMatrixCol++; } } finalMatrixCol = 0; finalMatrixRow++; } Covariance cov; try { cov = new Covariance(finalMatrixWithoutLowVariantCmds.getData()); } catch (Exception ex) { throw new IllegalArgumentException(String.format("Failed to create covariance from matrix [ %s x %s ]", finalMatrixWithoutLowVariantCmds.getRowDimension(), finalMatrixWithoutLowVariantCmds.getColumnDimension()), ex); } covarianceMatrix = cov.getCovarianceMatrix(); SingularValueDecomposition svd = new SingularValueDecomposition(covarianceMatrix); diagonalMatrix = svd.getS(); uMatrix = svd.getU(); vMatrix = svd.getV(); }
From source file:com.itemanalysis.psychometrics.mixture.MvNormalComponentDistributionTest.java
/** * This test computes mean vector and covariance matrix for a single component mixture model. * The estimates should be the same as the sample mean and covariance. *///from www .j a v a 2 s.c om // @Test public void mixModel2Test() { System.out.println("Mixture model test with Two variables and two components"); /** * x contains 2,000 cases and three variables. * The first two variables are continuous data. The last variable is a group indicator variable */ double[][] x = testData.getMixtureData2(); BlockRealMatrix allData = new BlockRealMatrix(x); int nrow = allData.getRowDimension(); BlockRealMatrix group1Data = allData.getSubMatrix(0, 799, 0, 1); BlockRealMatrix group2Data = allData.getSubMatrix(800, 1999, 0, 1); VectorialMean trueMean1 = new VectorialMean(2); VectorialMean trueMean2 = new VectorialMean(2); Covariance trueCov1 = new Covariance(group1Data); Covariance trueCov2 = new Covariance(group2Data); for (int i = 0; i < group1Data.getRowDimension(); i++) { trueMean1.increment(group1Data.getRow(i)); } for (int i = 0; i < group2Data.getRowDimension(); i++) { trueMean2.increment(group2Data.getRow(i)); } double[] trueMeanGroup1 = trueMean1.getResult(); double[] trueMeanGroup2 = trueMean2.getResult(); /** * actual data in first two columns */ BlockRealMatrix data = allData.getSubMatrix(0, nrow - 1, 0, 1); //run two group mixture model int startGroup = 2; int endGroup = 2; int maxIter = 500; double converge = 1e-12; int starts = 100; MvNormalMixtureModel model = null; model = new MvNormalMixtureModel(data, endGroup); model.setModelConstraints(true, true, true, true); model.setEmOptions(maxIter, converge, starts); model.call(); // System.out.println(model.printResults()); double[] pi = new double[2]; pi[0] = model.getMixingProportion(0); pi[1] = model.getMixingProportion(1); int[] index = { 0, 1 }; //group 1 should be the smaller group. If not, reverse indexes. if (pi[0] > pi[1]) { index[0] = 1; index[1] = 0; } //check convergence System.out.println(" Testing convergence..."); assertTrue("Convergence test", model.converged()); //check mixing proportions System.out.println(" Testing proportions..."); assertEquals("Proportion 1 test: ", 0.40, pi[index[0]], 5e-2); assertEquals("Proportion 2 test: ", 0.60, pi[index[1]], 5e-2); //get estimated parameters double[] estMeanGroup1 = model.getMean(index[0]); double[] estMeanGroup2 = model.getMean(index[1]); double[][] estCovGroup1 = model.getCov(index[0]); double[][] estCovGroup2 = model.getCov(index[1]); //check mean vector for each group System.out.println(" Testing means..."); assertEquals("Mean (1,1) test", trueMeanGroup1[index[0]], estMeanGroup1[index[0]], 5e-2); assertEquals("Mean (1,2) test", trueMeanGroup1[index[1]], estMeanGroup1[index[1]], 5e-2); assertEquals("Mean (2,1) test", trueMeanGroup2[index[0]], estMeanGroup2[index[0]], 5e-2); assertEquals("Mean (2,2) test", trueMeanGroup2[index[1]], estMeanGroup2[index[1]], 5e-2); //check covariance matrix for group 1 System.out.println(" Testing covariances..."); double[][] trueCovGroup1 = trueCov1.getCovarianceMatrix().getData(); for (int i = 0; i < 2; i++) { for (int j = 0; j < 2; j++) { assertEquals("CovGroup1[" + i + "," + j + "] test:", trueCovGroup1[i][j], estCovGroup1[i][j], 5e-2); } } //check covariance matrix for group 2 double[][] trueCovGroup2 = trueCov2.getCovarianceMatrix().getData(); for (int i = 0; i < 2; i++) { for (int j = 0; j < 2; j++) { assertEquals("CovGroup2[" + i + "," + j + "] test:", trueCovGroup2[i][j], estCovGroup2[i][j], 5e-2); } } }
From source file:edu.ucla.stat.SOCR.analyses.gui.PrincipalComponentAnalysis.java
/**This method defines the specific statistical Analysis to be carried our on the user specified data. ANOVA is done in this case. */ public void doAnalysis() { if (dataTable.isEditing()) dataTable.getCellEditor().stopCellEditing(); if (!hasExample) { JOptionPane.showMessageDialog(this, DATA_MISSING_MESSAGE); return;/* w w w. j av a 2 s. c o m*/ } Data data = new Data(); String firstMessage = "Would you like to use all the data columns in this Principal Components Analysis?"; String title = "SOCR - Principal Components Analysis"; String secondMessage = "Please enter the column numbers (seperated by comma) that you would like to use."; String columnNumbers = ""; int reply = JOptionPane.showConfirmDialog(null, firstMessage, title, JOptionPane.YES_NO_OPTION); if (reply == JOptionPane.YES_OPTION) { String cellValue = null; int originalRow = 0; for (int k = 0; k < dataTable.getRowCount(); k++) { cellValue = ((String) dataTable.getValueAt(k, 0)); if (cellValue != null && !cellValue.equals("")) { originalRow++; } } cellValue = null; int originalColumn = 0; for (int k = 0; k < dataTable.getColumnCount(); k++) { cellValue = ((String) dataTable.getValueAt(0, k)); if (cellValue != null && !cellValue.equals("")) { originalColumn++; } } dataRow = originalRow; dataColumn = originalColumn; String PCA_Data1[][] = new String[originalRow][originalColumn]; double PCA_Data[][] = new double[originalRow][originalColumn]; for (int k = 0; k < originalRow; k++) for (int j = 0; j < originalColumn; j++) { if (dataTable.getValueAt(k, j) != null && !dataTable.getValueAt(k, j).equals("")) { PCA_Data1[k][j] = (String) dataTable.getValueAt(k, j); PCA_Data[k][j] = Double.parseDouble(PCA_Data1[k][j]); } } double PCA_Adjusted_Data[][] = new double[originalRow][originalColumn]; double column_Total = 0; double column_Mean = 0; for (int j = 0; j < originalColumn; j++) for (int k = 0; k < originalRow; k++) { column_Total += PCA_Data[k][j]; if (k == (originalRow - 1)) { column_Mean = column_Total / originalRow; for (int p = 0; p < originalRow; p++) { PCA_Adjusted_Data[p][j] = PCA_Data[p][j] - column_Mean; } column_Total = 0; column_Mean = 0; } } Covariance cov = new Covariance(PCA_Adjusted_Data); RealMatrix matrix = cov.getCovarianceMatrix(); EigenDecomposition eigenDecomp = new EigenDecomposition(matrix, 0); storedData = eigenDecomp; RealMatrix eigenvectorMatrix = eigenDecomp.getV(); EValueArray = eigenDecomp.getRealEigenvalues(); eigenvectorMatrix = eigenvectorMatrix.transpose(); double eigenvectorArray[][] = eigenvectorMatrix.getData(); /*for (int j = 0; j < 3; j++) for (int k = 0; k < 3; k++) { System.out.println(eigenvectorArray[j][k] + " "); } */ Matrix matrix1 = new Matrix(eigenvectorArray); Matrix matrix2 = new Matrix(PCA_Adjusted_Data); matrix2 = matrix2.transpose(); Matrix finalProduct = matrix1.times(matrix2); finalProduct = finalProduct.transpose(); double finalArray[][] = finalProduct.getArrayCopy(); for (int j = 0; j < originalRow; j++) for (int k = 0; k < originalColumn; k++) { PCATable.setValueAt(finalArray[j][k], j, k); } xData = new double[originalRow]; yData = new double[originalRow]; for (int i = 0; i < originalRow; i++) { xData[i] = finalArray[i][0]; } for (int i = 0; i < originalRow; i++) { yData[i] = finalArray[i][1]; } dependentHeader = "C1"; independentHeader = "C2"; } else { // start here try { columnNumbers = JOptionPane.showInputDialog(secondMessage); } catch (Exception e) { } String columnNumbersFinal = "," + columnNumbers.replaceAll("\\s", "") + ","; Vector<Integer> locationOfComma = new Vector<Integer>(100); for (int i = 0; i < columnNumbersFinal.length(); i++) { char d = columnNumbersFinal.charAt(i); if (d == ',') locationOfComma.add(i); } Vector<Integer> vector = new Vector<Integer>(100); // vector containing column selected numbers for (int i = 0; i < locationOfComma.size() - 1; i++) { String temp = columnNumbersFinal.substring(locationOfComma.get(i) + 1, locationOfComma.get(i + 1)); if (temp == "") continue; vector.add((Integer.parseInt(temp) - 1)); } dependentHeader = "C" + (vector.get(0) + 1); independentHeader = "C" + (vector.get(1) + 1); // System.out.println("Vector size is: " + vector.size() + "\n"); String cellValue = null; int originalRow = 0; for (int k = 0; k < dataTable.getRowCount(); k++) { cellValue = ((String) dataTable.getValueAt(k, 0)); if (cellValue != null && !cellValue.equals("")) { originalRow++; } } int originalColumn = vector.size(); dataRow = originalRow; dataColumn = originalColumn; String PCA_Data1[][] = new String[originalRow][originalColumn]; double PCA_Data[][] = new double[originalRow][originalColumn]; for (int k = 0; k < originalRow; k++) for (int j = 0; j < originalColumn; j++) { if (dataTable.getValueAt(k, vector.get(j)) != null && !dataTable.getValueAt(k, vector.get(j)).equals("")) { PCA_Data1[k][j] = (String) dataTable.getValueAt(k, vector.get(j)); PCA_Data[k][j] = Double.parseDouble(PCA_Data1[k][j]); } } double PCA_Adjusted_Data[][] = new double[originalRow][originalColumn]; double column_Total = 0; double column_Mean = 0; for (int j = 0; j < originalColumn; j++) for (int k = 0; k < originalRow; k++) { column_Total += PCA_Data[k][j]; if (k == (originalRow - 1)) { column_Mean = column_Total / originalRow; for (int p = 0; p < originalRow; p++) { PCA_Adjusted_Data[p][j] = PCA_Data[p][j] - column_Mean; } column_Total = 0; column_Mean = 0; } } Covariance cov = new Covariance(PCA_Adjusted_Data); RealMatrix matrix = cov.getCovarianceMatrix(); EigenDecomposition eigenDecomp = new EigenDecomposition(matrix, 0); storedData = eigenDecomp; RealMatrix eigenvectorMatrix = eigenDecomp.getV(); EValueArray = eigenDecomp.getRealEigenvalues(); // added eigenvectorMatrix = eigenvectorMatrix.transpose(); double eigenvectorArray[][] = eigenvectorMatrix.getData(); /*for (int j = 0; j < 3; j++) for (int k = 0; k < 3; k++) { System.out.println(eigenvectorArray[j][k] + " "); } */ Matrix matrix1 = new Matrix(eigenvectorArray); Matrix matrix2 = new Matrix(PCA_Adjusted_Data); matrix2 = matrix2.transpose(); Matrix finalProduct = matrix1.times(matrix2); finalProduct = finalProduct.transpose(); double finalArray[][] = finalProduct.getArrayCopy(); /* for (int j = 0; j < dataTable.getColumnCount(); j++) for (int k = 0; k < dataTable.getRowCount(); k++) { System.out.println(finalArray[j][k] + " "); }*/ for (int j = 0; j < originalRow; j++) for (int k = 0; k < originalColumn; k++) { PCATable.setValueAt(finalArray[j][k], j, k); } xData = new double[originalRow]; yData = new double[originalRow]; for (int i = 0; i < originalRow; i++) { xData[i] = finalArray[i][0]; } for (int i = 0; i < originalRow; i++) { yData[i] = finalArray[i][1]; } } map = new HashMap<Double, double[]>(); for (int i = 0; i < dataColumn; i++) { map.put(EValueArray[i], storedData.getEigenvector(i).toArray()); } Arrays.sort(EValueArray); xData1 = new double[EValueArray.length]; // for Scree Plot yData1 = new double[EValueArray.length]; for (int i = 0; i < EValueArray.length; i++) { xData1[i] = i + 1; } for (int i = 0; i < EValueArray.length; i++) { yData1[i] = EValueArray[i]; } for (int i = 0; i < yData1.length / 2; i++) { double temp = yData1[i]; yData1[i] = yData1[yData1.length - i - 1]; yData1[yData1.length - i - 1] = temp; } for (int i = 0; i < xData1.length; i++) { System.out.println("xData1 contains: " + xData1[i] + "\n"); } for (int i = 0; i < yData1.length; i++) { System.out.println("yData1 contains: " + yData1[i] + "\n"); } for (int i = 0; i < EValueArray.length / 2; i++) { double temp = EValueArray[i]; EValueArray[i] = EValueArray[EValueArray.length - i - 1]; EValueArray[EValueArray.length - i - 1] = temp; } resultPanelTextArea.append( "Click on \"PCA RESULT\" panel to view the transformed data (Eigenvector Transposed * Adjusted Data Transposed)"); resultPanelTextArea.append("\n\nThe real eigenvalues (in descending order) are: \n\n"); resultPanelTextArea.append("" + round(EValueArray[0], 3)); for (int i = 1; i < EValueArray.length; i++) { resultPanelTextArea.append("\n" + round(EValueArray[i], 3)); } resultPanelTextArea.append("\n\nThe corresponding eigenvectors (in columns) are: \n\n"); double temp[] = new double[100]; for (int j = 0; j < temp.length; j++) for (int i = 0; i < EValueArray.length; i++) { temp = map.get(EValueArray[i]); resultPanelTextArea.append("" + round(temp[j], 3) + "\t"); if (i == EValueArray.length - 1) { resultPanelTextArea.append("\n"); } } doGraph(); }