List of usage examples for org.apache.commons.math3.linear Array2DRowRealMatrix Array2DRowRealMatrix
public Array2DRowRealMatrix(final double[][] d, final boolean copyArray) throws DimensionMismatchException, NoDataException, NullArgumentException
From source file:eagle.security.userprofile.model.eigen.UserProfileEigenModeler.java
private RealMatrix normalizeData(RealMatrix matrix) { RealMatrix normalizedData = new Array2DRowRealMatrix(matrix.getRowDimension(), matrix.getColumnDimension()); for (int i = 0; i < matrix.getRowDimension(); i++) { for (int j = 0; j < matrix.getColumnDimension(); j++) { // TODO: statistics[j].getStddev() == 0 what should the value be if stddev is o? double value = statistics[j].getStddev() == 0 ? 0 : (matrix.getEntry(i, j) - statistics[j].getMean()) / statistics[j].getStddev(); normalizedData.setEntry(i, j, value); }//from ww w . j a v a 2 s. c o m } return normalizedData; }
From source file:com.clust4j.algo.BallTree.java
@Override final BallTree newInstance(double[][] arr, int leaf, DistanceMetric dist, Loggable logger) { return new BallTree(new Array2DRowRealMatrix(arr, false), leaf, dist, logger); }
From source file:edu.stanford.cfuller.colocalization3d.correction.Correction.java
/** * Construct a new Correction from a set of locations and corrections at those points. * * For each of the matrix parameters, the row index is over objects. * * @param cX a RealMatrix containing the parameters describing the interpolating function centered at each object used for correction in the x dimension. * @param cY a RealMatrix containing the parameters describing the interpolating function centered at each object used for correction in the y dimension. * @param cZ a RealMatrix containing the parameters describing the interpolating function centered at each object used for correction in the z dimension. * @param distanceCutoffs a RealVector containing the distance to the farthest point used to generate each interpolating function. * @param imageObjects the ImageObjects used for correction. * @param referenceChannel the referenceChannel relative to which the other channel was corrected. * @param correctionChannel the channel being corrected. *///from w w w .j av a 2 s .c o m public Correction(RealMatrix cX, RealMatrix cY, RealMatrix cZ, RealVector distanceCutoffs, List<ImageObject> imageObjects, int referenceChannel, int correctionChannel) { this.correctionX = cX; this.correctionY = cY; this.correctionZ = cZ; this.referenceChannel = referenceChannel; this.correctionChannel = correctionChannel; this.positionsForCorrection = new Array2DRowRealMatrix(imageObjects.size(), 3); for (int i = 0; i < this.positionsForCorrection.getRowDimension(); i++) { this.positionsForCorrection.setRowVector(i, imageObjects.get(i).getPositionForChannel(referenceChannel)); } this.distanceCutoffs = distanceCutoffs; }
From source file:edu.cudenver.bios.matrix.DesignEssenceMatrix.java
/** * Expands the essence matrix into full design matrix for power * calculations./*w w w . ja va 2 s . com*/ * * Assumes that row meta data indicates the actual number * of repetitions of that row in the full design * * @return design matrix * @throws IllegalArgumentException */ public RealMatrix getFullDesignMatrix() { // allocate the full design matrix // #rows = total of repetitions for each unique row // #columns = number of columns in design matrix int fullRows = getTotalSampleSize(); int fullColumns = combinedMatrix.getColumnDimension(); Array2DRowRealMatrix fullDesignMatrix = new Array2DRowRealMatrix(fullRows, fullColumns); // copy in the columns // case 1: if predictor in a given column is fixed, the values are copied from // the design matrix // case 2: if the predictor is random, the values are set to a random value from // a normal curve with the mean/variance specified in the column meta data int fullDesignColumn = 0; if (fixedMatrix != null) { for (int col = 0; col < fixedMatrix.getColumnDimension(); col++, fullDesignColumn++) { fillFixedColumn(col, fullDesignColumn, fullDesignMatrix); } } if (randomMatrix != null) { for (int col = 0; col < randomMatrix.getColumnDimension(); col++, fullDesignColumn++) { fillRandomColumn(col, fullDesignColumn, fullDesignMatrix); } } return fullDesignMatrix; }
From source file:com.clust4j.algo.KDTree.java
@Override final KDTree newInstance(double[][] arr, int leaf, DistanceMetric dist, Loggable logger) { return new KDTree(new Array2DRowRealMatrix(arr, false), leaf, dist, logger); }
From source file:edu.stanford.cfuller.imageanalysistools.fitting.DifferentialEvolutionMinimizer.java
/** * Performs a minimization of a function starting with a given population. * /*from ww w. j ava2 s . c om*/ * @param population The population of parameters to start from, one population entry per row, one parameter per column. * @param f The function to be minimized. * @param parameterLowerBounds The lower bounds of each parameter. This must have the same size as the column dimension of the population. Generated parameter values less than these values will be discarded. * @param parameterUpperBounds The upper bounds of each paraemter. This must have the same size as the column dimension of the population. Generated parameter values greater than these values will be discarded. * @param populationSize The size of the population of parameters sets. This must be equal to the row dimension of the population. * @param scaleFactor Factor controlling the scale of crossed over entries during crossover events. * @param maxIterations The maximum number of iterations to allow before returning a result. * @param crossoverFrequency The frequency of crossover from 0 to 1. At any given parameter, this is the probability of initiating a crossover as well as the probability of ending one after it has started. * @param tol Relative function value tolerance controlling termination; if the maximal and minimal population values differ by less than this factor times the maximal value, optimization will terminate. * @return The parameter values at the minimal function value found. */ public RealVector minimizeWithPopulation(RealMatrix population, ObjectiveFunction f, RealVector parameterLowerBounds, RealVector parameterUpperBounds, int populationSize, double scaleFactor, int maxIterations, double crossoverFrequency, double tol) { int numberOfParameters = parameterLowerBounds.getDimension(); double currMinValue = Double.MAX_VALUE; double currMaxValue = -1.0 * Double.MAX_VALUE; int iterationCounter = maxIterations; double mutationProb = 0.01; // int totalIterations =0; RealVector values = new ArrayRealVector(populationSize); boolean[] valuesChanged = new boolean[populationSize]; java.util.Arrays.fill(valuesChanged, true); computeValues(f, population, values, valuesChanged); RealVector newVec = new ArrayRealVector(numberOfParameters); RealMatrix newPopulation = new Array2DRowRealMatrix(populationSize, numberOfParameters); while (iterationCounter > 0) { for (int i = 0; i < populationSize; i++) { int i1 = RandomGenerator.getGenerator().randInt(populationSize); int i2 = RandomGenerator.getGenerator().randInt(populationSize); int i3 = RandomGenerator.getGenerator().randInt(populationSize); newVec.mapMultiplyToSelf(0.0); boolean inBounds = true; boolean isCrossingOver = false; for (int j = 0; j < numberOfParameters; j++) { if ((RandomGenerator.rand() < crossoverFrequency) ^ isCrossingOver) { if (!isCrossingOver) { isCrossingOver = true; } newVec.setEntry(j, scaleFactor * (population.getEntry(i2, j) - population.getEntry(i1, j)) + population.getEntry(i3, j)); } else { if (isCrossingOver) { isCrossingOver = false; } newVec.setEntry(j, population.getEntry(i, j)); } //random 10% range +/- mutation if ((RandomGenerator.rand() < mutationProb)) { double magnitude = 0.2 * (parameterUpperBounds.getEntry(j) - parameterLowerBounds.getEntry(j)); newVec.setEntry(j, newVec.getEntry(j) + (RandomGenerator.rand() - 0.5) * magnitude); } if (newVec.getEntry(j) < parameterLowerBounds.getEntry(j) || newVec.getEntry(j) > parameterUpperBounds.getEntry(j)) { inBounds = false; } } double functionValue = Double.MAX_VALUE; if (inBounds) functionValue = f.evaluate(newVec); //if (inBounds) System.out.printf("in bounds candidate value: %1.2f old value: %1.2f \n", functionValue, values.getEntry(i)); if (functionValue < values.getEntry(i)) { newPopulation.setRowVector(i, newVec); valuesChanged[i] = true; values.setEntry(i, functionValue); } else { newPopulation.setRowVector(i, population.getRowVector(i)); valuesChanged[i] = false; } } population = newPopulation; double tempMinValue = Double.MAX_VALUE; double tempMaxValue = -1.0 * Double.MAX_VALUE; // double averageValue = 0; for (int i = 0; i < values.getDimension(); i++) { double value = values.getEntry(i); // averageValue += value; if (value < tempMinValue) { tempMinValue = value; } if (value > tempMaxValue) { tempMaxValue = value; } } // averageValue /= values.getDimension(); currMinValue = tempMinValue; currMaxValue = tempMaxValue; //LoggingUtilities.getLogger().info("Iteration counter: " + Integer.toString(totalIterations) + " best score: " + currMinValue + " worst score: " + currMaxValue + " average score: " + averageValue); if (Math.abs(currMaxValue - currMinValue) < Math.abs(tol * currMaxValue) + Math.abs(tol * currMinValue)) { iterationCounter--; } else { iterationCounter = 1; } // totalIterations++; } for (int i = 0; i < populationSize; i++) { valuesChanged[i] = true; } computeValues(f, population, values, valuesChanged); double tempMinValue = Double.MAX_VALUE; int tempMinIndex = 0; for (int i = 0; i < populationSize; i++) { if (values.getEntry(i) < tempMinValue) { tempMinValue = values.getEntry(i); tempMinIndex = i; } } RealVector output = new ArrayRealVector(numberOfParameters); for (int i = 0; i < numberOfParameters; i++) { output.setEntry(i, population.getEntry(tempMinIndex, i)); } return output; }
From source file:com.simiacryptus.mindseye.applications.ObjectLocationBase.java
/** * Run./*from w ww. j a v a 2 s.c o m*/ * * @param log the log */ public void run(@Nonnull final NotebookOutput log) { // @Nonnull String logName = "cuda_" + log.getName() + ".log"; // log.p(log.file((String) null, logName, "GPU Log")); // CudaSystem.addLog(new PrintStream(log.file(logName))); ImageClassifierBase classifier = getClassifierNetwork(); Layer classifyNetwork = classifier.getNetwork(); ImageClassifierBase locator = getLocatorNetwork(); Layer locatorNetwork = locator.getNetwork(); ArtistryUtil.setPrecision((DAGNetwork) classifyNetwork, Precision.Float); ArtistryUtil.setPrecision((DAGNetwork) locatorNetwork, Precision.Float); Tensor[][] inputData = loadImages_library(); // Tensor[][] inputData = loadImage_Caltech101(log); double alphaPower = 0.8; final AtomicInteger index = new AtomicInteger(0); Arrays.stream(inputData).limit(10).forEach(row -> { log.h3("Image " + index.getAndIncrement()); final Tensor img = row[0]; log.p(log.image(img.toImage(), "")); Result classifyResult = classifyNetwork.eval(new MutableResult(row)); Result locationResult = locatorNetwork.eval(new MutableResult(row)); Tensor classification = classifyResult.getData().get(0); List<CharSequence> categories = classifier.getCategories(); int[] sortedIndices = IntStream.range(0, categories.size()).mapToObj(x -> x) .sorted(Comparator.comparing(i -> -classification.get(i))).mapToInt(x -> x).limit(10).toArray(); logger.info(Arrays.stream(sortedIndices) .mapToObj( i -> String.format("%s: %s = %s%%", i, categories.get(i), classification.get(i) * 100)) .reduce((a, b) -> a + "\n" + b).orElse("")); LinkedHashMap<CharSequence, Tensor> vectors = new LinkedHashMap<>(); List<CharSequence> predictionList = Arrays.stream(sortedIndices).mapToObj(categories::get) .collect(Collectors.toList()); Arrays.stream(sortedIndices).limit(6).forEach(category -> { CharSequence name = categories.get(category); log.h3(name); Tensor alphaTensor = renderAlpha(alphaPower, img, locationResult, classification, category); log.p(log.image(img.toRgbImageAlphaMask(0, 1, 2, alphaTensor), "")); vectors.put(name, alphaTensor.unit()); }); Tensor avgDetection = vectors.values().stream().reduce((a, b) -> a.add(b)).get() .scale(1.0 / vectors.size()); Array2DRowRealMatrix covarianceMatrix = new Array2DRowRealMatrix(predictionList.size(), predictionList.size()); for (int x = 0; x < predictionList.size(); x++) { for (int y = 0; y < predictionList.size(); y++) { Tensor l = vectors.get(predictionList.get(x)); Tensor r = vectors.get(predictionList.get(y)); covarianceMatrix.setEntry(x, y, null == l || null == r ? 0 : (l.minus(avgDetection)).dot(r.minus(avgDetection))); } } @Nonnull final EigenDecomposition decomposition = new EigenDecomposition(covarianceMatrix); for (int objectVector = 0; objectVector < 10; objectVector++) { log.h3("Eigenobject " + objectVector); double eigenvalue = decomposition.getRealEigenvalue(objectVector); RealVector eigenvector = decomposition.getEigenvector(objectVector); Tensor detectionRegion = IntStream.range(0, eigenvector.getDimension()).mapToObj(i -> { Tensor tensor = vectors.get(predictionList.get(i)); return null == tensor ? null : tensor.scale(eigenvector.getEntry(i)); }).filter(x -> null != x).reduce((a, b) -> a.add(b)).get(); detectionRegion = detectionRegion.scale(255.0 / detectionRegion.rms()); CharSequence categorization = IntStream.range(0, eigenvector.getDimension()).mapToObj(i -> { CharSequence category = predictionList.get(i); double component = eigenvector.getEntry(i); return String.format("<li>%s = %.4f</li>", category, component); }).reduce((a, b) -> a + "" + b).get(); log.p(String.format("Object Detected: <ol>%s</ol>", categorization)); log.p("Object Eigenvalue: " + eigenvalue); log.p("Object Region: " + log.image(img.toRgbImageAlphaMask(0, 1, 2, detectionRegion), "")); log.p("Object Region Compliment: " + log.image(img.toRgbImageAlphaMask(0, 1, 2, detectionRegion.scale(-1)), "")); } // final int[] orderedVectors = IntStream.range(0, 10).mapToObj(x -> x) // .sorted(Comparator.comparing(x -> -decomposition.getRealEigenvalue(x))).mapToInt(x -> x).toArray(); // IntStream.range(0, orderedVectors.length) // .mapToObj(i -> { // //double realEigenvalue = decomposition.getRealEigenvalue(orderedVectors[i]); // return decomposition.getEigenvector(orderedVectors[i]).toArray(); // } // ).toArray(i -> new double[i][]); log.p(String.format( "<table><tr><th>Cosine Distance</th>%s</tr>%s</table>", Arrays.stream(sortedIndices).limit(10) .mapToObj(col -> "<th>" + categories.get(col) + "</th>").reduce((a, b) -> a + b).get(), Arrays.stream(sortedIndices).limit(10).mapToObj(r -> { return String.format("<tr><td>%s</td>%s</tr>", categories.get(r), Arrays.stream(sortedIndices).limit(10).mapToObj(col -> { Tensor l = vectors.get(categories.get(r)); Tensor r2 = vectors.get(categories.get(col)); return String.format("<td>%.4f</td>", (null == l || null == r2) ? 0 : Math.acos(l.dot(r2))); }).reduce((a, b) -> a + b).get()); }).reduce((a, b) -> a + b).orElse(""))); }); log.setFrontMatterProperty("status", "OK"); }
From source file:com.clust4j.algo.NearestCentroidTests.java
@Test public void testCentroidViabilityKMeans() { final double[][] X = new double[][] { new double[] { 0, 0, 0 }, new double[] { 4, 4, 4 }, new double[] { 8, 8, 8 } }; final Array2DRowRealMatrix mat = new Array2DRowRealMatrix(X, false); NearestCentroid nn = new NearestCentroid(mat, new int[] { 0, 1, 2 }, new NearestCentroidParameters().setVerbose(true)).fit(); Array2DRowRealMatrix Y = new Array2DRowRealMatrix( new double[][] { new double[] { 0, 0, 0 }, new double[] { 1, 1, 1 }, new double[] { 4, 4, 4 }, new double[] { 5, 5, 5 }, new double[] { 8, 8, 8 }, new double[] { 9, 9, 9 } }, false);/* w w w . java2s . c o m*/ assertTrue(VecUtils.equalsExactly(nn.predict(Y), new int[] { 0, 0, 1, 1, 2, 2 })); }
From source file:eagle.security.userprofile.impl.UserProfileAnomalyEigenEvaluator.java
@Override public List<MLCallbackResult> detect(final String user, final String algorithm, UserActivityAggModel userActivity, UserProfileEigenModel aModel) { RealMatrix inputData = userActivity.matrix(); LOG.warn("EigenBasedAnomalyDetection predictAnomaly called with dimension: " + inputData.getRowDimension() + "x" + inputData.getColumnDimension()); if (aModel == null) { LOG.warn(/*w ww . j a va 2 s. com*/ "nothing to do as the input model does not have required values, returning from evaluating this algorithm.."); return null; } List<MLCallbackResult> mlCallbackResults = new ArrayList<MLCallbackResult>(); RealMatrix normalizedMat = normalizeData(inputData, aModel); UserCommandStatistics[] listStats = aModel.statistics(); int colWithHighVariant = 0; for (int j = 0; j < normalizedMat.getColumnDimension(); j++) { if (listStats[j].isLowVariant() == false) { colWithHighVariant++; } } final Map<String, String> context = new HashMap<String, String>() { { put(UserProfileConstants.USER_TAG, user); put(UserProfileConstants.ALGORITHM_TAG, algorithm); } }; Map<Integer, String> lineNoWithVariantBasedAnomalyDetection = new HashMap<Integer, String>(); for (int i = 0; i < normalizedMat.getRowDimension(); i++) { MLCallbackResult aResult = new MLCallbackResult(); aResult.setAnomaly(false); aResult.setContext(context); for (int j = 0; j < normalizedMat.getColumnDimension(); j++) { //LOG.info("mean for j=" + j + " is:" + listStats[j].getMean()); //LOG.info("stddev for j=" + j + " is:" + listStats[j].getStddev()); if (listStats[j].isLowVariant() == true) { //LOG.info(listOfCmds[j] + " is low variant"); if (normalizedMat.getEntry(i, j) > listStats[j].getMean()) { lineNoWithVariantBasedAnomalyDetection.put(i, "lowVariantAnomaly"); aResult.setAnomaly(true); aResult.setTimestamp(userActivity.timestamp()); aResult.setFeature(listStats[j].getCommandName()); aResult.setAlgorithm(UserProfileConstants.EIGEN_DECOMPOSITION_ALGORITHM); List<String> datapoints = new ArrayList<String>(); double[] rowVals = inputData.getRow(i); for (double rowVal : rowVals) datapoints.add(rowVal + ""); aResult.setDatapoints(datapoints); aResult.setId(user); //mlCallbackResults.add(aResult); } /*else { aResult.setAnomaly(false); aResult.setTimestamp(userActivity.timestamp()); mlCallbackResults.add(aResult); }*/ } } mlCallbackResults.add(i, aResult); //return results; } //LOG.info("results size here: " + results.length); //LOG.info("col with high variant: " + colWithHighVariant); RealMatrix finalMatWithoutLowVariantFeatures = new Array2DRowRealMatrix(normalizedMat.getRowDimension(), colWithHighVariant); //LOG.info("size of final test data: " + finalMatWithoutLowVariantFeatures.getRowDimension() +"x"+ finalMatWithoutLowVariantFeatures.getColumnDimension()); int finalMatrixRow = 0; int finalMatrixCol = 0; for (int i = 0; i < normalizedMat.getRowDimension(); i++) { for (int j = 0; j < normalizedMat.getColumnDimension(); j++) { if (listStats[j].isLowVariant() == false) { finalMatWithoutLowVariantFeatures.setEntry(finalMatrixRow, finalMatrixCol, normalizedMat.getEntry(i, j)); finalMatrixCol++; } } finalMatrixCol = 0; finalMatrixRow++; } RealVector[] pcs = aModel.principalComponents(); //LOG.info("pc size: " + pcs.getRowDimension() +"x" + pcs.getColumnDimension()); RealMatrix finalInputMatTranspose = finalMatWithoutLowVariantFeatures.transpose(); for (int i = 0; i < finalMatWithoutLowVariantFeatures.getRowDimension(); i++) { if (lineNoWithVariantBasedAnomalyDetection.get(i) == null) { //MLCallbackResult result = new MLCallbackResult(); MLCallbackResult result = mlCallbackResults.get(i); //result.setContext(context); for (int sz = 0; sz < pcs.length; sz++) { double[] pc1 = pcs[sz].toArray(); RealMatrix pc1Mat = new Array2DRowRealMatrix(pc1); RealMatrix transposePC1Mat = pc1Mat.transpose(); RealMatrix testData = pc1Mat.multiply(transposePC1Mat) .multiply(finalInputMatTranspose.getColumnMatrix(i)); //LOG.info("testData size: " + testData.getRowDimension() + "x" + testData.getColumnDimension()); RealMatrix testDataTranspose = testData.transpose(); //LOG.info("testData transpose size: " + testDataTranspose.getRowDimension() + "x" + testDataTranspose.getColumnDimension()); RealVector iRowVector = testDataTranspose.getRowVector(0); //RealVector pc1Vector = transposePC1Mat.getRowVector(sz); RealVector pc1Vector = transposePC1Mat.getRowVector(0); double distanceiRowAndPC1 = iRowVector.getDistance(pc1Vector); //LOG.info("distance from pc sz: " + sz + " " + distanceiRowAndPC1 + " " + model.getMaxL2Norm().getEntry(sz)); //LOG.info("model.getMaxL2Norm().getEntry(sz):" + model.getMaxL2Norm().getEntry(sz)); if (distanceiRowAndPC1 > aModel.maximumL2Norm().getEntry(sz)) { //LOG.info("distance from pc sz: " + sz + " " + distanceiRowAndPC1 + " " + model.getMaxL2Norm().getEntry(sz)); result.setAnomaly(true); result.setFeature(aModel.statistics()[sz].getCommandName()); result.setTimestamp(System.currentTimeMillis()); result.setAlgorithm(UserProfileConstants.EIGEN_DECOMPOSITION_ALGORITHM); List<String> datapoints = new ArrayList<String>(); double[] rowVals = inputData.getRow(i); for (double rowVal : rowVals) datapoints.add(rowVal + ""); result.setDatapoints(datapoints); result.setId(user); } } mlCallbackResults.set(i, result); } } return mlCallbackResults; }
From source file:com.clust4j.TestSuite.java
public static Array2DRowRealMatrix getRandom(final int rows, final int cols) { final Random rand = new Random(); final double[][] data = new double[rows][cols]; for (int i = 0; i < rows; i++) for (int j = 0; j < cols; j++) data[i][j] = rand.nextDouble() * (rand.nextDouble() > 0.5 ? -1 : 1); return new Array2DRowRealMatrix(data, false); }