List of usage examples for org.apache.commons.math3.stat.descriptive.moment StandardDeviation StandardDeviation
public StandardDeviation()
From source file:br.unicamp.ic.recod.gpsi.applications.gpsiJGAPSelectorEvolver.java
@Override public void run() throws InvalidConfigurationException, InterruptedException, Exception { int i, j, k;//ww w . ja v a2 s. co m byte nFolds = 5; gpsiDescriptor descriptor; gpsiMLDataset mlDataset; gpsiVoxelRawDataset dataset; GPGenotype gp; double[][] fitnessCurves; String[] curveLabels = new String[] { "train", "train_val", "val" }; double bestScore, currentScore; IGPProgram current; IGPProgram[] elite = null; Mean mean = new Mean(); StandardDeviation sd = new StandardDeviation(); double validationScore, trainScore; double[][][] samples; for (byte f = 0; f < nFolds; f++) { System.out.println("\nRun " + (f + 1) + "\n"); rawDataset.assignFolds(new byte[] { f, (byte) ((f + 1) % nFolds), (byte) ((f + 2) % nFolds) }, new byte[] { (byte) ((f + 3) % nFolds) }, new byte[] { (byte) ((f + 4) % nFolds) }); dataset = (gpsiVoxelRawDataset) rawDataset; gp = create(config, dataset.getnBands(), fitness, null); // 0: train, 1: train_val, 2: val fitnessCurves = new double[super.numGenerations + numGenerationsSel][]; bestScore = -Double.MAX_VALUE; if (validation > 0) elite = new IGPProgram[validation]; for (int generation = 0; generation < numGenerationsSel; generation++) { gp.evolve(1); gp.getGPPopulation().sortByFitness(); if (validation > 0) elite = mergeElite(elite, gp.getGPPopulation().getGPPrograms(), generation); if (this.dumpGens) { double[][][] dists; descriptor = new gpsiScalarSpectralIndexDescriptor( new gpsiJGAPVoxelCombiner(fitness.getB(), gp.getGPPopulation().getGPPrograms()[0])); mlDataset = new gpsiMLDataset(descriptor); mlDataset.loadWholeDataset(rawDataset, true); dists = (new gpsiWholeSampler()).sample(mlDataset.getTrainingEntities(), this.classLabels); for (i = 0; i < this.classLabels.length; i++) { stream.register(new gpsiDoubleCsvIOElement(dists[i], null, "gens/f" + (f + 1) + "/" + classLabels[i] + "/" + (generation + 1) + ".csv")); } } fitnessCurves[generation] = new double[] { gp.getAllTimeBest().getFitnessValue() - 1.0 }; System.out.printf("%3dg: %.4f\n", generation + 1, fitnessCurves[generation][0]); } HashSet<Integer> variables = new HashSet<>(); for (IGPProgram ind : elite) { for (CommandGene node : ind.getChromosome(0).getFunctions()) { if (node instanceof Variable) { variables.add(Integer.parseInt(node.getName().replace('b', '0'))); } } } int[] vars = variables.stream().mapToInt(p -> p).toArray(); Arrays.sort(vars); stream.register(new gpsiStringIOElement(Arrays.toString(vars), "selected_bands/f" + (f + 1) + ".out")); gp = create(config, dataset.getnBands(), fitness, vars); gp.addFittestProgram(elite[0]); for (int generation = numGenerationsSel; generation < numGenerationsSel + super.numGenerations; generation++) { gp.evolve(1); gp.getGPPopulation().sortByFitness(); if (validation > 0) elite = mergeElite(elite, gp.getGPPopulation().getGPPrograms(), generation); if (this.dumpGens) { double[][][] dists; descriptor = new gpsiScalarSpectralIndexDescriptor( new gpsiJGAPVoxelCombiner(fitness.getB(), gp.getGPPopulation().getGPPrograms()[0])); mlDataset = new gpsiMLDataset(descriptor); mlDataset.loadWholeDataset(rawDataset, true); dists = (new gpsiWholeSampler()).sample(mlDataset.getTrainingEntities(), this.classLabels); for (i = 0; i < this.classLabels.length; i++) { stream.register(new gpsiDoubleCsvIOElement(dists[i], null, "gens/f" + (f + 1) + "/" + classLabels[i] + "/" + (generation + 1) + ".csv")); } } fitnessCurves[generation] = new double[] { gp.getAllTimeBest().getFitnessValue() - 1.0 }; System.out.printf("%3dg: %.4f\n", generation + 1, fitnessCurves[generation][0]); } best = new IGPProgram[2]; best[0] = gp.getAllTimeBest(); for (i = 0; i < super.validation; i++) { current = elite[i]; descriptor = new gpsiScalarSpectralIndexDescriptor( new gpsiJGAPVoxelCombiner(fitness.getB(), current)); mlDataset = new gpsiMLDataset(descriptor); mlDataset.loadWholeDataset(rawDataset, true); samples = this.fitness.getSampler().sample(mlDataset.getValidationEntities(), classLabels); validationScore = fitness.getScore().score(samples); trainScore = current.getFitnessValue() - 1.0; currentScore = mean.evaluate(new double[] { trainScore, validationScore }) - sd.evaluate(new double[] { trainScore, validationScore }); if (currentScore > bestScore) { best[1] = current; bestScore = currentScore; } } stream.register(new gpsiDoubleCsvIOElement(fitnessCurves, curveLabels, "curves/f" + (f + 1) + ".csv")); System.out.println("Best solution for trainning: " + gp.getAllTimeBest().toStringNorm(0)); stream.register(new gpsiStringIOElement(gp.getAllTimeBest().toStringNorm(0), "programs/f" + (f + 1) + "train.program")); if (validation > 0) { System.out.println("Best solution for trainning and validation: " + best[1].toStringNorm(0)); stream.register(new gpsiStringIOElement(best[1].toStringNorm(0), "programs/f" + (f + 1) + "train_val.program")); } descriptor = new gpsiScalarSpectralIndexDescriptor(new gpsiJGAPVoxelCombiner(fitness.getB(), best[0])); gpsi1NNToMomentScalarClassificationAlgorithm classificationAlgorithm = new gpsi1NNToMomentScalarClassificationAlgorithm( new Mean()); gpsiClassifier classifier = new gpsiClassifier(descriptor, classificationAlgorithm); classifier.fit(this.rawDataset.getTrainingEntities()); classifier.predict(this.rawDataset.getTestEntities()); int[][] confusionMatrix = classifier.getConfusionMatrix(); stream.register(new gpsiIntegerCsvIOElement(confusionMatrix, null, "confusion_matrices/f" + (f + 1) + "_train.csv")); if (validation > 0) { descriptor = new gpsiScalarSpectralIndexDescriptor( new gpsiJGAPVoxelCombiner(fitness.getB(), best[1])); classificationAlgorithm = new gpsi1NNToMomentScalarClassificationAlgorithm(new Mean()); classifier = new gpsiClassifier(descriptor, classificationAlgorithm); classifier.fit(this.rawDataset.getTrainingEntities()); classifier.predict(this.rawDataset.getTestEntities()); confusionMatrix = classifier.getConfusionMatrix(); stream.register(new gpsiIntegerCsvIOElement(confusionMatrix, null, "confusion_matrices/f" + (f + 1) + "_train_val.csv")); } } }
From source file:gedi.util.math.stat.distributions.NormalMixtureDistribution.java
public static NormalMixtureDistribution fit(NormalMixtureDistribution initialMixture, double[] data, final int maxIterations, final double threshold) { if (maxIterations < 1) { throw new NotStrictlyPositiveException(maxIterations); }/* ww w. j av a2s. c om*/ if (threshold < Double.MIN_VALUE) { throw new NotStrictlyPositiveException(threshold); } final int n = data.length; final int k = initialMixture.getNumComponents(); if (k == 1) return new NormalMixtureDistribution(new NormalDistribution[] { new NormalDistribution(new Mean().evaluate(data), new StandardDeviation().evaluate(data)) }, new double[] { 1 }); int numIterations = 0; double previousLogLikelihood = 0d; double logLikelihood = Double.NEGATIVE_INFINITY; // Initialize model to fit to initial mixture. NormalMixtureDistribution fittedModel = new NormalMixtureDistribution(initialMixture.components, initialMixture.mixing); while (numIterations++ <= maxIterations && FastMath.abs(previousLogLikelihood - logLikelihood) > threshold) { previousLogLikelihood = logLikelihood; logLikelihood = 0d; // E-step: compute the data dependent parameters of the expectation // function. // The percentage of row's total density between a row and a // component final double[][] gamma = new double[n][k]; // Sum of gamma for each component final double[] gammaSums = new double[k]; for (int i = 0; i < n; i++) { final double rowDensity = fittedModel.density(data[i]); logLikelihood += FastMath.log(rowDensity); for (int j = 0; j < k; j++) { gamma[i][j] = fittedModel.mixing[j] * fittedModel.components[j].density(data[i]) / rowDensity; gammaSums[j] += gamma[i][j]; } } logLikelihood /= n; // System.out.println(logLikelihood); // M-step: compute the new parameters based on the expectation // function. final double[] newWeights = gammaSums.clone(); ArrayUtils.mult(newWeights, 1.0 / n); NormalDistribution[] comp = new NormalDistribution[k]; for (int j = 0; j < k; j++) { double m = 0; for (int i = 0; i < n; i++) { m += gamma[i][j] * data[i]; } m /= gammaSums[j]; double var = 0; for (int i = 0; i < n; i++) { double d = m - data[i]; var += gamma[i][j] * d * d; } var /= gammaSums[j]; comp[j] = new NormalDistribution(m, Math.sqrt(var)); } // Update current model fittedModel = new NormalMixtureDistribution(comp, newWeights); } if (FastMath.abs(previousLogLikelihood - logLikelihood) > threshold) { // Did not converge before the maximum number of iterations throw new ConvergenceException(); } return fittedModel; }
From source file:com.itemanalysis.psychometrics.histogram.Histogram.java
private void createHistogram(double[] x) { n = x.length;/* w w w . j a v a 2 s .c om*/ Min min = new Min(); Max max = new Max(); Mean mean = new Mean(); StandardDeviation sd = new StandardDeviation(); for (int i = 0; i < x.length; i++) { min.increment(x[i]); max.increment(x[i]); mean.increment(x[i]); sd.increment(x[i]); } double range = max.getResult() - min.getResult(); double lowestBoundary = min.getResult() - range / 1000; double largestBoundary = max.getResult() + range / 1000; if (binCalculationType == BinCalculationType.SCOTT) { binCalc = new ScottBinCalculation(n, min.getResult(), max.getResult(), sd.getResult()); } else if (binCalculationType == BinCalculationType.FREEDMAN_DIACONIS) { Percentile percentile = new Percentile(); double q1 = percentile.evaluate(x, 25); double q3 = percentile.evaluate(x, 75); binCalc = new FreedmanDiaconisBinCalculation(n, min.getResult(), max.getResult(), q1, q3); } else if (binCalculationType == BinCalculationType.STURGES) { binCalc = new SturgesBinCalculation(n, min.getResult(), max.getResult()); } numberOfBins = binCalc.numberOfBins(); binWidth = binCalc.binWidth(); //create bins createBins(lowestBoundary, largestBoundary); //count observations in each bin for (int i = 0; i < n; i++) { for (Bin b : bins) { b.increment(x[i]); } } }
From source file:gamlss.distributions.PE.java
/** Calculate and set initial value of sigma. * @param y - vector of values of response variable * @return vector of initial values of sigma */// w w w . j a va 2 s. c o m private ArrayRealVector setSigmaInitial(final ArrayRealVector y) { //sigma.initial = expression( sigma <- (abs(y-mean(y))+sd(y))/2 ) final double mean = new Mean().evaluate(y.getDataRef()); final double sd = new StandardDeviation().evaluate(y.getDataRef()); size = y.getDimension(); double[] out = new double[size]; for (int i = 0; i < size; i++) { out[i] = (FastMath.abs(y.getEntry(i) - mean) + sd) / 2; } return new ArrayRealVector(out, false); }
From source file:com.graphhopper.jsprit.core.algorithm.acceptor.ExperimentalSchrimpfAcceptance.java
@Override public void informAlgorithmStarts(VehicleRoutingProblem problem, VehicleRoutingAlgorithm algorithm, Collection<VehicleRoutingProblemSolution> solutions) { reset();//from w w w. ja v a 2s. c o m logger.info("---------------------------------------------------------------------"); logger.info("prepare schrimpfAcceptanceFunction, i.e. determine initial threshold"); logger.info("start random-walk (see randomWalk.xml)"); double now = System.currentTimeMillis(); this.nOfTotalIterations = algorithm.getMaxIterations(); /* * randomWalk to determine standardDev */ final double[] results = new double[nOfRandomWalks]; Jsprit.Builder builder = new GreedySchrimpfFactory().createGreedyAlgorithmBuilder(problem); builder.setCustomAcceptor(new AcceptNewRemoveFirst(1)); VehicleRoutingAlgorithm vra = builder.buildAlgorithm(); vra.setMaxIterations(nOfRandomWalks); vra.getAlgorithmListeners().addListener(new IterationEndsListener() { @Override public void informIterationEnds(int iteration, VehicleRoutingProblem problem, Collection<VehicleRoutingProblemSolution> solutions) { double result = Solutions.bestOf(solutions).getCost(); // logger.info("result={}", result); results[iteration - 1] = result; } }); vra.searchSolutions(); StandardDeviation dev = new StandardDeviation(); double standardDeviation = dev.evaluate(results); initialThreshold = standardDeviation / 2; logger.info("warmup done"); logger.info("total time: {}s", ((System.currentTimeMillis() - now) / 1000.0)); logger.info("initial threshold: {}", initialThreshold); logger.info("---------------------------------------------------------------------"); }
From source file:gamlss.distributions.SST.java
/** Calculate and set initial value of sigma. * @param y - vector of values of response variable * @return vector of initial values of sigma *//*from w w w . j a v a 2s . c o m*/ private ArrayRealVector setSigmaInitial(final ArrayRealVector y) { //sigma.initial = expression(sigma<- rep(sd(y), length(y))), tempV = new ArrayRealVector(y.getDimension()); final double out = new StandardDeviation().evaluate(y.getDataRef()); tempV.set(out); return tempV; }
From source file:eu.eexcess.dataformats.PartnerBadge.java
/** * updates the partner response times (shortTime and longTime) and short time deviation * //from ww w. j av a2 s.com * @param partner * @param respTime */ public void updatePartnerResponseTime(long respTime) { pushLastResponseTimes(respTime); java.util.Iterator<Long> iter = getLastResponseTimes().iterator(); Long first = iter.next(); while (first == null && iter.hasNext()) first = iter.next(); if (first != null) setShortTimeResponseTime(first); while (iter.hasNext()) { Long next = iter.next(); if (next != null) setShortTimeResponseTime((getShortTimeResponseTime() + next) / 2); } // if (getLongTimeResponseTime() != null) // setLongTimeResponseTime((getLongTimeResponseTime() + respTime) / 2); // else // setLongTimeResponseTime(respTime); double[] values = new double[getLastResponseTimes().toArray().length]; StandardDeviation standartDeviation = new StandardDeviation(); Object[] respTimes = getLastResponseTimes().toArray(); int count = 0; for (Object long1 : respTimes) { if (long1 != null) values[count] = ((Long) long1).doubleValue(); count++; } // setShortTimeResponsDeviation(new Double(standartDeviation.evaluate(values)).longValue()); }
From source file:edu.umd.umiacs.clip.tools.classifier.LibSVMUtils.java
public static Map<Integer, Pair<Double, Double>> learnZscoringModel(List<String> training) { return training.stream().map(LibSVMUtils::split).map(Triple::getMiddle).flatMap(List::stream) .collect(groupingBy(Pair::getKey, ConcurrentHashMap::new, reducing(new ArrayList<Float>(), pair -> asList(pair.getRight()), (p1, p2) -> Stream.of(p1, p2).flatMap(List::stream).collect(toList())))) .entrySet().stream()/*from www .j a v a 2 s .c o m*/ .map(entry -> Pair.of(entry.getKey(), entry.getValue().stream().mapToDouble(f -> f).toArray())) .collect(toMap(Entry::getKey, entry -> Pair.of(new Mean().evaluate(entry.getValue()), new StandardDeviation().evaluate(entry.getValue())))); }
From source file:com.itemanalysis.jmetrik.stats.transformation.LinearTransformationAnalysis.java
public String transformScore() throws SQLException { Statement stmt = null;//from w ww . j av a 2 s. c o m ResultSet rs = null; Double constrainedScore = null; try { //add variable to db dao.addColumnToDb(conn, tableName, addedVariableInfo); conn.setAutoCommit(false);//begin transaction Table sqlTable = new Table(tableName.getNameForDatabase()); SelectQuery select = new SelectQuery(); select.addColumn(sqlTable, selectedVariable.getName().nameForDatabase()); select.addColumn(sqlTable, addedVariableInfo.getName().nameForDatabase()); stmt = conn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_UPDATABLE); rs = stmt.executeQuery(select.toString()); this.firePropertyChange("message", "", "Transforming scores..."); double origValue = 0.0; double transValue = 0.0; double z = 0.0; StandardDeviation sd = new StandardDeviation(); Mean mean = new Mean(); Min min = new Min(); Max max = new Max(); while (rs.next()) { origValue = rs.getDouble(selectedVariable.getName().nameForDatabase()); if (!rs.wasNull()) { sd.increment(origValue); mean.increment(origValue); min.increment(origValue); max.increment(origValue); } updateProgress(); } double meanValue = mean.getResult(); double sdValue = sd.getResult(); double minValue = min.getResult(); double maxValue = max.getResult(); double A = 1.0; double B = 0.0; rs.beforeFirst(); while (rs.next()) { origValue = rs.getDouble(selectedVariable.getName().nameForDatabase()); if (!rs.wasNull()) { if (type1) { z = (origValue - meanValue) / sdValue; transValue = scaleSd * z + scaleMean; transValue = checkConstraints(transValue); } else { A = (maxPossibleScore - minPossibleScore) / (maxValue - minValue); B = minPossibleScore - minValue * A; transValue = origValue * A + B; transValue = checkConstraints(transValue); } descriptiveStatistics.increment(transValue); rs.updateDouble(addedVariableInfo.getName().nameForDatabase(), transValue); rs.updateRow(); } updateProgress(); } conn.commit(); conn.setAutoCommit(true); //create output DefaultLinearTransformation linearTransformation = new DefaultLinearTransformation(); linearTransformation.setScale(A); linearTransformation.setIntercept(B); StringBuilder sb = new StringBuilder(); Formatter f = new Formatter(sb); f.format(publishHeader()); f.format(descriptiveStatistics.toString()); f.format(linearTransformation.toString()); f.format("%n"); f.format("%n"); return f.toString(); } catch (SQLException ex) { conn.rollback(); conn.setAutoCommit(true); throw ex; } finally { if (rs != null) rs.close(); if (stmt != null) stmt.close(); } }
From source file:gamlss.distributions.GT.java
/** Calculate and set initial value of sigma. * @param y - vector of values of response variable * @return vector of initial values of sigma *///w w w . j av a 2 s . co m private ArrayRealVector setSigmaInitial(final ArrayRealVector y) { //sigma.initial = expression(sigma<- rep(sd(y)/4, length(y))), tempV = new ArrayRealVector(y.getDimension()); final double out = new StandardDeviation().evaluate(y.getDataRef()); tempV.set(out / 4.0); return tempV; }