List of usage examples for org.apache.commons.math3.stat.descriptive.moment Mean Mean
public Mean()
From source file:com.cloudera.oryx.als.computation.iterate.row.ComputeUserAPFn.java
@Override public void process(Pair<Long, float[]> input, Emitter<Double> emitter) { LongSet ids = testData.get(input.first()); if (ids == null || ids.isEmpty()) { return;/*from ww w.j a v a 2s . c o m*/ } float[] userVector = input.second(); LongObjectMap<float[]> Y = yState.getY(); long[] itemIDs = ids.toArray(); double[] scores = new double[itemIDs.length]; for (int i = 0; i < itemIDs.length; i++) { long itemID = itemIDs[i]; float[] itemVector = Y.get(itemID); if (itemVector == null) { continue; } scores[i] = SimpleVectorMath.dot(userVector, itemVector); } int[] rank = new int[itemIDs.length]; for (LongObjectMap.MapEntry<float[]> entry : Y.entrySet()) { double score = SimpleVectorMath.dot(userVector, entry.getValue()); for (int i = 0; i < itemIDs.length; i++) { if (score > scores[i]) { rank[i]++; } } } Arrays.sort(rank); Mean precision = new Mean(); double totalPrecisionTimesRelevance = 0.0; for (int i = 0; i < rank.length; i++) { int relevantRetrieved = i + 1; int precisionAt = rank[i] + 1; precision.increment((double) relevantRetrieved / precisionAt); totalPrecisionTimesRelevance += precision.getResult(); } double averagePrecision = totalPrecisionTimesRelevance / rank.length; log.info("Average precision: {}", averagePrecision); emitter.emit(averagePrecision); }
From source file:bigtweet.model.StudyingBeacons.java
@Override public void updateMetricsForParametersValues(JSONArray parametersValues, int parametersValuesIndex) { DecimalFormatSymbols otherSymbols = new DecimalFormatSymbols(Locale.US); DecimalFormat df = new DecimalFormat("0.000", otherSymbols); //get infected array for each seed double infected[] = new double[seedAndStates.values().size()]; for (int i = 0; i < infected.length; i++) { //number of infected for each seed //System.out.println( ((Map<String,Integer>) (seedAndStates.values().toArray()[i])).get("INFECTED")); infected[i] = (new Double( ((Map<String, Integer>) (seedAndStates.values().toArray()[i])).get("ENDORSER"))); }/* w w w . j a v a 2 s.c om*/ //get metrics for this experiment and this dataset double minInfected = (new Min()).evaluate(infected); double mean = (new Mean()).evaluate(infected); double sd = (new StandardDeviation()).evaluate(infected); //put in metricsForLastExperiment, JSON object metricsForLastExperiment = new JSONObject();//create new object for the last experiment metricsForLastExperiment.put("minEndorsers", df.format(minInfected)); metricsForLastExperiment.put("mean", df.format(mean)); metricsForLastExperiment.put("sd", df.format(sd)); JSONObject statesJSON = new JSONObject(); for (Map.Entry<Long, Map<String, Integer>> entry : seedAndStates.entrySet()) { statesJSON.put(entry.getKey().toString(), entry.getValue()); } metricsForLastExperiment.put("randomSeedsAndStates", statesJSON); //update metrics for all experiments if (mean < this.bestMean) { //System.out.println(name + " bestmean " + df.format(bestMean).toString() + " mean " + df.format(mean).toString()); bestMean = mean; experimentWithBestMean = parametersValuesIndex; metricsForAllExperiments.put("bestMean", df.format(bestMean)); metricsForAllExperiments.put("experimentWithBestMean", parametersValuesIndex); } if (minInfected < this.bestEndorser) { bestEndorser = minInfected; experimentWithBestEndorser = parametersValuesIndex; metricsForAllExperiments.put("bestEndorsers", df.format(minInfected)); metricsForAllExperiments.put("experimentWithBestEndorsers", parametersValuesIndex); } generateBatchOuputForChart(parametersValues, parametersValuesIndex, mean); }
From source file:gamlss.distributions.GA.java
/** Calculates initial value of mu, by assumption these * values lie between observed data and the trend line. * @param y - vector of values of response variable * @return a vector of initial values of mu *//* w w w . j a va2 s.c om*/ private ArrayRealVector setMuInitial(final ArrayRealVector y) { //mu.initial = expression({mu <- (y+mean(y))/2}) size = y.getDimension(); double[] out = new double[size]; Mean mean = new Mean(); double yMean = mean.evaluate(y.getDataRef()); for (int i = 0; i < size; i++) { out[i] = (y.getEntry(i) + yMean) / 2; } return new ArrayRealVector(out, false); }
From source file:gamlss.algorithm.AdditiveFit.java
/** * Constructor of AdditiveFit class./*from w ww . j av a 2 s .c om*/ * @param distribution - object of the fitted * distribution of to the gamlss family **@param smoothMatrices - initially supplied smoother matrices * each of the distribution parameterss * @param smoothMatrices * @param wlsReg - Object of WLSMultipleLinearRegression class. */ public AdditiveFit(final GAMLSSFamilyDistribution distribution, final HashMap<Integer, BlockRealMatrix> smoothMatrices, final WLSMultipleLinearRegression wlsReg) { mean = new Mean(); rConnection = new ConnectionToR(); this.wls = wlsReg; smoother = null; switch (Controls.SMOOTHER) { case Controls.PB: smoother = new PB(distribution, rConnection, smoothMatrices); break; case Controls.RW: if (Controls.BIG_DATA) { smoother = new RWcallR(distribution, rConnection, smoothMatrices); } else { smoother = new RW(distribution, smoothMatrices); } break; default: System.err.println("The specific smoother " + "has not been implemented yet in Gamlss!"); } }
From source file:net.myrrix.common.LoadRunner.java
public void runLoad() throws ExecutionException, InterruptedException { final Mean recommendedBecause = new Mean(); final Mean setPreference = new Mean(); final Mean removePreference = new Mean(); final Mean setTag = new Mean(); final Mean ingest = new Mean(); final Mean refresh = new Mean(); final Mean estimatePreference = new Mean(); final Mean mostSimilarItems = new Mean(); final Mean similarityToItem = new Mean(); final Mean mostPopularItems = new Mean(); final Mean recommendToMany = new Mean(); final Mean recommend = new Mean(); Processor<Integer> processor = new Processor<Integer>() { private final RandomGenerator random = RandomManager.getRandom(); @Override//from ww w . ja v a 2s . c o m public void process(Integer step, long count) { double r; long userID; long itemID; long itemID2; float value; synchronized (random) { r = random.nextDouble(); userID = uniqueUserIDs[random.nextInt(uniqueUserIDs.length)]; itemID = uniqueItemIDs[random.nextInt(uniqueItemIDs.length)]; itemID2 = uniqueItemIDs[random.nextInt(uniqueItemIDs.length)]; value = random.nextInt(10); } long stepStart = System.currentTimeMillis(); try { if (r < 0.05) { client.recommendedBecause(userID, itemID, 10); recommendedBecause.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.07) { client.setPreference(userID, itemID); setPreference.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.08) { client.setPreference(userID, itemID, value); setPreference.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.09) { client.setUserTag(userID, Long.toString(itemID)); setTag.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.10) { client.setItemTag(Long.toString(userID), itemID); setTag.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.11) { client.removePreference(userID, itemID); removePreference.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.12) { StringReader reader = new StringReader(userID + "," + itemID + ',' + value + '\n'); client.ingest(reader); ingest.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.13) { client.refresh(); refresh.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.14) { client.similarityToItem(itemID, itemID2); similarityToItem.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.15) { client.mostPopularItems(10); mostPopularItems.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.19) { client.estimatePreference(userID, itemID); estimatePreference.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.20) { client.estimateForAnonymous(itemID, new long[] { itemID2 }); estimatePreference.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.25) { client.mostSimilarItems(new long[] { itemID }, 10); mostSimilarItems.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.30) { client.recommendToMany(new long[] { userID, userID }, 10, true, null); recommendToMany.increment(System.currentTimeMillis() - stepStart); } else { client.recommend(userID, 10); recommend.increment(System.currentTimeMillis() - stepStart); } } catch (TasteException te) { log.warn("Error during request", te); } if (count % 1000 == 0) { log.info("Finished {} load steps", count); } } }; log.info("Starting load test..."); long start = System.currentTimeMillis(); new Paralleler<Integer>(new CountingIterator(steps), processor, "Load").runInParallel(); long end = System.currentTimeMillis(); log.info("Finished {} steps in {}ms", steps, end - start); log.info("recommendedBecause: {}", recommendedBecause.getResult()); log.info("setPreference: {}", setPreference.getResult()); log.info("removePreference: {}", removePreference.getResult()); log.info("setTag: {}", setTag.getResult()); log.info("ingest: {}", ingest.getResult()); log.info("refresh: {}", refresh.getResult()); log.info("estimatePreference: {}", estimatePreference.getResult()); log.info("mostSimilarItems: {}", mostSimilarItems.getResult()); log.info("similarityToItem: {}", similarityToItem.getResult()); log.info("mostPopularItems: {}", mostPopularItems.getResult()); log.info("recommendToMany: {}", recommendToMany.getResult()); log.info("recommend: {}", recommend.getResult()); }
From source file:gamlss.distributions.NO.java
/** Calculates initial value of mu, by assumption these values lie between observed data and the trend line. * @param y - vector of values of response variable * @return vector of initial values of mu *///from w w w . ja va 2 s . c om private ArrayRealVector setMuInitial(final ArrayRealVector y) { size = y.getDimension(); double[] out = new double[size]; final double yMean = new Mean().evaluate(y.getDataRef()); for (int i = 0; i < size; i++) { out[i] = (y.getEntry(i) + yMean) / 2; } return new ArrayRealVector(out, false); }
From source file:com.itemanalysis.psychometrics.irt.equating.RobustZEquatingTest.java
private void testA() throws IllegalArgumentException { double[] aDiff = new double[nA]; za = new RobustZ[nA]; for (int i = 0; i < nA; i++) { aDiff[i] = Math.log(aX[i]) - Math.log(aY[i]); }/*from w w w. ja v a2 s . com*/ double median = percentile.evaluate(aDiff, 50); double q3 = percentile.evaluate(aDiff, 75); double q1 = percentile.evaluate(aDiff, 25); double iqr = q3 - q1; Mean mean = new Mean(); for (int i = 0; i < nA; i++) { za[i] = new RobustZ(aDiff[i], median, iqr); if (!za[i].significant(significanceLevel)) { mean.increment(aDiff[i]); } } slope = Math.exp(mean.getResult()); }
From source file:com.cloudera.oryx.rdf.common.rule.CategoricalDecision.java
private static List<Decision> categoricalDecisionsForNumericTarget(int featureNumber, ExampleSet examples, int suggestedMaxSplitCandidates) { // PLANET paper claims this is optimal: int categoryCount = examples.getCategoryCount(featureNumber); Mean[] averageTargetForCategory = new Mean[categoryCount]; for (Example example : examples) { CategoricalFeature feature = (CategoricalFeature) example.getFeature(featureNumber); if (feature == null) { continue; }/*from www . j a v a2 s .c o m*/ int category = feature.getValueID(); Mean categoryAverage = averageTargetForCategory[category]; if (categoryAverage == null) { categoryAverage = new Mean(); averageTargetForCategory[category] = categoryAverage; } categoryAverage.increment(((NumericFeature) example.getTarget()).getValue()); } int maxCategory = -1; int maxCount = -1; for (int i = 0; i < averageTargetForCategory.length; i++) { Mean average = averageTargetForCategory[i]; if (average != null && average.getN() > maxCount) { maxCount = (int) averageTargetForCategory[i].getN(); maxCategory = i; } } Preconditions.checkArgument(maxCategory >= 0); List<Pair<Double, Integer>> byScore = Lists.newArrayListWithCapacity(averageTargetForCategory.length); for (int featureCategory = 0; featureCategory < averageTargetForCategory.length; featureCategory++) { StorelessUnivariateStatistic mean = averageTargetForCategory[featureCategory]; if (mean != null) { byScore.add(new Pair<Double, Integer>(mean.getResult(), featureCategory)); } } return sortAndGetDecisionsOverSubset(featureNumber, categoryCount, byScore, maxCategory, suggestedMaxSplitCandidates); }
From source file:br.unicamp.ic.recod.gpsi.applications.gpsiJGAPEvolver.java
@Override public void run() throws InvalidConfigurationException, InterruptedException, Exception { int i, j, k;/*from ww w .ja v a 2s. co m*/ byte nFolds = 5; gpsiDescriptor descriptor; gpsiMLDataset mlDataset; gpsiVoxelRawDataset dataset; GPGenotype gp; double[][] fitnessCurves; String[] curveLabels = new String[] { "train", "train_val", "val" }; double bestScore, currentScore; IGPProgram current, bestVal; Mean mean = new Mean(); StandardDeviation sd = new StandardDeviation(); double validationScore, trainScore, bestValidationScore, bestTrainScore; double[][][] samples; for (byte f = 0; f < nFolds; f++) { System.out.println("\nRun " + (f + 1) + "\n"); rawDataset.assignFolds(new byte[] { f, (byte) ((f + 1) % nFolds), (byte) ((f + 2) % nFolds) }, new byte[] { (byte) ((f + 3) % nFolds) }, new byte[] { (byte) ((f + 4) % nFolds) }); dataset = (gpsiVoxelRawDataset) rawDataset; gp = create(config, dataset.getnBands(), fitness); // 0: train, 1: train_val, 2: val fitnessCurves = new double[super.numGenerations][]; current = null; bestVal = null; bestScore = -Double.MAX_VALUE; bestValidationScore = -1.0; bestTrainScore = -1.0; for (int generation = 0; generation < super.numGenerations; generation++) { gp.evolve(1); gp.getGPPopulation().sortByFitness(); if (this.dumpGens) { double[][][] dists; descriptor = new gpsiScalarSpectralIndexDescriptor( new gpsiJGAPVoxelCombiner(fitness.getB(), gp.getGPPopulation().getGPPrograms()[0])); mlDataset = new gpsiMLDataset(descriptor); mlDataset.loadWholeDataset(rawDataset, true); dists = (new gpsiWholeSampler()).sample(mlDataset.getTrainingEntities(), this.classLabels); for (i = 0; i < this.classLabels.length; i++) { stream.register(new gpsiDoubleCsvIOElement(dists[i], null, "gens/f" + (f + 1) + "/" + classLabels[i] + "/" + (generation + 1) + ".csv")); } } for (i = 0; i < super.validation; i++) { current = gp.getGPPopulation().getGPPrograms()[i]; descriptor = new gpsiScalarSpectralIndexDescriptor( new gpsiJGAPVoxelCombiner(fitness.getB(), current)); mlDataset = new gpsiMLDataset(descriptor); mlDataset.loadWholeDataset(rawDataset, true); samples = this.fitness.getSampler().sample(mlDataset.getValidationEntities(), classLabels); validationScore = fitness.getScore().score(samples); trainScore = current.getFitnessValue() - 1.0; currentScore = mean.evaluate(new double[] { trainScore, validationScore }) - sd.evaluate(new double[] { trainScore, validationScore }); if (currentScore > bestScore) { bestVal = current; bestScore = currentScore; bestTrainScore = trainScore; bestValidationScore = validationScore; } } if (validation > 0) { best = new IGPProgram[2]; best[0] = gp.getAllTimeBest(); best[1] = bestVal; fitnessCurves[generation] = new double[] { best[0].getFitnessValue() - 1.0, bestTrainScore, bestValidationScore }; System.out.printf("%3dg: %.4f %.4f %.4f\n", generation + 1, fitnessCurves[generation][0], fitnessCurves[generation][1], fitnessCurves[generation][2]); } else { best = new IGPProgram[1]; best[0] = gp.getAllTimeBest(); fitnessCurves[generation] = new double[] { gp.getAllTimeBest().getFitnessValue() - 1.0 }; System.out.printf("%3dg: %.4f\n", generation + 1, fitnessCurves[generation][0]); } } stream.register(new gpsiDoubleCsvIOElement(fitnessCurves, curveLabels, "curves/f" + (f + 1) + ".csv")); System.out.println("Best solution for trainning: " + gp.getAllTimeBest().toStringNorm(0)); stream.register(new gpsiStringIOElement(gp.getAllTimeBest().toStringNorm(0), "programs/f" + (f + 1) + "train.program")); if (validation > 0) { System.out.println("Best solution for trainning and validation: " + bestVal.toStringNorm(0)); stream.register(new gpsiStringIOElement(bestVal.toStringNorm(0), "programs/f" + (f + 1) + "train_val.program")); } descriptor = new gpsiScalarSpectralIndexDescriptor(new gpsiJGAPVoxelCombiner(fitness.getB(), best[0])); gpsi1NNToMomentScalarClassificationAlgorithm classificationAlgorithm = new gpsi1NNToMomentScalarClassificationAlgorithm( new Mean()); gpsiClassifier classifier = new gpsiClassifier(descriptor, classificationAlgorithm); classifier.fit(this.rawDataset.getTrainingEntities()); classifier.predict(this.rawDataset.getTestEntities()); int[][] confusionMatrix = classifier.getConfusionMatrix(); stream.register(new gpsiIntegerCsvIOElement(confusionMatrix, null, "confusion_matrices/f" + (f + 1) + "_train.csv")); if (validation > 0) { descriptor = new gpsiScalarSpectralIndexDescriptor( new gpsiJGAPVoxelCombiner(fitness.getB(), best[1])); classificationAlgorithm = new gpsi1NNToMomentScalarClassificationAlgorithm(new Mean()); classifier = new gpsiClassifier(descriptor, classificationAlgorithm); classifier.fit(this.rawDataset.getTrainingEntities()); classifier.predict(this.rawDataset.getTestEntities()); confusionMatrix = classifier.getConfusionMatrix(); stream.register(new gpsiIntegerCsvIOElement(confusionMatrix, null, "confusion_matrices/f" + (f + 1) + "_train_val.csv")); } } }
From source file:eagle.security.userprofile.model.eigen.UserProfileEigenModeler.java
private void computeStats(RealMatrix m) { if (m.getColumnDimension() != this.cmdTypes.length) { LOG.error("Please fix the commands list in config file"); return;//from www .jav a 2 s. com } statistics = new UserCommandStatistics[m.getColumnDimension()]; for (int i = 0; i < m.getColumnDimension(); i++) { UserCommandStatistics stats = new UserCommandStatistics(); stats.setCommandName(this.cmdTypes[i]); RealVector colData = m.getColumnVector(i); StandardDeviation deviation = new StandardDeviation(); double stddev = deviation.evaluate(colData.toArray()); //LOG.info("stddev is nan?" + (stddev == Double.NaN? "yes":"no")); if (stddev <= lowVarianceVal) stats.setLowVariant(true); else stats.setLowVariant(false); stats.setStddev(stddev); Mean mean = new Mean(); double mu = mean.evaluate(colData.toArray()); //LOG.info("mu is nan?" + (mu == Double.NaN? "yes":"no")); stats.setMean(mu); statistics[i] = stats; } }