List of usage examples for org.apache.commons.math3.stat.descriptive.moment Mean Mean
public Mean()
From source file:com.itemanalysis.jmetrik.stats.irt.linking.CommonItemSummaryStatistics.java
private void createArrays() { xAMean = new Mean(); yAMean = new Mean(); xBMean = new Mean(); yBMean = new Mean(); xCMean = new Mean(); yCMean = new Mean(); xASd = new StandardDeviation(); yASd = new StandardDeviation(); xBSd = new StandardDeviation(); yBSd = new StandardDeviation(); xCSd = new StandardDeviation(); yCSd = new StandardDeviation(); int xBinary = 0; int xPolytomous = 0; int xTotalSteps = 0; ItemResponseModel irm;/*from ww w . j a v a 2s .c o m*/ for (String s : irmX.keySet()) { irm = irmX.get(s); if (irm.getNcat() == 2) { xBinary++; } else { xPolytomous++; } xTotalSteps += (irm.getNcat() - 1); } aX = new double[xBinary + xPolytomous]; bX = new double[xTotalSteps]; cX = new double[xBinary]; int yBinary = 0; int yPolytomous = 0; int yTotalSteps = 0; for (String s : irmY.keySet()) { irm = irmY.get(s); if (irm.getNcat() == 2) { yBinary++; } else { yPolytomous++; } yTotalSteps += (irm.getNcat() - 1); } aY = new double[yBinary + yPolytomous]; bY = new double[yTotalSteps]; cY = new double[yBinary]; //populate arrays for Form X int itemIndex = 0; int binaryIndex = 0; int polyIndex = 0; int ncat = 0; for (String s : irmX.keySet()) { irm = irmX.get(s); ncat = irm.getNcat(); varNamesA.add(new VariableName(s)); aX[itemIndex] = irm.getDiscrimination(); xAMean.increment(aX[itemIndex]); xASd.increment(aX[itemIndex]); if (ncat == 2) { cX[binaryIndex] = irm.getGuessing(); xCMean.increment(cX[binaryIndex]); xCSd.increment(cX[binaryIndex]); bX[polyIndex] = irm.getDifficulty(); xBMean.increment(bX[polyIndex]); xBSd.increment(bX[polyIndex]); binaryIndex++; polyIndex++; varNamesB.add(new VariableName(s)); } else { double[] steps; if (irm.getType() == IrmType.GPCM || irm.getType() == IrmType.PCM2) { steps = irm.getStepParameters(); steps = Arrays.copyOfRange(steps, 1, steps.length);//Omit first step since it is zero. } else { steps = irm.getStepParameters(); } for (int k = 0; k < steps.length; k++) { bX[polyIndex] = steps[k]; xBMean.increment(bX[polyIndex]); xBSd.increment(bX[polyIndex]); polyIndex++; varNamesB.add(new VariableName(s + "_" + (k + 1))); } } itemIndex++; } //end loop over items //populate arrays for Form Y itemIndex = 0; binaryIndex = 0; polyIndex = 0; ncat = 0; for (String s : irmY.keySet()) { irm = irmY.get(s); ncat = irm.getNcat(); aY[itemIndex] = irm.getDiscrimination(); yAMean.increment(aY[itemIndex]); yASd.increment(aY[itemIndex]); if (ncat == 2) { cY[binaryIndex] = irm.getGuessing(); yCMean.increment(cY[binaryIndex]); yCSd.increment(cY[binaryIndex]); bY[polyIndex] = irm.getDifficulty(); yBMean.increment(bY[polyIndex]); yBSd.increment(bY[polyIndex]); binaryIndex++; polyIndex++; } else { double[] steps; if (irm.getType() == IrmType.GPCM || irm.getType() == IrmType.PCM2) { steps = irm.getStepParameters(); steps = Arrays.copyOfRange(steps, 1, steps.length);//Omit first step since it is zero. } else { steps = irm.getStepParameters(); } for (int k = 0; k < steps.length; k++) { bY[polyIndex] = steps[k]; yBMean.increment(bY[polyIndex]); yBSd.increment(bY[polyIndex]); polyIndex++; } } itemIndex++; } //end loop over items }
From source file:com.itemanalysis.psychometrics.irt.equating.MeanSigmaMethod.java
private void evaluate() { Mean mX = new Mean(); StandardDeviation sdX = new StandardDeviation(populationStdDev); Mean mY = new Mean(); StandardDeviation sdY = new StandardDeviation(populationStdDev); ItemResponseModel irmX;//from w ww .j av a2 s. c om ItemResponseModel irmY; for (String s : sY) { irmX = itemFormX.get(s); irmY = itemFormY.get(s); irmX.incrementMeanSigma(mX, sdX); irmY.incrementMeanSigma(mY, sdY); } if (checkRaschModel()) { slope = 1.0; } else { slope = sdY.getResult() / sdX.getResult(); } intercept = mY.getResult() - slope * mX.getResult(); }
From source file:eagle.security.userprofile.model.kde.UserProfileKDEModeler.java
private void computeStats(RealMatrix m) { if (m.getColumnDimension() != this.cmdTypes.length) { LOG.error("Please fix the commands list in config file"); }/* ww w . j a va 2s . c o m*/ statistics = new UserCommandStatistics[m.getColumnDimension()]; for (int i = 0; i < m.getColumnDimension(); i++) { UserCommandStatistics stats = new UserCommandStatistics(); stats.setCommandName(this.cmdTypes[i]); RealVector colData = m.getColumnVector(i); StandardDeviation deviation = new StandardDeviation(); double stddev = deviation.evaluate(colData.toArray()); if (LOG.isDebugEnabled()) LOG.debug("Stddev is NAN ? " + (Double.isNaN(stddev) ? "yes" : "no")); if (stddev <= lowVarianceVal) stats.setLowVariant(true); else stats.setLowVariant(false); stats.setStddev(stddev); Mean mean = new Mean(); double mu = mean.evaluate(colData.toArray()); if (LOG.isDebugEnabled()) LOG.debug("mu is NAN ? " + (Double.isNaN(mu) ? "yes" : "no")); stats.setMean(mu); statistics[i] = stats; } }
From source file:com.sciaps.utils.Util.java
public static Spectrum createAverage(Collection<? extends Spectrum> shots, double sampleRate) { Min minWL = new Min(); Max maxWL = new Max(); for (Spectrum shot : shots) { minWL.increment(shot.getValidRange().getMinimumDouble()); maxWL.increment(shot.getValidRange().getMaximumDouble()); }/*from www. j a v a2s.co m*/ double range = maxWL.getResult() - minWL.getResult(); int numSamples = (int) Math.floor(range * sampleRate); double[][] data = new double[2][numSamples]; Mean avgy = new Mean(); for (int i = 0; i < numSamples; i++) { double x = minWL.getResult() + i * (1 / sampleRate); avgy.clear(); for (Spectrum shot : shots) { if (shot.getValidRange().containsDouble(x)) { UnivariateFunction iv = shot.getIntensityFunction(); double y = iv.value(x); avgy.increment(y); } } data[0][i] = x; data[1][i] = avgy.getResult(); } RawDataSpectrum newSpectrum = new RawDataSpectrum(data); return newSpectrum; }
From source file:com.itemanalysis.psychometrics.rasch.RatingScaleThresholds.java
/** * The number of categories is maxCategory+1 * * @param groupId identifier for this rating scale category group * @param maxCategory largest category rho *//*w w w. ja v a 2s. co m*/ public RatingScaleThresholds(String groupId, int maxCategory) { this.groupId = groupId; numberOfCategories = maxCategory; thresholds = new double[numberOfCategories]; proposalThresholds = new double[numberOfCategories]; standardErrors = new double[numberOfCategories]; mean = new Mean(); sd = new StandardDeviation(); items = new ArrayList<RatingScaleItem>(); rsm = new RatingScaleModel(); fit = new LinkedHashMap<Integer, CategoryFitStatistics>(); initializeAllFit(); }
From source file:eu.crisis_economics.abm.algorithms.series.TestAbstractSeries.java
/** * Assert that the long term mean of values drawn from a {@link RandomSeries} object * is as expected.//from w w w . j a v a 2 s . c om * * @param series (<code>S</code>) <br> * The {@link RandomSeries} object to test. * @param numberOfSamples <br> * The number of samples to draw from <code>S</code>. * @param expectedLongTermMean <br> * The expected long term mean of the series. */ protected void assertLongTermMean(RandomSeries series, final int numberOfSamples, final double expectedLongTermMean) { final Mean mean = new Mean(); for (int i = 0; i < numberOfSamples; ++i) mean.increment(series.next()); Assert.assertEquals(mean.getResult(), expectedLongTermMean, 1.e-2); }
From source file:com.github.pitzcarraldo.dissimilar.Dissimilar.java
/** * Calculate the SSIM value on a window of pixels * @param pLumaOne luma values for first image * @param pLumaTwo luma values for second image * @param pWidth width of the two images * @param pHeight height of the two images * @param pWindowSize window size/*from w w w . j av a2s .c om*/ * @param pStartX start x coordinate for the window * @param pStartY start y coordinate for the window * @return SSIM for the window */ private static double calcSSIMOnWindow(final double[] pLumaOne, final double[] pLumaTwo, final int pWidth, final int pHeight, final int pWindowSize, final int pStartX, final int pStartY) { final double k1 = 0.01; final double k2 = 0.03; final double L = Math.pow(2, 8) - 1;//255 (at least for the moment all pixel values are 8-bit) final double c1 = Math.pow(k1 * L, 2); final double c2 = Math.pow(k2 * L, 2); final int windowWidth = ((pWindowSize + pStartX) > pWidth) ? (pWidth - pStartX) : (pWindowSize); final int windowHeight = ((pWindowSize + pStartY) > pHeight) ? (pHeight - pStartY) : (pWindowSize); if (windowWidth <= 0 || windowHeight <= 0) { System.out.println(pWidth + " " + pStartX + " " + windowWidth + " " + windowHeight); System.out.println(pHeight + " " + pStartY + " " + windowWidth + " " + windowHeight); } //store a temporary copy of the pixels double[] pixelsOne = new double[windowWidth * windowHeight]; double[] pixelsTwo = new double[windowWidth * windowHeight]; for (int h = 0; h < windowHeight; h++) { for (int w = 0; w < windowWidth; w++) { pixelsOne[(h * windowWidth) + w] = pLumaOne[(pStartY + h) * pWidth + pStartX + w]; pixelsTwo[(h * windowWidth) + w] = pLumaTwo[(pStartY + h) * pWidth + pStartX + w]; } } final double ux = new Mean().evaluate(pixelsOne, 0, pixelsOne.length); final double uy = new Mean().evaluate(pixelsTwo, 0, pixelsTwo.length); final double o2x = new Variance().evaluate(pixelsOne); final double o2y = new Variance().evaluate(pixelsTwo); final double oxy = new Covariance().covariance(pixelsOne, pixelsTwo); final double num = (2 * ux * uy + c1) * (2 * oxy + c2); final double den = (ux * ux + uy * uy + c1) * (o2x + o2y + c2); final double ssim = num / den; return ssim; }
From source file:com.cloudera.oryx.als.computation.LoadRunner.java
public void runLoad() throws InterruptedException { final StorelessUnivariateStatistic recommendedBecause = new Mean(); final StorelessUnivariateStatistic setPreference = new Mean(); final StorelessUnivariateStatistic removePreference = new Mean(); final StorelessUnivariateStatistic ingest = new Mean(); final StorelessUnivariateStatistic refresh = new Mean(); final StorelessUnivariateStatistic estimatePreference = new Mean(); final StorelessUnivariateStatistic mostSimilarItems = new Mean(); final StorelessUnivariateStatistic similarityToItem = new Mean(); final StorelessUnivariateStatistic mostPopularItems = new Mean(); final StorelessUnivariateStatistic recommendToMany = new Mean(); final StorelessUnivariateStatistic recommend = new Mean(); final RandomGenerator random = RandomManager.getRandom(); int numCores = Runtime.getRuntime().availableProcessors(); final int stepsPerWorker = steps / numCores; Collection<Callable<Object>> workers = Lists.newArrayListWithCapacity(numCores); for (int i = 0; i < numCores; i++) { workers.add(new Callable<Object>() { @Override/*from www . j a v a 2 s . co m*/ public Void call() throws Exception { for (int i = 0; i < stepsPerWorker; i++) { double r; String userID; String itemID; String itemID2; float value; synchronized (random) { r = random.nextDouble(); userID = uniqueUserIDs[random.nextInt(uniqueUserIDs.length)]; itemID = uniqueItemIDs[random.nextInt(uniqueItemIDs.length)]; itemID2 = uniqueItemIDs[random.nextInt(uniqueItemIDs.length)]; value = random.nextInt(10); } long stepStart = System.currentTimeMillis(); if (r < 0.05) { client.recommendedBecause(userID, itemID, 10); recommendedBecause.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.07) { client.setPreference(userID, itemID); setPreference.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.08) { client.setPreference(userID, itemID, value); setPreference.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.11) { client.removePreference(userID, itemID); removePreference.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.12) { Reader reader = new StringReader( DelimitedDataUtils.encode(userID, itemID, Float.toString(value)) + '\n'); client.ingest(reader); ingest.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.13) { client.refresh(); refresh.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.14) { client.similarityToItem(itemID, itemID2); similarityToItem.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.15) { client.mostPopularItems(10); mostPopularItems.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.19) { client.estimatePreference(userID, itemID); estimatePreference.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.20) { client.estimateForAnonymous(itemID, new String[] { itemID2 }); estimatePreference.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.25) { client.mostSimilarItems(new String[] { itemID }, 10); mostSimilarItems.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.30) { client.recommendToMany(new String[] { userID, userID }, 10, true, null); recommendToMany.increment(System.currentTimeMillis() - stepStart); } else { client.recommend(userID, 10); recommend.increment(System.currentTimeMillis() - stepStart); } } return null; } }); } log.info("Starting load test..."); long start = System.currentTimeMillis(); ExecutorService executor = Executors.newFixedThreadPool(numCores); Iterable<Future<Object>> futures; try { futures = executor.invokeAll(workers); } finally { ExecutorUtils.shutdownNowAndAwait(executor); } long end = System.currentTimeMillis(); ExecutorUtils.checkExceptions(futures); log.info("Finished {} steps in {}ms", steps, end - start); log.info("recommendedBecause: {}", recommendedBecause.getResult()); log.info("setPreference: {}", setPreference.getResult()); log.info("removePreference: {}", removePreference.getResult()); log.info("ingest: {}", ingest.getResult()); log.info("refresh: {}", refresh.getResult()); log.info("estimatePreference: {}", estimatePreference.getResult()); log.info("mostSimilarItems: {}", mostSimilarItems.getResult()); log.info("similarityToItem: {}", similarityToItem.getResult()); log.info("mostPopularItems: {}", mostPopularItems.getResult()); log.info("recommendToMany: {}", recommendToMany.getResult()); log.info("recommend: {}", recommend.getResult()); }
From source file:net.myrrix.online.eval.ReconstructionEvaluator.java
public EvaluationResult evaluate(File originalDataDir) throws TasteException, IOException, InterruptedException { Preconditions.checkArgument(originalDataDir.exists() && originalDataDir.isDirectory(), "%s is not a directory", originalDataDir); File tempDir = Files.createTempDir(); ServerRecommender recommender = null; try {//from w w w .j av a 2s. co m Multimap<Long, RecommendedItem> data; try { data = readAndCopyDataFiles(originalDataDir, tempDir); } catch (IOException ioe) { throw new TasteException(ioe); } recommender = new ServerRecommender(tempDir); recommender.await(); Generation generation = recommender.getGenerationManager().getCurrentGeneration(); FastByIDMap<float[]> X = generation.getX(); FastByIDMap<float[]> Y = generation.getY(); Mean averageError = new Mean(); // Only compute average over existing entries... for (Map.Entry<Long, RecommendedItem> entry : data.entries()) { long userID = entry.getKey(); long itemID = entry.getValue().getItemID(); // Each of which was a "1" in the factor P matrix double value = SimpleVectorMath.dot(X.get(userID), Y.get(itemID)); // So store abs(1-value), except, don't penalize for reconstructing > 1. Error is 0 in this case. averageError.increment(FastMath.max(0.0, 1.0 - value)); } return new EvaluationResultImpl(averageError.getResult()); } finally { recommender.close(); IOUtils.deleteRecursively(tempDir); } }
From source file:bigtweet.model.ComparingToRealData.java
/** * Update the min distnce , mean and sd in all experiments and the min * distance mean and sd for one experiments with several seeds in this * dataset/*from ww w .ja v a2s . c om*/ * The experiment id is passed */ @Override public void updateMetricsForParametersValues(JSONArray parametersValues, int parametersValuesIndex) { DecimalFormatSymbols otherSymbols = new DecimalFormatSymbols(Locale.US); DecimalFormat df = new DecimalFormat("0.000", otherSymbols); //get distances array for each seed Object obDist[] = seedAndDistances.values().toArray(); double distances[] = new double[obDist.length]; for (int i = 0; i < obDist.length; i++) { distances[i] = (double) obDist[i]; } //get metrics for this experiment and this dataset double minDistance = (new Min()).evaluate(distances); double mean = (new Mean()).evaluate(distances); double sd = (new StandardDeviation()).evaluate(distances); //put in metricsForLastExperiment, JSON object metricsForLastExperiment = new JSONObject();//create new object for the last experiment metricsForLastExperiment.put("minDistance", df.format(minDistance)); metricsForLastExperiment.put("mean", df.format(mean)); metricsForLastExperiment.put("sd", df.format(sd)); JSONObject distancesJSON = new JSONObject(); for (Map.Entry<Long, Double> entry : seedAndDistances.entrySet()) { distancesJSON.put(entry.getKey().toString(), df.format(entry.getValue())); } metricsForLastExperiment.put("randomSeedsAndDistances", distancesJSON); //update metrics for all experiments if (mean < this.bestMean) { //System.out.println(name + " bestmean " + df.format(bestMean).toString() + " mean " + df.format(mean).toString()); bestMean = mean; experimentWithBestMean = parametersValuesIndex; metricsForAllExperiments.put("bestMean", df.format(bestMean)); metricsForAllExperiments.put("experimentWithBestMean", parametersValuesIndex); } if (minDistance < this.bestDistance) { //System.out.println(name + " bestdos " + df.format(bestDistance).toString() + " dis " + df.format(minDistance).toString()); bestDistance = minDistance; experimentWithBestDistance = parametersValuesIndex; metricsForAllExperiments.put("bestDistance", df.format(minDistance)); metricsForAllExperiments.put("experimentWithBestDistance", parametersValuesIndex); } }