List of usage examples for org.apache.commons.math3.stat.descriptive SummaryStatistics getSum
public double getSum()
From source file:co.turnus.common.util.CommonDataUtil.java
public static StatisticalData createFrom(SummaryStatistics summary) { StatisticalData data = CommonFactory.eINSTANCE.createStatisticalData(); if (summary.getN() != 0) { data.setMax(summary.getMax());// ww w .ja v a 2 s . c om data.setMin(summary.getMin()); data.setSamples(summary.getN()); data.setSum(summary.getSum()); data.setVariance(summary.getVariance()); data.setMean(summary.getMean()); } return data; }
From source file:gr.cti.android.experimentation.controller.api.HistoryController.java
/** * Parse a time instant and create a TempReading object. * * @param millis the millis of the timestamp. * @param function the function to aggregate. * @param statistics the data values// w w w . j a v a 2 s .c om * @return the aggregated TempReading for this time instant. */ private TempReading parse(final long millis, final String function, SummaryStatistics statistics) { final Double value; switch (function) { case "avg": value = statistics.getMean(); break; case "max": value = statistics.getMax(); break; case "min": value = statistics.getMin(); break; case "var": value = statistics.getVariance(); break; case "sum": value = statistics.getSum(); break; default: value = statistics.getMean(); } return new TempReading(millis, value); }
From source file:fr.gael.dhus.server.http.valve.processings.ProcessingValve.java
protected void checkAccess(UserKey user, Cache window) throws ProcessingQuotaException { ProcessingInformation pi = user.getPi(); if (isWhiteListed(pi) || isInternal(pi)) { return;//from w w w.ja v a 2 s. c o m } SummaryStatistics cpu_time_stats = new SummaryStatistics(); SummaryStatistics user_time_stats = new SummaryStatistics(); SummaryStatistics memory_stats = new SummaryStatistics(); long now = System.nanoTime(); for (Object o : window.getKeysWithExpiryCheck()) { Long date_ns = (Long) o; if ((now - date_ns) > sToNs(getTimeWindow())) { // Cache returns element out of the expected time window // (delta is {} ns)." (now-date_ns) // This can happen when the cache settings are modified // on the fly by JMX. continue; } ProcessingInformation proc = (ProcessingInformation) window.get(o).getObjectValue(); if (proc.getCpuTimeNs() != null) { cpu_time_stats.addValue(proc.getCpuTimeNs().doubleValue()); } if (proc.getUserTimeNs() != null) { user_time_stats.addValue(proc.getUserTimeNs().doubleValue()); } if (proc.getMemoryUsage() != null) { memory_stats.addValue(proc.getMemoryUsage().doubleValue()); } } String username = pi.getUsername(); // Checks the system CPU time used in the time frame if (checkParameter(getMaxElapsedTimePerUserPerWindow()) && cpu_time_stats.getSum() > sToNs(getMaxElapsedTimePerUserPerWindow())) { StringBuilder sb = new StringBuilder(); sb.append("CPU usage quota exceeded (").append(formatMn(sToMn(getMaxElapsedTimePerUserPerWindow()))) .append("mn per period of ").append(formatMn(sToMn(getTimeWindow()))) .append("mn) - please wait and retry."); LOGGER.warn("[{}] CPU usage quota exceeded: {} (max={})", username, formatInterval((long) cpu_time_stats.getSum()), formatInterval((long) getMaxElapsedTimePerUserPerWindow() * 1000000000)); throw new ProcessingQuotaException(sb.toString()); } // Checks the user CPU time used in the time frame if (checkParameter(getMaxElapsedTimePerUserPerWindow()) && user_time_stats.getSum() >= sToNs(getMaxElapsedTimePerUserPerWindow())) { StringBuilder sb = new StringBuilder(); sb.append("User CPU usage quota exceeded (") .append(formatMn(sToMn(getMaxElapsedTimePerUserPerWindow()))).append("mn per period of ") .append(formatMn(sToMn(getTimeWindow()))).append("mn) - please wait and retry."); LOGGER.warn("[{}] User CPU usage quota exceeded: {} (max={})", username, formatInterval((long) user_time_stats.getSum()), formatInterval((long) getMaxElapsedTimePerUserPerWindow() * 1000000000)); throw new ProcessingQuotaException(sb.toString()); } // Checks the total memory used in the time frame if (checkParameter(getMaxUsedMemoryPerUserPerWindow()) && memory_stats.getSum() >= getMaxUsedMemoryPerUserPerWindow()) { StringBuilder sb = new StringBuilder(); sb.append("Memory quota exceeded (").append(formatSize(getMaxUsedMemoryPerUserPerWindow())) .append(" used in a period of ").append(formatMn(sToMn(getTimeWindow()))) .append("mn) - please wait and retry."); LOGGER.warn("[{}] Memory quota exceeded: {} (max={})", username, formatSize((long) memory_stats.getSum()), formatSize((long) getMaxUsedMemoryPerUserPerWindow())); throw new ProcessingQuotaException(sb.toString()); } // Checks the number of request in the time frame if (checkParameter(getMaxRequestNumberPerUserPerWindow()) && user_time_stats.getN() >= getMaxRequestNumberPerUserPerWindow()) { StringBuilder sb = new StringBuilder(); sb.append("Maximum number of request exceeded (").append(getMaxRequestNumberPerUserPerWindow()) .append("max calls in a period of ").append(formatMn(sToMn(getTimeWindow()))) .append("mn) - please wait and retry."); LOGGER.warn("[{}] Maximum number of request exceeded: {} (max={})", username, user_time_stats.getN(), getMaxRequestNumberPerUserPerWindow()); throw new ProcessingQuotaException(sb.toString()); } LOGGER.info("Time Window cumuls for user {}:{} cpu_time={}mn,user_time={}mn,memory={}", username, user_time_stats.getN(), formatMn(nsToMn(cpu_time_stats.getSum())), formatMn(nsToMn(user_time_stats.getSum())), formatSize((long) memory_stats.getSum())); }
From source file:org.alfresco.repo.cache.InMemoryCacheStatistics.java
@Override public void add(String cacheName, TransactionStats txStats) { boolean registerCacheStats = false; WriteLock writeLock = getWriteLock(cacheName); writeLock.lock();/* w w w . j a va 2 s.c om*/ try { // Are we adding new stats for a previously unseen cache? registerCacheStats = !cacheToStatsMap.containsKey(cacheName); if (registerCacheStats) { // There are no statistics yet for this cache. cacheToStatsMap.put(cacheName, new HashMap<OpType, OperationStats>()); } Map<OpType, OperationStats> cacheStats = cacheToStatsMap.get(cacheName); for (OpType opType : OpType.values()) { SummaryStatistics txOpSummary = txStats.getTimings(opType); long count = txOpSummary.getN(); double totalTime = txOpSummary.getSum(); OperationStats oldStats = cacheStats.get(opType); OperationStats newStats; if (oldStats == null) { newStats = new OperationStats(totalTime, count); } else { newStats = new OperationStats(oldStats, totalTime, count); } cacheStats.put(opType, newStats); } } finally { writeLock.unlock(); } if (registerCacheStats) { // We've added stats for a previously unseen cache, raise an event // so that an MBean for the cache may be registered, for example. applicationContext.publishEvent(new CacheStatisticsCreated(this, cacheName)); } }
From source file:org.apache.solr.client.solrj.io.eval.HistogramEvaluator.java
@Override public Object doWork(Object... values) throws IOException { if (Arrays.stream(values).anyMatch(item -> null == item)) { return null; }/*ww w.ja va 2 s . co m*/ List<?> sourceValues; Integer bins = 10; if (values.length >= 1) { sourceValues = values[0] instanceof List<?> ? (List<?>) values[0] : Arrays.asList(values[0]); if (values.length >= 2) { if (values[1] instanceof Number) { bins = ((Number) values[1]).intValue(); } else { throw new IOException(String.format(Locale.ROOT, "Invalid expression %s - if second parameter is provided then it must be a valid number but found %s instead", toExpression(constructingFactory), values[1].getClass().getSimpleName())); } } } else { throw new IOException( String.format(Locale.ROOT, "Invalid expression %s - expecting at least one value but found %d", toExpression(constructingFactory), containedEvaluators.size())); } EmpiricalDistribution distribution = new EmpiricalDistribution(bins); distribution.load( ((List<?>) sourceValues).stream().mapToDouble(value -> ((Number) value).doubleValue()).toArray()); ; List<Tuple> histogramBins = new ArrayList<>(); for (SummaryStatistics binSummary : distribution.getBinStats()) { Map<String, Number> map = new HashMap<>(); map.put("max", binSummary.getMax()); map.put("mean", binSummary.getMean()); map.put("min", binSummary.getMin()); map.put("stdev", binSummary.getStandardDeviation()); map.put("sum", binSummary.getSum()); map.put("N", binSummary.getN()); map.put("var", binSummary.getVariance()); map.put("cumProb", distribution.cumulativeProbability(binSummary.getMean())); map.put("prob", distribution.probability(binSummary.getMin(), binSummary.getMax())); histogramBins.add(new Tuple(map)); } return histogramBins; }
From source file:org.apache.tika.eval.AbstractProfiler.java
/** * Checks to see if metadata is null or content is empty (null or only whitespace). * If any of these, then this does no processing, and the fileId is not * entered into the content table./*ww w .ja v a2 s. c o m*/ * * @param fileId * @param m * @param fieldName * @param contentsTable */ protected void writeContentData(String fileId, Metadata m, String fieldName, TableInfo contentsTable) throws IOException { if (m == null) { return; } Map<Cols, String> data = new HashMap<>(); String content = getContent(m, maxContentLength, data); if (content == null || content.trim().length() == 0) { return; } tokenCounter.clear(fieldName); tokenCounter.add(fieldName, content); data.put(Cols.ID, fileId); data.put(Cols.CONTENT_LENGTH, Integer.toString(content.length())); langid(m, data); String langid = data.get(Cols.LANG_ID_1); langid = (langid == null) ? "" : langid; writeTokenCounts(data, fieldName, tokenCounter); CommonTokenResult commonTokenResult = null; try { commonTokenResult = commonTokenCountManager.countTokenOverlaps(langid, tokenCounter.getTokens(fieldName)); } catch (IOException e) { LOG.error("{}", e.getMessage(), e); } data.put(Cols.COMMON_TOKENS_LANG, commonTokenResult.getLangCode()); data.put(Cols.NUM_COMMON_TOKENS, Integer.toString(commonTokenResult.getCommonTokens())); TokenStatistics tokenStatistics = tokenCounter.getTokenStatistics(fieldName); data.put(Cols.NUM_UNIQUE_TOKENS, Integer.toString(tokenStatistics.getTotalUniqueTokens())); data.put(Cols.NUM_TOKENS, Integer.toString(tokenStatistics.getTotalTokens())); data.put(Cols.NUM_ALPHABETIC_TOKENS, Integer.toString(commonTokenResult.getAlphabeticTokens())); data.put(Cols.TOKEN_ENTROPY_RATE, Double.toString(tokenStatistics.getEntropy())); SummaryStatistics summStats = tokenStatistics.getSummaryStatistics(); data.put(Cols.TOKEN_LENGTH_SUM, Integer.toString((int) summStats.getSum())); data.put(Cols.TOKEN_LENGTH_MEAN, Double.toString(summStats.getMean())); data.put(Cols.TOKEN_LENGTH_STD_DEV, Double.toString(summStats.getStandardDeviation())); unicodeBlocks(m, data); try { writer.writeRow(contentsTable, data); } catch (IOException e) { throw new RuntimeException(e); } }
From source file:org.apache.tika.eval.tokens.TokenStatistics.java
@Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; TokenStatistics that = (TokenStatistics) o; if (totalTokens != that.totalTokens) return false; if (totalUniqueTokens != that.totalUniqueTokens) return false; if (!doubleEquals(that.entropy, entropy)) return false; // Probably incorrect - comparing Object[] arrays with Arrays.equals if (!Arrays.equals(topN, that.topN)) return false; SummaryStatistics thatS = ((TokenStatistics) o).summaryStatistics; if (summaryStatistics.getN() != thatS.getN()) return false; //if both have n==0, don't bother with the stats if (summaryStatistics.getN() == 0L) return true; //TODO: consider adding others... if (!doubleEquals(summaryStatistics.getGeometricMean(), thatS.getGeometricMean())) return false; if (!doubleEquals(summaryStatistics.getMax(), thatS.getMax())) return false; if (!doubleEquals(summaryStatistics.getMean(), thatS.getMean())) return false; if (!doubleEquals(summaryStatistics.getMin(), thatS.getMin())) return false; if (!doubleEquals(summaryStatistics.getSum(), thatS.getSum())) return false; if (!doubleEquals(summaryStatistics.getStandardDeviation(), thatS.getStandardDeviation())) return false; return true;//from w ww. j av a 2 s. c o m }
From source file:org.apereo.portal.events.aggr.stat.JpaStatisticalSummary.java
/** * Returns true iff <code>object</code> is a * <code>SummaryStatistics</code> instance and all statistics have the * same values as this.// w w w . j ava2 s . com * @param object the object to test equality against. * @return true if object equals this */ @Override public boolean equals(Object object) { if (object == this) { return true; } if (object instanceof SummaryStatistics == false) { return false; } SummaryStatistics stat = (SummaryStatistics) object; return Precision.equalsIncludingNaN(stat.getGeometricMean(), getGeometricMean()) && Precision.equalsIncludingNaN(stat.getMax(), getMax()) && Precision.equalsIncludingNaN(stat.getMean(), getMean()) && Precision.equalsIncludingNaN(stat.getMin(), getMin()) && Precision.equalsIncludingNaN(stat.getN(), getN()) && Precision.equalsIncludingNaN(stat.getSum(), getSum()) && Precision.equalsIncludingNaN(stat.getSumsq(), getSumsq()) && Precision.equalsIncludingNaN(stat.getVariance(), getVariance()); }
From source file:org.calrissian.accumulorecipes.metricsstore.ext.stats.impl.AccumuloStatsMetricStoreTest.java
@Test public void testStatisticAccuracy() throws Exception { AccumuloStatsMetricStore metricStore = new AccumuloStatsMetricStore(getConnector()); Random random = new Random(); List<Long> sampleData = asList((long) random.nextInt(10000), (long) random.nextInt(10000), (long) random.nextInt(10000), (long) random.nextInt(10000), (long) random.nextInt(10000)); //use commons math as a SummaryStatistics sumStats = new SummaryStatistics(); for (long num : sampleData) sumStats.addValue(num);//from w w w. j a va 2s . c o m final long timestamp = System.currentTimeMillis(); Iterable<Metric> testData = transform(sampleData, new Function<Long, Metric>() { @Override public Metric apply(Long num) { return new Metric(timestamp, "group", "type", "name", "", num); } }); metricStore.save(testData); List<Stats> stats = newArrayList(metricStore.queryStats(new Date(0), new Date(), "group", "type", "name", MetricTimeUnit.MINUTES, new Auths())); assertEquals(1, stats.size()); Stats stat = stats.get(0); assertEquals(sumStats.getMin(), stat.getMin(), Double.MIN_NORMAL); assertEquals(sumStats.getMax(), stat.getMax(), Double.MIN_NORMAL); assertEquals(sumStats.getSum(), stat.getSum(), Double.MIN_NORMAL); assertEquals(sumStats.getN(), stat.getCount(), Double.MIN_NORMAL); assertEquals(sumStats.getMean(), stat.getMean(), Double.MIN_NORMAL); assertEquals(sumStats.getPopulationVariance(), stat.getVariance(), 0.00000001); assertEquals(sumStats.getVariance(), stat.getVariance(true), 0.00000001); assertEquals(sqrt(sumStats.getPopulationVariance()), stat.getStdDev(), 0.00000001); assertEquals(sumStats.getStandardDeviation(), stat.getStdDev(true), 0.00000001); }
From source file:org.eclipse.dataset.AbstractDataset.java
/** * Calculate summary statistics for a dataset along an axis * @param ignoreNaNs if true, ignore NaNs * @param ignoreInfs if true, ignore infinities * @param axis/*from w ww. ja v a 2 s .c o m*/ */ protected void calculateSummaryStats(final boolean ignoreNaNs, final boolean ignoreInfs, final int axis) { int rank = getRank(); int[] oshape = getShape(); int alen = oshape[axis]; oshape[axis] = 1; int[] nshape = new int[rank - 1]; for (int i = 0; i < axis; i++) { nshape[i] = oshape[i]; } for (int i = axis + 1; i < rank; i++) { nshape[i - 1] = oshape[i]; } final int dtype = getDtype(); IntegerDataset count = new IntegerDataset(nshape); Dataset max = DatasetFactory.zeros(nshape, dtype); Dataset min = DatasetFactory.zeros(nshape, dtype); IntegerDataset maxIndex = new IntegerDataset(nshape); IntegerDataset minIndex = new IntegerDataset(nshape); Dataset sum = DatasetFactory.zeros(nshape, getLargestDType(dtype)); DoubleDataset mean = new DoubleDataset(nshape); DoubleDataset var = new DoubleDataset(nshape); IndexIterator qiter = max.getIterator(true); int[] qpos = qiter.getPos(); int[] spos = oshape.clone(); while (qiter.hasNext()) { int i = 0; for (; i < axis; i++) { spos[i] = qpos[i]; } spos[i++] = 0; for (; i < rank; i++) { spos[i] = qpos[i - 1]; } final SummaryStatistics stats = new SummaryStatistics(); double amax = Double.NEGATIVE_INFINITY; double amin = Double.POSITIVE_INFINITY; boolean hasNaNs = false; if (ignoreNaNs) { for (int j = 0; j < alen; j++) { spos[axis] = j; final double val = getDouble(spos); if (Double.isNaN(val)) { hasNaNs = true; continue; } else if (ignoreInfs && Double.isInfinite(val)) { continue; } if (val > amax) { amax = val; } if (val < amin) { amin = val; } stats.addValue(val); } } else { for (int j = 0; j < alen; j++) { spos[axis] = j; final double val = getDouble(spos); if (hasNaNs) { if (!Double.isNaN(val)) stats.addValue(0); continue; } if (Double.isNaN(val)) { amax = Double.NaN; amin = Double.NaN; hasNaNs = true; } else if (ignoreInfs && Double.isInfinite(val)) { continue; } else { if (val > amax) { amax = val; } if (val < amin) { amin = val; } } stats.addValue(val); } } count.setAbs(qiter.index, (int) stats.getN()); max.setObjectAbs(qiter.index, amax); min.setObjectAbs(qiter.index, amin); boolean fmax = false; boolean fmin = false; if (hasNaNs) { if (ignoreNaNs) { for (int j = 0; j < alen; j++) { spos[axis] = j; final double val = getDouble(spos); if (Double.isNaN(val)) continue; if (!fmax && val == amax) { maxIndex.setAbs(qiter.index, j); fmax = true; if (fmin) break; } if (!fmin && val == amin) { minIndex.setAbs(qiter.index, j); fmin = true; if (fmax) break; } } } else { for (int j = 0; j < alen; j++) { spos[axis] = j; final double val = getDouble(spos); if (Double.isNaN(val)) { maxIndex.setAbs(qiter.index, j); minIndex.setAbs(qiter.index, j); break; } } } } else { for (int j = 0; j < alen; j++) { spos[axis] = j; final double val = getDouble(spos); if (!fmax && val == amax) { maxIndex.setAbs(qiter.index, j); fmax = true; if (fmin) break; } if (!fmin && val == amin) { minIndex.setAbs(qiter.index, j); fmin = true; if (fmax) break; } } } sum.setObjectAbs(qiter.index, stats.getSum()); mean.setAbs(qiter.index, stats.getMean()); var.setAbs(qiter.index, stats.getVariance()); } setStoredValue(storeName(ignoreNaNs, ignoreInfs, STORE_COUNT + "-" + axis), count); storedValues.put(storeName(ignoreNaNs, ignoreInfs, STORE_MAX + "-" + axis), max); storedValues.put(storeName(ignoreNaNs, ignoreInfs, STORE_MIN + "-" + axis), min); storedValues.put(storeName(ignoreNaNs, ignoreInfs, STORE_SUM + "-" + axis), sum); storedValues.put(storeName(ignoreNaNs, ignoreInfs, STORE_MEAN + "-" + axis), mean); storedValues.put(storeName(ignoreNaNs, ignoreInfs, STORE_VAR + "-" + axis), var); storedValues.put(storeName(ignoreNaNs, ignoreInfs, STORE_MAX + STORE_INDEX + "-" + axis), maxIndex); storedValues.put(storeName(ignoreNaNs, ignoreInfs, STORE_MIN + STORE_INDEX + "-" + axis), minIndex); }