List of usage examples for org.apache.commons.math3.stat.descriptive SummaryStatistics getStandardDeviation
public double getStandardDeviation()
From source file:com.civprod.writerstoolbox.testarea.UnsupervisedDiscourseSegmentation.java
public static List<List<String>> segment(Document<?> inDocument, SentenceDetector inSentenceDetector, StringTokenizer inStringTokenizer) { List<String> concatenateTokens = concatenateTokens(inDocument, inSentenceDetector, inStringTokenizer); List<String> stemmAndFilterList = TokenUtil.stemmAndFilterList(concatenateTokens); List<List<String>> splitIntoFixLengthLists = splitIntoFixLengthLists(stemmAndFilterList, 20); List<Counter<String>> counters = splitIntoFixLengthLists.parallelStream() .map((List<String> curSentence) -> CounterUtils.count(curSentence)).collect(Collectors.toList()); List<Double> cosineSimilarity = new ArrayList<>(counters.size() - 20); for (int i = 0; i < (counters.size() - 20); i++) { cosineSimilarity.add(cosineSimilarityStemmedAndFiltered(Counter.join(counters.subList(i, i + 10)), Counter.join(counters.subList(i + 11, i + 20)))); }//from w w w. ja v a 2 s . c om List<Double> valleys = new ArrayList<>(cosineSimilarity.size() - 2); for (int i = 0; i < valleys.size(); i++) { double ya1 = cosineSimilarity.get(i); double ya2 = cosineSimilarity.get(i + 1); double ya3 = cosineSimilarity.get(i + 2); valleys.add((ya1 - ya2) + (ya3 - ya2)); } SummaryStatistics valleyStatistics = valleys.parallelStream().collect(SummaryStatisticCollector.instance); double cutoffThreshold = valleyStatistics.getMean() - valleyStatistics.getStandardDeviation(); int lastLocation = 0; List<Span> spans = new ArrayList<>(1); for (int i = 0; i < valleys.size(); i++) { double curValley = valleys.get(i); if (curValley < cutoffThreshold) { int curLocation = (i + 11) * 20; spans.add(new Span(lastLocation, curLocation)); lastLocation = curLocation; } } spans.add(new Span(lastLocation, concatenateTokens.size())); return spans.parallelStream() .map((Span curSpan) -> concatenateTokens.subList(curSpan.getStart(), curSpan.getEnd())) .collect(Collectors.toList()); }
From source file:com.github.rinde.dynurg.PoissonDynamismExperiment.java
static void createDynamismHistogram(TimeSeriesGenerator generator, long seed, File file, int repetitions) { try {/*from w w w . j a va 2 s.c o m*/ Files.createParentDirs(file); } catch (final IOException e1) { throw new IllegalStateException(e1); } final RandomGenerator rng = new MersenneTwister(seed); final List<Double> values = newArrayList(); final SummaryStatistics ss = new SummaryStatistics(); for (int i = 0; i < repetitions; i++) { final List<Double> times = generator.generate(rng.nextLong()); ss.addValue(times.size()); final double dynamism = Metrics.measureDynamism(times, LENGTH_OF_DAY); values.add(dynamism); } System.out.println( file.getName() + " has #events: mean: " + ss.getMean() + " +- " + ss.getStandardDeviation()); final StringBuilder sb = new StringBuilder(); sb.append(Joiner.on("\n").join(values)); try { Files.write(sb.toString(), file, Charsets.UTF_8); } catch (final IOException e) { throw new IllegalStateException(e); } }
From source file:gr.aueb.cs.nlp.wordtagger.data.structure.features.FeatureBuilder.java
/** * normalize feature vectors of words using mean and stadard devaition * provided the vectora values are/* ww w. ja va 2 s .c o m*/ * higher than 1. * @param words * @param indeces */ private static void normalize(List<Word> words, List<Integer> indeces) { Map<Integer, Double> means = new WeakHashMap<>(); Map<Integer, Double> stds = new WeakHashMap<>(); for (Integer i : indeces) { SummaryStatistics smt = new SummaryStatistics(); for (Word w : words) { smt.addValue(w.getFeatureVec().getValues()[i]); } means.put(i, smt.getMean()); stds.put(i, smt.getStandardDeviation()); } for (Integer i : indeces) { for (Word w : words) { double value = w.getFeatureVec().getValues()[i]; w.getFeatureVec().getValues()[i] = (value - means.get(i)) / stds.get(i); } } }
From source file:net.recommenders.rival.evaluation.statistics.EffectSize.java
/** * Computes Cohen's d, either the classical formulation (dividing the pooled * standard deviation by the sum of the number of samples) or using the * least squares estimation (substracting 2 to the sum of the number of * samples when normalizing the pooled standard deviation). * * @param <V> type of the keys of each map. * @param baselineMetricPerDimension map for the baseline method, one value * for each user (dimension)//from ww w .ja v a2 s . c o m * @param testMetricPerDimension map for the test method, one value for each * user (dimension) * @param doLeastSquares flag to use one formulation or the other (see * description above) * @return the computed Cohen's d as estimation of the effect size.. */ public static <V> double getCohenD(final Map<V, Double> baselineMetricPerDimension, final Map<V, Double> testMetricPerDimension, final boolean doLeastSquares) { SummaryStatistics statsBaseline = new SummaryStatistics(); for (double d : baselineMetricPerDimension.values()) { statsBaseline.addValue(d); } SummaryStatistics statsTest = new SummaryStatistics(); for (double d : testMetricPerDimension.values()) { statsTest.addValue(d); } if (doLeastSquares) { return getCohenDLeastSquares((int) statsBaseline.getN(), statsBaseline.getMean(), statsBaseline.getStandardDeviation(), (int) statsTest.getN(), statsTest.getMean(), statsTest.getStandardDeviation()); } return getCohenD((int) statsBaseline.getN(), statsBaseline.getMean(), statsBaseline.getStandardDeviation(), (int) statsTest.getN(), statsTest.getMean(), statsTest.getStandardDeviation()); }
From source file:model.experiments.WhenDoesAveragingMatters.java
public static void learnedRun(int competitors, Class<? extends AskPricingStrategy> pricing, int weightedAverageSize) { final MacroII macroII = new MacroII(System.currentTimeMillis()); final TripolistScenario scenario1 = new TripolistScenario(macroII); scenario1.setSalesDepartmentType(SalesDepartmentOneAtATime.class); scenario1.setAskPricingStrategy(pricing); scenario1.setControlType(MonopolistScenario.MonopolistScenarioIntegratedControlEnum.MARGINAL_PLANT_CONTROL); scenario1.setAdditionalCompetitors(competitors); scenario1.setWorkersToBeRehiredEveryDay(true); scenario1.setDemandIntercept(102);// ww w . j a v a2 s . c o m scenario1.setSalesPricePreditorStrategy(FixedDecreaseSalesPredictor.class); //assign scenario macroII.setScenario(scenario1); macroII.start(); macroII.schedule.step(macroII); for (Firm firm : scenario1.getCompetitors()) { for (HumanResources hr : firm.getHRs()) hr.setPredictor(new FixedIncreasePurchasesPredictor(0)); final SalesDepartment salesDepartment = firm.getSalesDepartment(UndifferentiatedGoodType.GENERIC); salesDepartment.setPriceAverager(new WeightedPriceAverager(weightedAverageSize)); salesDepartment.setPredictorStrategy(new FixedDecreaseSalesPredictor(0)); } while (macroII.schedule.getTime() < 5000) { macroII.schedule.step(macroII); } SummaryStatistics prices = new SummaryStatistics(); SummaryStatistics quantities = new SummaryStatistics(); for (int j = 0; j < 500; j++) { macroII.schedule.step(macroII); assert !Float.isNaN(macroII.getMarket(UndifferentiatedGoodType.GENERIC).getTodayAveragePrice()); prices.addValue(macroII.getMarket(UndifferentiatedGoodType.GENERIC).getTodayAveragePrice()); quantities.addValue(macroII.getMarket(UndifferentiatedGoodType.GENERIC).getTodayVolume()); } System.out.println(prices.getMean() + " - " + quantities.getMean() + "----" + macroII.seed() + " | " + macroII.getMarket(UndifferentiatedGoodType.GENERIC).getLastDaysAveragePrice()); System.out.println("standard deviations: price : " + prices.getStandardDeviation() + " , quantity: " + quantities.getStandardDeviation()); }
From source file:model.experiments.tuningRuns.CompetitiveAveragingGridSearch.java
public static CompetitiveAveragingResult intervalRuns(float hrWeight, float salesWeight) { SummaryStatistics averageResultingPrice = new SummaryStatistics(); SummaryStatistics averageResultingQuantity = new SummaryStatistics(); SummaryStatistics averageStandardDeviation = new SummaryStatistics(); for (int i = 0; i < 5; i++) { final MacroII macroII = new MacroII(System.currentTimeMillis()); final TripolistScenario scenario1 = new TripolistScenario(macroII); scenario1.setSalesDepartmentType(SalesDepartmentOneAtATime.class); scenario1.setAskPricingStrategy(SalesControlWithFixedInventoryAndPID.class); scenario1.setControlType(/*from w w w .j a v a2 s .c om*/ MonopolistScenario.MonopolistScenarioIntegratedControlEnum.MARGINAL_PLANT_CONTROL); scenario1.setAdditionalCompetitors(4); scenario1.setWorkersToBeRehiredEveryDay(true); scenario1.setDemandIntercept(102); scenario1.setSalesPricePreditorStrategy(FixedDecreaseSalesPredictor.class); //assign scenario macroII.setScenario(scenario1); macroII.start(); macroII.schedule.step(macroII); for (Firm firm : scenario1.getCompetitors()) { for (HumanResources hr : firm.getHRs()) { hr.setPredictor(new FixedIncreasePurchasesPredictor(0)); hr.setPriceAverager(new AveragerOverSmallIntervalOnly(hrWeight)); } firm.getSalesDepartment(UndifferentiatedGoodType.GENERIC) .setPriceAverager(new AveragerOverSmallIntervalOnly(salesWeight)); firm.getSalesDepartment(UndifferentiatedGoodType.GENERIC) .setPredictorStrategy(new FixedDecreaseSalesPredictor(0)); } while (macroII.schedule.getTime() < 10000) { macroII.schedule.step(macroII); } SummaryStatistics prices = new SummaryStatistics(); SummaryStatistics quantities = new SummaryStatistics(); for (int j = 0; j < 500; j++) { macroII.schedule.step(macroII); prices.addValue(macroII.getMarket(UndifferentiatedGoodType.GENERIC).getTodayAveragePrice()); quantities.addValue(macroII.getMarket(UndifferentiatedGoodType.GENERIC).getTodayVolume()); } //okay? averageResultingPrice.addValue(prices.getMean()); averageResultingQuantity.addValue(quantities.getMean()); averageStandardDeviation.addValue(prices.getStandardDeviation()); } //okay? return new CompetitiveAveragingResult(averageResultingPrice.getMean(), averageResultingQuantity.getMean(), averageStandardDeviation.getMean()); }
From source file:model.experiments.tuningRuns.CompetitiveAveragingGridSearch.java
public static CompetitiveAveragingResult weightedRun(int hrDays, int salesDays, boolean decoratedHr, boolean decoratedSales) { SummaryStatistics averageResultingPrice = new SummaryStatistics(); SummaryStatistics averageResultingQuantity = new SummaryStatistics(); SummaryStatistics averageStandardDeviation = new SummaryStatistics(); for (int i = 0; i < 5; i++) { final MacroII macroII = new MacroII(System.currentTimeMillis()); final TripolistScenario scenario1 = new TripolistScenario(macroII); scenario1.setSalesDepartmentType(SalesDepartmentOneAtATime.class); scenario1.setAskPricingStrategy(SalesControlWithFixedInventoryAndPID.class); scenario1.setControlType(/* ww w .j a v a 2 s . com*/ MonopolistScenario.MonopolistScenarioIntegratedControlEnum.MARGINAL_PLANT_CONTROL); scenario1.setAdditionalCompetitors(4); scenario1.setWorkersToBeRehiredEveryDay(true); scenario1.setDemandIntercept(102); scenario1.setSalesPricePreditorStrategy(FixedDecreaseSalesPredictor.class); //assign scenario macroII.setScenario(scenario1); macroII.start(); macroII.schedule.step(macroII); for (Firm firm : scenario1.getCompetitors()) { for (HumanResources hr : firm.getHRs()) { hr.setPredictor(new FixedIncreasePurchasesPredictor(0)); PriceAverager priceAverager = new WeightedPriceAverager(hrDays); if (decoratedHr) priceAverager = new NoTradingOverrideAveragerDecorator(priceAverager); hr.setPriceAverager(priceAverager); } PriceAverager priceAverager = new WeightedPriceAverager(salesDays); if (decoratedSales) priceAverager = new NoTradingOverrideAveragerDecorator(priceAverager); firm.getSalesDepartment(UndifferentiatedGoodType.GENERIC).setPriceAverager(priceAverager); firm.getSalesDepartment(UndifferentiatedGoodType.GENERIC) .setPredictorStrategy(new FixedDecreaseSalesPredictor(0)); } while (macroII.schedule.getTime() < 10000) { macroII.schedule.step(macroII); } SummaryStatistics prices = new SummaryStatistics(); SummaryStatistics quantities = new SummaryStatistics(); for (int j = 0; j < 500; j++) { macroII.schedule.step(macroII); prices.addValue(macroII.getMarket(UndifferentiatedGoodType.GENERIC).getTodayAveragePrice()); quantities.addValue(macroII.getMarket(UndifferentiatedGoodType.GENERIC).getTodayVolume()); } //okay? averageResultingPrice.addValue(prices.getMean()); averageResultingQuantity.addValue(quantities.getMean()); averageStandardDeviation.addValue(prices.getStandardDeviation()); } //okay? return new CompetitiveAveragingResult(averageResultingPrice.getMean(), averageResultingQuantity.getMean(), averageStandardDeviation.getMean()); }
From source file:model.experiments.tuningRuns.CompetitiveAveragingGridSearch.java
public static CompetitiveAveragingResult exponentialRuns(float hrWeight, PriceAverager.NoTradingDayPolicy hrPolicy, float salesWeight, PriceAverager.NoTradingDayPolicy salesPolicy, int maximizerAveragePeriod) { SummaryStatistics averageResultingPrice = new SummaryStatistics(); SummaryStatistics averageResultingQuantity = new SummaryStatistics(); SummaryStatistics averageStandardDeviation = new SummaryStatistics(); for (int i = 0; i < 5; i++) { final MacroII macroII = new MacroII(i); final TripolistScenario scenario1 = new TripolistScenario(macroII); scenario1.setSalesDepartmentType(SalesDepartmentOneAtATime.class); scenario1.setAskPricingStrategy(SalesControlWithFixedInventoryAndPID.class); scenario1.setControlType(// w ww . j av a 2s. com MonopolistScenario.MonopolistScenarioIntegratedControlEnum.MARGINAL_PLANT_CONTROL); scenario1.setAdditionalCompetitors(4); scenario1.setWorkersToBeRehiredEveryDay(true); scenario1.setDemandIntercept(102); scenario1.setSalesPricePreditorStrategy(FixedDecreaseSalesPredictor.class); //assign scenario macroII.setScenario(scenario1); macroII.start(); macroII.schedule.step(macroII); for (Firm firm : scenario1.getCompetitors()) { for (HumanResources hr : firm.getHRs()) { hr.setPredictor(new FixedIncreasePurchasesPredictor(0)); hr.setPriceAverager(new ExponentialPriceAverager(hrWeight, hrPolicy)); } firm.getSalesDepartment(UndifferentiatedGoodType.GENERIC) .setPriceAverager(new ExponentialPriceAverager(salesWeight, salesPolicy)); firm.getSalesDepartment(UndifferentiatedGoodType.GENERIC) .setPredictorStrategy(new FixedDecreaseSalesPredictor(0)); } for (final PlantControl control : scenario1.getMaximizers()) { ((MarginalPlantControl) control).getMaximizer() .setHowManyDaysBeforeEachCheck(maximizerAveragePeriod); } while (macroII.schedule.getTime() < 10000) { macroII.schedule.step(macroII); } SummaryStatistics prices = new SummaryStatistics(); SummaryStatistics quantities = new SummaryStatistics(); for (int j = 0; j < 500; j++) { macroII.schedule.step(macroII); prices.addValue(macroII.getMarket(UndifferentiatedGoodType.GENERIC).getTodayAveragePrice()); quantities.addValue(macroII.getMarket(UndifferentiatedGoodType.GENERIC).getTodayVolume()); } //okay? averageResultingPrice.addValue(prices.getMean()); averageResultingQuantity.addValue(quantities.getMean()); averageStandardDeviation.addValue(prices.getStandardDeviation()); } //okay? return new CompetitiveAveragingResult(averageResultingPrice.getMean(), averageResultingQuantity.getMean(), averageStandardDeviation.getMean()); }
From source file:ijfx.core.timer.DefaultTimer.java
protected String getRow(String id, SummaryStatistics statics) { return String.format("%30s | %6.0fms | %6.0fms | %8d\n", id.length() > 22 ? id.substring(0, 21) : id, statics.getMean(), statics.getStandardDeviation(), statics.getN()); }
From source file:io.fabric8.example.stddev.http.StdDevProcessor.java
@Override public void process(Exchange exchange) throws Exception { String message = exchange.getIn().getBody(String.class); ObjectMapper objectMapper = new ObjectMapper(); TypeFactory typeFactory = objectMapper.getTypeFactory(); List<Double> values = objectMapper.readValue(message, typeFactory.constructCollectionType(List.class, Double.class)); SummaryStatistics summaryStatistics = new SummaryStatistics(); List<Double> list = new ObjectMapper().readValue(message, List.class); for (Double value : list) { summaryStatistics.addValue(value); }// ww w . ja va2 s .c o m String stdDev = Double.toString(summaryStatistics.getStandardDeviation()); exchange.getOut().setBody(stdDev); }