List of usage examples for org.apache.commons.math.stat.descriptive DescriptiveStatistics addValue
public void addValue(double v)
From source file:fantail.algorithms.RankingWithBinaryPCT.java
private double getMedian(Instances data, int attIndex) throws Exception { DescriptiveStatistics stats = new DescriptiveStatistics(); for (int i = 0; i < data.numInstances(); i++) { Instance inst = (Instance) data.instance(i); stats.addValue(inst.value(attIndex)); }/* w ww. j a va 2 s. com*/ double median = stats.getPercentile(50); return median; }
From source file:guineu.modules.dataanalysis.variationCoefficient.VariationCoefficientTask.java
private double getvariationCoefficient(Dataset dataset) { DescriptiveStatistics superStats = new DescriptiveStatistics(); DescriptiveStatistics stats = new DescriptiveStatistics(); for (PeakListRow row : dataset.getRows()) { stats.clear();//from ww w . j a va 2 s . com for (String experimentName : dataset.getAllColumnNames()) { Object value = row.getPeak(experimentName); if (value != null && value instanceof Double) { stats.addValue((Double) value); } else { try { stats.addValue(Double.valueOf((String) value)); } catch (Exception e) { } } } if (stats.getMean() > 0) { double value = stats.getStandardDeviation() / stats.getMean(); superStats.addValue(value); } } return superStats.getMean(); }
From source file:com.netxforge.netxstudio.common.math.NativeFunctions.java
public double standardDeviation(double[] range) { assert range != null : new MathException("Range can't be empty"); DescriptiveStatistics stats = new DescriptiveStatistics(); // Add the data from the array for (int i = 0; i < range.length; i++) { stats.addValue(range[i]); }//ww w . j a v a 2 s . c o m return stats.getStandardDeviation(); }
From source file:de.mpicbg.knime.hcs.base.nodes.preproc.OutlierRemoval.java
@Override protected BufferedDataTable[] execute(BufferedDataTable[] inData, ExecutionContext exec) throws Exception { BufferedDataTable input = inData[0]; DataTableSpec inputSpec = input.getDataTableSpec(); // Get the parameter and make sure there all double value columns List<Attribute> parameter = new ArrayList<Attribute>(); for (String item : parameterNames.getIncludeList()) { Attribute attribute = new InputTableAttribute(item, input); if (attribute.getType().isCompatible(DoubleValue.class)) { parameter.add(attribute);/*from www.ja v a2 s . c om*/ } else { logger.warn("The parameter '" + attribute.getName() + "' will not be considered for outlier removal, since it is not compatible to double."); } } // Get the groups defined by the nominal column. Attribute groupingAttribute = new InputTableAttribute(this.groupingColumn.getStringValue(), input); Map<Object, List<DataRow>> subsets = AttributeUtils.splitRowsGeneric(input, groupingAttribute); // Initialize BufferedDataContainer keepContainer = exec.createDataContainer(inputSpec); BufferedDataContainer discartContainer = exec.createDataContainer(inputSpec); int S = subsets.size(); int s = 1; // Outlier analysis for each subset for (Object key : subsets.keySet()) { // Get the subset having all constraints in common List<DataRow> rowSubset = subsets.get(key); // Get the valid values RealMatrix data = extractMatrix(rowSubset, parameter); int N = data.getColumnDimension(); int M = data.getRowDimension(); if (M == 0) { logger.warn("The group '" + key + "' has no valid values and will be removed entirely'"); } else { // Determine upper and lower outlier bounds double[] lowerBound = new double[N]; double[] upperBound = new double[N]; if (method.getStringValue().equals("Boxplot")) { for (int c = 0; c < N; ++c) { RealVector vect = data.getColumnVector(c); DescriptiveStatistics stats = new DescriptiveStatistics(); for (double value : vect.getData()) { stats.addValue(value); } double lowerQuantile = stats.getPercentile(25); double upperQuantile = stats.getPercentile(85); double whisker = factor.getDoubleValue() * Math.abs(lowerQuantile - upperQuantile); lowerBound[c] = lowerQuantile - whisker; upperBound[c] = upperQuantile + whisker; } } else { for (int c = 0; c < N; ++c) { RealVector vect = data.getColumnVector(c); double mean = StatUtils.mean(vect.getData()); double sd = Math.sqrt(StatUtils.variance(vect.getData())); lowerBound[c] = mean - factor.getDoubleValue() * sd; upperBound[c] = mean + factor.getDoubleValue() * sd; } } // Remove The outlier if (rule.getBooleanValue()) { // The row is only discarted if the row is an outlier in all parameter. for (DataRow row : rowSubset) { int c = 0; for (Attribute column : parameter) { DataCell valueCell = row.getCell(((InputTableAttribute) column).getColumnIndex()); // a missing value will be treated as data point inside the bounds if (valueCell.isMissing()) { continue; } Double value = ((DoubleValue) valueCell).getDoubleValue(); if ((value != null) && (lowerBound[c] <= value) && (value <= upperBound[c])) { break; } else { c++; } } if (c != N) { keepContainer.addRowToTable(row); } else { discartContainer.addRowToTable(row); } } } else { // The row is discarted if it has a outlier for at least one parameter. for (DataRow row : rowSubset) { int c = 0; for (Attribute column : parameter) { DataCell valueCell = row.getCell(((InputTableAttribute) column).getColumnIndex()); // a missing value will be treated as data point inside the bounds if (valueCell.isMissing()) { c++; continue; } Double value = ((DoubleValue) valueCell).getDoubleValue(); if ((value != null) && (lowerBound[c] <= value) && (value <= upperBound[c])) { c++; } else { break; } } if (c == N) { keepContainer.addRowToTable(row); } else { discartContainer.addRowToTable(row); } } } } BufTableUtils.updateProgress(exec, s++, S); } keepContainer.close(); discartContainer.close(); return new BufferedDataTable[] { keepContainer.getTable(), discartContainer.getTable() }; }
From source file:com.netxforge.netxstudio.common.math.NativeFunctions.java
public BigDecimal standardDeviation(List<?> range) { assert range != null : new MathException("Range can't be empty"); DescriptiveStatistics stats = new DescriptiveStatistics(); double[] dRange = rangeSelection(range); for (int i = 0; i < dRange.length; i++) { stats.addValue(dRange[i]); }//from ww w . j av a2s . co m return new BigDecimal(stats.getStandardDeviation()); }
From source file:com.gtwm.pb.model.manageData.WordCloud.java
/** * @param textLowerCase// w ww .j ava 2 s . co m * Input text, must be lower case * @param minWeight * Minimum tag weight, e.g. a font size * @param maxWeight * Max. tag weight * @param maxTags * Maximum number of tags to return, -1 for all tags * @param additionalStopWords * Set of words to specifically exclude, in addition to the * standard set [and, not, after, yes, no, ...] */ public WordCloud(String textLowerCase, int minWeight, int maxWeight, int maxTags, Set<String> additionalStopWords) { String[] wordArray = textLowerCase.split("\\W"); Set<String> stopWords = new HashSet<String>(Arrays.asList(stopWordsArray)); for (String additionalStopWord : additionalStopWords) { stopWords.add(additionalStopWord.toLowerCase().trim()); } LancasterStemmer stemmer = new LancasterStemmer(); String wordStem; Frequency frequencies = new Frequency(); for (String wordString : wordArray) { if ((!stopWords.contains(wordString)) && (wordString.length() >= minWordLength)) { wordStem = stemmer.stripSuffixes(wordString); // Record the mapping of the stem to its origin so the most // common origin can be re-introduced when the cloud is // generated this.recordStemOrigin(wordString, wordStem); frequencies.addValue(wordStem); } } // Compute std. dev of frequencies so we can remove outliers DescriptiveStatistics stats = new DescriptiveStatistics(); Iterator freqIt = frequencies.valuesIterator(); long stemFreq; while (freqIt.hasNext()) { stemFreq = frequencies.getCount(freqIt.next()); stats.addValue(stemFreq); } double mean = stats.getMean(); double stdDev = stats.getStandardDeviation(); long minFreq = Long.MAX_VALUE; long maxFreq = 0; // Remove outliers freqIt = frequencies.valuesIterator(); int upperLimit = (int) (mean + (stdDev * 10)); int lowerLimit = (int) (mean - stdDev); if (lowerLimit < 2) { lowerLimit = 2; } int numWords = 0; int numRawWords = wordArray.length; boolean removeLowOutliers = (numRawWords > (maxTags * 10)); while (freqIt.hasNext()) { wordStem = (String) freqIt.next(); stemFreq = frequencies.getCount(wordStem); // For a large input set, remove high and low outliers. // For a smaller set, just high freq. outliers if ((stemFreq > upperLimit) || ((stemFreq < lowerLimit) && removeLowOutliers)) { freqIt.remove(); } else { numWords++; if (stemFreq > maxFreq) { maxFreq = stemFreq; } else if (stemFreq < minFreq) { minFreq = stemFreq; } } } // Cut down to exact required number of tags by removing smallest if (lowerLimit < minFreq) { lowerLimit = (int) minFreq; } if (numWords > maxTags) { while (numWords > maxTags) { freqIt = frequencies.valuesIterator(); SMALLREMOVAL: while (freqIt.hasNext()) { stemFreq = frequencies.getCount(freqIt.next()); if (stemFreq < lowerLimit) { freqIt.remove(); numWords--; if (numWords == maxTags) { break SMALLREMOVAL; } } } int step = (int) ((mean - lowerLimit) / 3); if (step < 1) { step = 1; } lowerLimit += step; } // The new min. freq. may have changed minFreq = Long.MAX_VALUE; freqIt = frequencies.valuesIterator(); while (freqIt.hasNext()) { stemFreq = frequencies.getCount(freqIt.next()); if (stemFreq < minFreq) { minFreq = stemFreq; } } } // Scale and create tag objects double scaleFactor; if (maxFreq == minFreq) { scaleFactor = (double) (maxWeight - minWeight) / 4; // TODO: a realistic // scale factor in this // case } else { scaleFactor = (double) (maxWeight - minWeight) / (maxFreq - minFreq); } freqIt = frequencies.valuesIterator(); int weight; while (freqIt.hasNext()) { wordStem = (String) freqIt.next(); stemFreq = frequencies.getCount(wordStem); // Might still be some left less than the min. threshold if (stemFreq <= minFreq) { weight = minWeight; } else { weight = (int) (Math.ceil((double) (stemFreq - minFreq) * scaleFactor) + minWeight); } SortedSet<WordInfo> origins = this.stemOriginMap.get(wordStem); String mostCommonOrigin = origins.last().getName(); Set<String> synonyms = new TreeSet<String>(); for (WordInfo origin : origins) { synonyms.add(origin.getName()); } WordInfo word = new Word(mostCommonOrigin, weight, synonyms); this.words.add(word); } }
From source file:guineu.modules.dataanalysis.wilcoxontest.WilcoxonTestTask.java
public double[] Ttest(int mol) throws IllegalArgumentException { DescriptiveStatistics stats1 = new DescriptiveStatistics(); DescriptiveStatistics stats2 = new DescriptiveStatistics(); double[] values = new double[3]; String parameter1 = ""; if (parameter == null) { for (int i = 0; i < group1.length; i++) { try { stats1.addValue((Double) this.dataset.getRow(mol).getPeak(group1[i])); } catch (Exception e) { e.printStackTrace();//from w w w . j a v a 2 s .c o m } } for (int i = 0; i < group2.length; i++) { try { stats2.addValue((Double) this.dataset.getRow(mol).getPeak(group2[i])); } catch (Exception e) { e.printStackTrace(); } } } else { try { // Determine groups for selected raw data files List<String> availableParameterValues = dataset.getParameterAvailableValues(parameter); int numberOfGroups = availableParameterValues.size(); if (numberOfGroups > 1) { parameter1 = availableParameterValues.get(0); String parameter2 = availableParameterValues.get(1); for (String sampleName : dataset.getAllColumnNames()) { if (dataset.getParametersValue(sampleName, parameter) != null && dataset.getParametersValue(sampleName, parameter).equals(parameter1)) { try { stats1.addValue((Double) this.dataset.getRow(mol).getPeak(sampleName)); } catch (Exception e) { } } else if (dataset.getParametersValue(sampleName, parameter) != null && dataset.getParametersValue(sampleName, parameter).equals(parameter2)) { try { stats2.addValue((Double) this.dataset.getRow(mol).getPeak(sampleName)); } catch (Exception e) { } } } } else { return null; } } catch (Exception e) { e.printStackTrace(); } } try { final Rengine rEngine; try { rEngine = RUtilities.getREngine(); } catch (Throwable t) { throw new IllegalStateException( "Wilcoxon test requires R but it couldn't be loaded (" + t.getMessage() + ')'); } synchronized (RUtilities.R_SEMAPHORE) { rEngine.eval("x <- 0"); rEngine.eval("y <- 0"); long group1 = rEngine.rniPutDoubleArray(stats1.getValues()); rEngine.rniAssign("x", group1, 0); long group2 = rEngine.rniPutDoubleArray(stats2.getValues()); rEngine.rniAssign("y", group2, 0); /* if(mol == 1){ rEngine.eval("write.csv(x, \"x.csv\")"); rEngine.eval("write.csv(y, \"y.csv\")"); }*/ rEngine.eval("result <- 0"); rEngine.eval("result <- wilcox.test(as.numeric(t(x)),as.numeric(t(y)))"); long e = rEngine.rniParse("result$p.value", 1); long r = rEngine.rniEval(e, 0); REXP x = new REXP(rEngine, r); values[0] = x.asDouble(); } rEngine.end(); setStatus(TaskStatus.FINISHED); } catch (Exception ex) { Logger.getLogger(WilcoxonTestTask.class.getName()).log(Level.SEVERE, null, ex); setStatus(TaskStatus.ERROR); } values[1] = stats1.getMean(); values[2] = stats2.getMean(); return values; }
From source file:edu.usc.goffish.gopher.sample.stats.N_Hop_Stats.java
@Override public void reduce(List<SubGraphMessage> messageList) { if (getSuperStep() == 0) { if (messageList == null || messageList.isEmpty()) { voteToHalt();/*from ww w .j a va 2 s. c o m*/ return; } for (SubGraphMessage msg : messageList) { SubGraphMessage m = new SubGraphMessage(msg.getData()); for (int id : partitions) { sendMessage(id, m); } } } else { DescriptiveStatistics statistics = new DescriptiveStatistics(); for (SubGraphMessage message : messageList) { String data = new String(message.getData()); Double d = Double.parseDouble(data); statistics.addValue(d); } PrintWriter writer = null; try { writer = new PrintWriter(new FileWriter("Hop_Stats.log", true)); System.out.println("LOGGER STD_DIV: " + statistics.getStandardDeviation()); writer.println("LOGGER STD_DIV: " + statistics.getStandardDeviation()); writer.flush(); writer.close(); } catch (IOException e) { e.printStackTrace(); } } System.out.println( "[Gopher]Current Reduce Iteration : " + getIteration() + " Current SuperStep : " + getSuperStep()); }
From source file:hdr_plugin.ImageCompare.java
private void button1ActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_button1ActionPerformed ImagePlus imp = WindowManager.getImage(chcStack.getSelectedItem()); for (int i = 0; i < imp.getNSlices() - 1; i++) { DescriptiveStatistics des = new DescriptiveStatistics(); Object pixels = imp.getImageStack().getPixels(i + 1); Object pixels2 = imp.getImageStack().getPixels(i + 2); for (int j = 0; j < imp.getWidth() * imp.getHeight(); j++) { try { // get pixel value at position j and the current channel int m = ImageJTools.getPixelValue(pixels, j, imp.getType(), 0); int n = ImageJTools.getPixelValue(pixels2, j, imp.getType(), 0); des.addValue((double) n / m); } catch (TypeNotSupportedException ex) { Logger.getLogger(ImageCompare.class.getName()).log(Level.SEVERE, null, ex); }// w ww. j a v a 2 s. co m } System.out.println(des.getMean()); } }
From source file:info.raack.appliancedetection.evaluation.model.EvaluationGroup.java
private void calculateEvaluationMetrics( Map<ApplianceEnergyConsumptionDetectionAlgorithm, List<Evaluation>> evaluationInfo) { for (ApplianceEnergyConsumptionDetectionAlgorithm algorithm : evaluationInfo.keySet()) { // do wattage stats DescriptiveStatistics stats = new DescriptiveStatistics(); List<Evaluation> evaluationList = evaluationInfo.get(algorithm); if (evaluationList.size() == 0) { throw new IllegalArgumentException( "No evaluations for " + algorithm + " in simulation group " + simulationGroup); }//from w w w .ja v a 2 s . c o m for (Evaluation evaluation : evaluationList) { // calculation produces watts stats.addValue((double) (evaluation.getOverallEnergyError()) * (3600.0 / (double) (evaluation.getSimulation().getDurationInSeconds()))); } errorMetrics.put(algorithm, new Double[] { stats.getMean(), stats.getPercentile(50), stats.getStandardDeviation() }); // stats = new DescriptiveStatistics(); evaluationList = evaluationInfo.get(algorithm); if (evaluationList.size() == 0) { throw new IllegalArgumentException( "No evaluations for " + algorithm + " in simulation group " + simulationGroup); } for (Evaluation evaluation : evaluationList) { // calculation produces watts stats.addValue((double) (evaluation.getOverallAccuracy())); } accuracyErrorMetrics.put(algorithm, new Double[] { stats.getMean(), stats.getPercentile(50), stats.getStandardDeviation() }); // stats = new DescriptiveStatistics(); evaluationList = evaluationInfo.get(algorithm); if (evaluationList.size() == 0) { throw new IllegalArgumentException( "No evaluations for " + algorithm + " in simulation group " + simulationGroup); } for (Evaluation evaluation : evaluationList) { // calculation produces watts stats.addValue((double) (evaluation.getStateTransitionAccuracy())); } stateTransitionAccuracyErrorMetrics.put(algorithm, new Double[] { stats.getMean(), stats.getPercentile(50), stats.getStandardDeviation() }); // stats = new DescriptiveStatistics(); evaluationList = evaluationInfo.get(algorithm); if (evaluationList.size() == 0) { throw new IllegalArgumentException( "No evaluations for " + algorithm + " in simulation group " + simulationGroup); } for (Evaluation evaluation : evaluationList) { // calculation produces watts stats.addValue((double) (evaluation.getStateTransitionRecall())); } stateTransitionRecallErrorMetrics.put(algorithm, new Double[] { stats.getMean(), stats.getPercentile(50), stats.getStandardDeviation() }); // stats = new DescriptiveStatistics(); evaluationList = evaluationInfo.get(algorithm); if (evaluationList.size() == 0) { throw new IllegalArgumentException( "No evaluations for " + algorithm + " in simulation group " + simulationGroup); } for (Evaluation evaluation : evaluationList) { // calculation produces watts stats.addValue((double) (evaluation.getStateTransitionPrecision())); } stateTransitionPrecisionErrorMetrics.put(algorithm, new Double[] { stats.getMean(), stats.getPercentile(50), stats.getStandardDeviation() }); } }