List of usage examples for org.apache.commons.math3.stat.descriptive DescriptiveStatistics getSkewness
public double getSkewness()
From source file:com.itemanalysis.jmetrik.stats.descriptives.DescriptiveAnalysis.java
public void publishTable(VariableAttributes v) { TextTable table = null;/*from w w w. j a va2 s . c o m*/ TextTableColumnFormat[] cformats = new TextTableColumnFormat[2]; cformats[0] = new TextTableColumnFormat(); cformats[0].setStringFormat(15, TextTableColumnFormat.OutputAlignment.LEFT); cformats[1] = new TextTableColumnFormat(); cformats[1].setDoubleFormat(10, 4, TextTableColumnFormat.OutputAlignment.RIGHT); DescriptiveStatistics temp = data.get(v); table = new TextTable(); table.addAllColumnFormats(cformats, 17); table.getRowAt(0).addHeader(0, 2, v.getName().toString(), TextTablePosition.CENTER); table.getRowAt(1).addHorizontalRule(0, 2, "="); table.getRowAt(2).addHeader(0, 1, "Statistic", TextTablePosition.CENTER); table.getRowAt(2).addHeader(1, 1, "Value", TextTablePosition.CENTER); table.getRowAt(3).addHorizontalRule(0, 2, "-"); table.addStringAt(4, 0, "N"); table.addDoubleAt(4, 1, maxProgress); table.addStringAt(5, 0, "Valid N"); table.addDoubleAt(5, 1, temp.getN()); table.addStringAt(6, 0, "Min"); table.addDoubleAt(6, 1, temp.getMin()); table.addStringAt(7, 0, "Max"); table.addDoubleAt(7, 1, temp.getMax()); table.addStringAt(8, 0, "Mean"); table.addDoubleAt(8, 1, temp.getMean()); table.addStringAt(9, 0, "Std. Dev."); table.addDoubleAt(9, 1, temp.getStandardDeviation()); table.addStringAt(10, 0, "Skewness"); table.addDoubleAt(10, 1, temp.getSkewness()); table.addStringAt(11, 0, "Kurtosis"); table.addDoubleAt(11, 1, temp.getKurtosis()); table.addStringAt(12, 0, "First Quartile"); table.addDoubleAt(12, 1, temp.getPercentile(25)); table.addStringAt(13, 0, "Median"); table.addDoubleAt(13, 1, temp.getPercentile(50)); table.addStringAt(14, 0, "Third Quartile"); table.addDoubleAt(14, 1, temp.getPercentile(75)); table.addStringAt(15, 0, "IQR"); table.addDoubleAt(15, 1, temp.getPercentile(75) - temp.getPercentile(25)); table.getRowAt(16).addHorizontalRule(0, 2, "="); publish(table.toString() + "\n"); }
From source file:info.financialecology.finance.utilities.datastruct.VersatileTimeSeries.java
/** * Computes the skewness of the data points in this time series. * //from ww w.j av a 2s.c o m * @return the skewness of this time series */ public double operatorSkewness() { DescriptiveStatistics stats = new DescriptiveStatistics(); for (int i = 0; i < this.getItemCount(); i++) stats.addValue(getValue(i).doubleValue()); return stats.getSkewness(); }
From source file:com.screenslicer.core.scrape.type.ComparableNode.java
public ComparableNode(final Node node) { this.node = node; List<Node> separated = node.childNodes(); int children = 0; int childBlocks = 0; int childFormatting = 0; int childContent = 0; int childItems = 0; int childDecoration = 0; int anchorChildren = 0; int textChildren = 0; int anchorTextChildren = 0; int anchorChildItems = 0; int textChildItems = 0; int anchorTextChildItems = 0; int itemChars = 0; int itemAnchorChars = 0; List<String> firstChildTags = null; List<List<String>> orderedTags = new ArrayList<List<String>>(); List<String> allChildTags = new ArrayList<String>(); ArrayList<List<String>> childTags = new ArrayList<List<String>>(); boolean childrenConsistent = true; String childName = null;// w ww . ja va 2 s .com boolean childrenSame = true; double avgChildLengthDouble = 0d; int nodeStrLen = Util.trimmedLen(node.toString()); DescriptiveStatistics statAnchorChars = new DescriptiveStatistics(); DescriptiveStatistics statAnchors = new DescriptiveStatistics(); DescriptiveStatistics statChars = new DescriptiveStatistics(); DescriptiveStatistics statDescendants = new DescriptiveStatistics(); DescriptiveStatistics statFields = new DescriptiveStatistics(); DescriptiveStatistics statLevels = new DescriptiveStatistics(); DescriptiveStatistics statLongestField = new DescriptiveStatistics(); DescriptiveStatistics statNonAnchorChars = new DescriptiveStatistics(); DescriptiveStatistics statTextAnchors = new DescriptiveStatistics(); DescriptiveStatistics statStrLen = new DescriptiveStatistics(); DescriptiveStatistics statItemChars = new DescriptiveStatistics(); DescriptiveStatistics statItemAnchorChars = new DescriptiveStatistics(); for (Node child : separated) { if (!Util.isEmpty(child)) { children++; int childStrLen = Util.trimmedLen(child.toString()); avgChildLengthDouble += childStrLen; NodeCounter counter = new NodeCounter(child); if (Util.isItem(child.nodeName())) { ++childItems; anchorChildItems += counter.anchors() > 0 ? 1 : 0; textChildItems += counter.fields() > 0 ? 1 : 0; anchorTextChildItems += counter.anchors() > 0 && counter.fields() > 0 ? 1 : 0; itemChars += counter.chars(); itemAnchorChars += counter.anchorChars(); statItemChars.addValue(counter.chars()); statItemAnchorChars.addValue(counter.anchorChars()); } if (Util.isBlock(child.nodeName())) { ++childBlocks; } if (Util.isDecoration(child.nodeName())) { ++childDecoration; } if (Util.isFormatting(child.nodeName())) { ++childFormatting; } if (Util.isContent(child)) { ++childContent; } anchorChildren += counter.anchors() > 0 ? 1 : 0; textChildren += counter.fields() > 0 ? 1 : 0; anchorTextChildren += counter.anchors() > 0 && counter.fields() > 0 ? 1 : 0; statAnchorChars.addValue(counter.anchorChars()); statAnchors.addValue(counter.anchors()); statChars.addValue(counter.chars()); statDescendants.addValue(counter.descendants()); statFields.addValue(counter.fields()); statLevels.addValue(counter.levels()); statLongestField.addValue(counter.longestField()); statNonAnchorChars.addValue(counter.nonAnchorChars()); statTextAnchors.addValue(counter.textAnchors()); statStrLen.addValue(childStrLen); List<String> curChildTags = counter.tags(); allChildTags = Util.join(allChildTags, curChildTags); childTags.add(curChildTags); if (firstChildTags == null) { firstChildTags = curChildTags; } else if (childrenConsistent && !Util.isSame(firstChildTags, curChildTags)) { childrenConsistent = false; } if (childName == null) { childName = child.nodeName(); } else if (childrenSame && !childName.equals(child.nodeName())) { childrenSame = false; } if (!Util.contains(counter.orderedTags(), orderedTags)) { orderedTags.add(counter.orderedTags()); } } } avgChildLengthDouble = children == 0 ? 0 : avgChildLengthDouble / (double) children; int avgChildLength = (int) avgChildLengthDouble; double avgChildDiff = 0; int maxChildDiff = 0; for (List<String> tagList : childTags) { avgChildDiff += allChildTags.size() - tagList.size(); maxChildDiff = Math.max(maxChildDiff, allChildTags.size() - tagList.size()); } avgChildDiff = childTags.size() == 0 ? 0 : avgChildDiff / (double) childTags.size(); childrenConsistent = firstChildTags != null && !firstChildTags.isEmpty() && childrenConsistent; NodeCounter counter = new NodeCounter(separated); int siblings = 0; for (Node sibling : node.parent().childNodes()) { if (!Util.isEmpty(sibling)) { siblings++; } } this.scores = new int[] { counter.items(), counter.blocks(), counter.decoration(), counter.formatting(), counter.content(), div(counter.items(), children), div(counter.blocks(), children), div(counter.decoration(), children), div(counter.formatting(), children), div(counter.content(), children), childItems, childBlocks, childDecoration, childFormatting, childContent, avgChildLength, counter.fields(), textChildItems, counter.images(), counter.anchors(), counter.textAnchors(), div(counter.chars(), Math.max(1, counter.fields())), div(itemChars, Math.max(1, textChildItems)), counter.longestField(), nodeStrLen, div(nodeStrLen, children), counter.anchorLen(), counter.chars(), itemChars, div(counter.chars(), children), div(itemChars, childItems), counter.nonAnchorChars(), div(counter.nonAnchorChars(), children), div(counter.nonAnchorChars(), childItems), div(counter.nonAnchorChars(), childBlocks), div(counter.nonAnchorChars(), childContent), div(counter.nonAnchorChars(), counter.anchors()), div(counter.nonAnchorChars(), counter.textAnchors()), counter.anchorChars(), itemAnchorChars, div(itemAnchorChars, anchorChildItems), div(counter.anchorChars(), counter.anchors()), div(counter.anchorChars(), counter.textAnchors()), div(counter.anchorChars(), children), counter.descendants(), counter.levels(), div(counter.descendants(), children), div(children, counter.levels()), siblings, children, maxChildDiff, toInt(avgChildDiff), toInt(childrenSame), toInt(childrenConsistent), orderedTags.size(), mod0(children, RESULT_GROUP_LARGE), mod0(children, RESULT_GROUP_SMALL), distance(children, RESULT_GROUP_LARGE), distance(children, RESULT_GROUP_SMALL), mod0(childItems, RESULT_GROUP_LARGE), mod0(childItems, RESULT_GROUP_SMALL), distance(childItems, RESULT_GROUP_LARGE), distance(childItems, RESULT_GROUP_SMALL), mod0(childBlocks, RESULT_GROUP_LARGE), mod0(childBlocks, RESULT_GROUP_SMALL), distance(childBlocks, RESULT_GROUP_LARGE), distance(childBlocks, RESULT_GROUP_SMALL), mod0(childContent, RESULT_GROUP_LARGE), mod0(childContent, RESULT_GROUP_SMALL), distance(childContent, RESULT_GROUP_LARGE), distance(childContent, RESULT_GROUP_SMALL), mod0(counter.anchors(), RESULT_GROUP_LARGE), mod0(counter.anchors(), RESULT_GROUP_SMALL), distance(counter.anchors(), RESULT_GROUP_LARGE), distance(counter.anchors(), RESULT_GROUP_SMALL), mod0(anchorChildItems, RESULT_GROUP_LARGE), mod0(anchorChildItems, RESULT_GROUP_SMALL), distance(anchorChildItems, RESULT_GROUP_LARGE), distance(anchorChildItems, RESULT_GROUP_SMALL), mod0(textChildItems, RESULT_GROUP_LARGE), mod0(textChildItems, RESULT_GROUP_SMALL), distance(textChildItems, RESULT_GROUP_LARGE), distance(textChildItems, RESULT_GROUP_SMALL), mod0(counter.textAnchors(), RESULT_GROUP_LARGE), mod0(counter.textAnchors(), RESULT_GROUP_SMALL), distance(counter.textAnchors(), RESULT_GROUP_LARGE), distance(counter.textAnchors(), RESULT_GROUP_SMALL), Math.abs(children - counter.anchors()), Math.abs(childItems - counter.anchors()), evenlyDivisible(children, counter.anchors()), evenlyDivisible(childItems, counter.anchors()), smallestMod(children, counter.anchors()), smallestMod(childItems, counter.anchors()), Math.abs(children - counter.textAnchors()), Math.abs(childItems - counter.textAnchors()), Math.abs(children - anchorChildren), Math.abs(childItems - anchorChildItems), Math.abs(children - textChildren), Math.abs(childItems - textChildItems), Math.abs(children - anchorTextChildren), Math.abs(childItems - anchorTextChildItems), evenlyDivisible(children, counter.textAnchors()), evenlyDivisible(childItems, counter.textAnchors()), evenlyDivisible(children, anchorChildren), evenlyDivisible(childItems, anchorChildItems), evenlyDivisible(children, textChildren), evenlyDivisible(childItems, textChildItems), evenlyDivisible(children, anchorTextChildren), evenlyDivisible(childItems, anchorTextChildItems), smallestMod(children, counter.textAnchors()), smallestMod(childItems, counter.textAnchors()), smallestMod(children, anchorChildren), smallestMod(childItems, anchorChildItems), smallestMod(children, textChildren), smallestMod(childItems, textChildItems), smallestMod(children, anchorTextChildren), smallestMod(childItems, anchorTextChildItems), Math.abs(anchorChildren - anchorChildItems), Math.abs(textChildren - textChildItems), Math.abs(anchorTextChildren - anchorTextChildItems), toInt(statAnchorChars.getSkewness()), toInt(statAnchorChars.getStandardDeviation()), toInt(statAnchorChars.getMean()), toInt(statAnchors.getSkewness()), toInt(statAnchors.getStandardDeviation()), toInt(statAnchors.getMean()), toInt(statChars.getSkewness()), toInt(statChars.getStandardDeviation()), toInt(statChars.getMean()), toInt(statDescendants.getSkewness()), toInt(statDescendants.getStandardDeviation()), toInt(statDescendants.getMean()), toInt(statFields.getSkewness()), toInt(statFields.getStandardDeviation()), toInt(statFields.getMean()), toInt(statLevels.getSkewness()), toInt(statLevels.getStandardDeviation()), toInt(statLevels.getMean()), toInt(statLongestField.getSkewness()), toInt(statLongestField.getStandardDeviation()), toInt(statLongestField.getMean()), toInt(statNonAnchorChars.getSkewness()), toInt(statNonAnchorChars.getStandardDeviation()), toInt(statNonAnchorChars.getMean()), toInt(statStrLen.getSkewness()), toInt(statStrLen.getStandardDeviation()), toInt(statStrLen.getMean()), toInt(statTextAnchors.getSkewness()), toInt(statTextAnchors.getStandardDeviation()), toInt(statTextAnchors.getMean()), toInt(statItemChars.getSkewness()), toInt(statItemChars.getStandardDeviation()), toInt(statItemChars.getMean()), toInt(statItemAnchorChars.getSkewness()), toInt(statItemAnchorChars.getStandardDeviation()), toInt(statItemAnchorChars.getMean()), }; }
From source file:org.apache.metron.common.math.stats.OnlineStatisticsProviderTest.java
public static void validateStatisticsProvider(StatisticsProvider statsProvider, SummaryStatistics summaryStats, DescriptiveStatistics stats) { //N// www. j ava 2 s . co m Assert.assertEquals(statsProvider.getCount(), stats.getN()); //sum Assert.assertEquals(statsProvider.getSum(), stats.getSum(), 1e-3); //sum of squares Assert.assertEquals(statsProvider.getSumSquares(), stats.getSumsq(), 1e-3); //sum of squares Assert.assertEquals(statsProvider.getSumLogs(), summaryStats.getSumOfLogs(), 1e-3); //Mean Assert.assertEquals(statsProvider.getMean(), stats.getMean(), 1e-3); //Quadratic Mean Assert.assertEquals(statsProvider.getQuadraticMean(), summaryStats.getQuadraticMean(), 1e-3); //SD Assert.assertEquals(statsProvider.getStandardDeviation(), stats.getStandardDeviation(), 1e-3); //Variance Assert.assertEquals(statsProvider.getVariance(), stats.getVariance(), 1e-3); //Min Assert.assertEquals(statsProvider.getMin(), stats.getMin(), 1e-3); //Max Assert.assertEquals(statsProvider.getMax(), stats.getMax(), 1e-3); //Kurtosis Assert.assertEquals(stats.getKurtosis(), statsProvider.getKurtosis(), 1e-3); //Skewness Assert.assertEquals(stats.getSkewness(), statsProvider.getSkewness(), 1e-3); for (double d = 10.0; d < 100.0; d += 10) { //This is a sketch, so we're a bit more forgiving here in our choice of \epsilon. Assert.assertEquals("Percentile mismatch for " + d + "th %ile", statsProvider.getPercentile(d), stats.getPercentile(d), 1e-2); } }
From source file:org.apache.solr.client.solrj.io.eval.DescribeEvaluator.java
@Override public Object doWork(Object value) throws IOException { if (!(value instanceof List<?>)) { throw new IOException( String.format(Locale.ROOT, "Invalid expression %s - expecting a numeric list but found %s", toExpression(constructingFactory), value.getClass().getSimpleName())); }/*from w w w .ja v a 2 s . c o m*/ // we know each value is a BigDecimal or a list of BigDecimals DescriptiveStatistics descriptiveStatistics = new DescriptiveStatistics(); ((List<?>) value).stream().mapToDouble(innerValue -> ((BigDecimal) innerValue).doubleValue()) .forEach(innerValue -> descriptiveStatistics.addValue(innerValue)); Map<String, Number> map = new HashMap<>(); map.put("max", descriptiveStatistics.getMax()); map.put("mean", descriptiveStatistics.getMean()); map.put("min", descriptiveStatistics.getMin()); map.put("stdev", descriptiveStatistics.getStandardDeviation()); map.put("sum", descriptiveStatistics.getSum()); map.put("N", descriptiveStatistics.getN()); map.put("var", descriptiveStatistics.getVariance()); map.put("kurtosis", descriptiveStatistics.getKurtosis()); map.put("skewness", descriptiveStatistics.getSkewness()); map.put("popVar", descriptiveStatistics.getPopulationVariance()); map.put("geometricMean", descriptiveStatistics.getGeometricMean()); map.put("sumsq", descriptiveStatistics.getSumsq()); return new Tuple(map); }
From source file:org.jenetics.stat.DoubleMomentStatisticsTest.java
@Test(dataProvider = "sampleCounts") public void summary(final Integer sampleCounts, final Double epsilon) { final List<Double> numbers = numbers(sampleCounts); final DescriptiveStatistics expected = new DescriptiveStatistics(); numbers.forEach(expected::addValue); final DoubleMomentStatistics summary = numbers.stream() .collect(toDoubleMomentStatistics(Double::doubleValue)); Assert.assertEquals(summary.getCount(), numbers.size()); assertEqualsDouble(min(summary.getMin()), expected.getMin(), 0.0); assertEqualsDouble(max(summary.getMax()), expected.getMax(), 0.0); assertEqualsDouble(summary.getSum(), expected.getSum(), epsilon); assertEqualsDouble(summary.getMean(), expected.getMean(), epsilon); assertEqualsDouble(summary.getVariance(), expected.getVariance(), epsilon); assertEqualsDouble(summary.getSkewness(), expected.getSkewness(), epsilon); assertEqualsDouble(summary.getKurtosis(), expected.getKurtosis(), epsilon); }
From source file:org.jenetics.stat.DoubleMomentStatisticsTest.java
@Test(dataProvider = "parallelSampleCounts") public void parallelSummary(final Integer sampleCounts, final Double epsilon) { final List<Double> numbers = numbers(sampleCounts); final DescriptiveStatistics expected = new DescriptiveStatistics(); numbers.forEach(expected::addValue); final DoubleMomentStatistics summary = numbers.parallelStream() .collect(toDoubleMomentStatistics(Double::doubleValue)); Assert.assertEquals(summary.getCount(), numbers.size()); assertEqualsDouble(min(summary.getMin()), expected.getMin(), 0.0); assertEqualsDouble(max(summary.getMax()), expected.getMax(), 0.0); assertEqualsDouble(summary.getSum(), expected.getSum(), epsilon); assertEqualsDouble(summary.getMean(), expected.getMean(), epsilon); assertEqualsDouble(summary.getVariance(), expected.getVariance(), epsilon); assertEqualsDouble(summary.getSkewness(), expected.getSkewness(), epsilon); assertEqualsDouble(summary.getKurtosis(), expected.getKurtosis(), epsilon); }
From source file:org.jenetics.stat.IntMomentStatisticsTest.java
@Test(dataProvider = "sampleCounts") public void summary(final Integer sampleCounts, final Double epsilon) { final List<Integer> numbers = numbers(sampleCounts); final DescriptiveStatistics expected = new DescriptiveStatistics(); numbers.forEach(expected::addValue); final IntMomentStatistics summary = numbers.stream().collect(toIntMomentStatistics(Integer::intValue)); Assert.assertEquals(summary.getCount(), numbers.size()); assertEqualsDouble(min(summary.getMin()), expected.getMin(), 0.0); assertEqualsDouble(max(summary.getMax()), expected.getMax(), 0.0); assertEqualsDouble(summary.getSum(), expected.getSum(), epsilon); assertEqualsDouble(summary.getMean(), expected.getMean(), epsilon); assertEqualsDouble(summary.getVariance(), expected.getVariance(), epsilon); assertEqualsDouble(summary.getSkewness(), expected.getSkewness(), epsilon); assertEqualsDouble(summary.getKurtosis(), expected.getKurtosis(), epsilon); }
From source file:org.jenetics.stat.IntMomentStatisticsTest.java
@Test(dataProvider = "parallelSampleCounts") public void parallelSummary(final Integer sampleCounts, final Double epsilon) { final List<Integer> numbers = numbers(sampleCounts); final DescriptiveStatistics expected = new DescriptiveStatistics(); numbers.forEach(expected::addValue); final IntMomentStatistics summary = numbers.parallelStream() .collect(toIntMomentStatistics(Integer::intValue)); Assert.assertEquals(summary.getCount(), numbers.size()); assertEqualsDouble(min(summary.getMin()), expected.getMin(), 0.0); assertEqualsDouble(max(summary.getMax()), expected.getMax(), 0.0); assertEqualsDouble(summary.getSum(), expected.getSum(), epsilon); assertEqualsDouble(summary.getMean(), expected.getMean(), epsilon); assertEqualsDouble(summary.getVariance(), expected.getVariance(), epsilon); assertEqualsDouble(summary.getSkewness(), expected.getSkewness(), epsilon); assertEqualsDouble(summary.getKurtosis(), expected.getKurtosis(), epsilon); }
From source file:org.jenetics.stat.LongMomentStatisticsTest.java
@Test(dataProvider = "sampleCounts") public void summary(final Integer sampleCounts, final Double epsilon) { final List<Long> numbers = numbers(sampleCounts); final DescriptiveStatistics expected = new DescriptiveStatistics(); numbers.forEach(expected::addValue); final LongMomentStatistics summary = numbers.stream().collect(toLongMomentStatistics(Long::longValue)); Assert.assertEquals(summary.getCount(), numbers.size()); assertEqualsDouble(min(summary.getMin()), expected.getMin(), 0.0); assertEqualsDouble(max(summary.getMax()), expected.getMax(), 0.0); assertEqualsDouble(summary.getSum(), expected.getSum(), epsilon); assertEqualsDouble(summary.getMean(), expected.getMean(), epsilon); assertEqualsDouble(summary.getVariance(), expected.getVariance(), epsilon); assertEqualsDouble(summary.getSkewness(), expected.getSkewness(), epsilon); assertEqualsDouble(summary.getKurtosis(), expected.getKurtosis(), epsilon); }