List of usage examples for org.apache.commons.math.stat.descriptive DescriptiveStatistics getStandardDeviation
public double getStandardDeviation()
From source file:com.joliciel.jochre.graphics.SourceImageImpl.java
public SourceImageImpl(GraphicsServiceInternal graphicsService, String name, BufferedImage image) { super(image); this.name = name; this.setOriginalImage(image); this.setGraphicsService(graphicsService); this.setWidth(this.getPixelGrabber().getWidth()); this.setHeight(this.getPixelGrabber().getHeight()); // to normalise the image, we need to figure out where black and white are // we want to leave out anomalies (ink blots!) int[] pixelSpread = new int[256]; // To save on memory for (int y = 0; y < this.getHeight(); y++) for (int x = 0; x < this.getWidth(); x++) { int pixel = this.getPixelGrabber().getPixelBrightness(x, y); pixelSpread[pixel]++;/*from w w w . j a va2s.c o m*/ } if (LOG.isTraceEnabled()) { for (int i = 0; i < 256; i++) LOG.trace("Brightness " + i + ": " + pixelSpread[i]); } DescriptiveStatistics countStats = new DescriptiveStatistics(); for (int i = 0; i < 256; i++) { countStats.addValue(pixelSpread[i]); } int startWhite = -1; int endWhite = -1; for (int i = 255; i >= 0; i--) { if (startWhite < 0 && pixelSpread[i] > countStats.getMean()) startWhite = i; if (startWhite >= 0 && endWhite < 0 && pixelSpread[i] < countStats.getMean()) { endWhite = i; break; } } LOG.debug("Start white: " + startWhite); LOG.debug("End white: " + endWhite); DescriptiveStatistics blackCountStats = new DescriptiveStatistics(); DescriptiveStatistics blackSpread = new DescriptiveStatistics(); for (int i = 0; i <= endWhite; i++) { blackCountStats.addValue(pixelSpread[i]); for (int j = 0; j < pixelSpread[i]; j++) { blackSpread.addValue(i); } } LOG.debug("mean counts: " + countStats.getMean()); LOG.debug("mean black counts: " + blackCountStats.getMean()); LOG.debug("std dev black counts: " + blackCountStats.getStandardDeviation()); int startBlack = -1; for (int i = 0; i < 256; i++) { if (pixelSpread[i] > blackCountStats.getMean()) { startBlack = i; break; } } LOG.debug("Start black: " + startBlack); this.setBlackLimit(startBlack); this.setWhiteLimit(startWhite); this.greyscaleMultiplier = (255.0 / (double) (whiteLimit - blackLimit)); // use mean + 2 sigma to find the black threshold // we make the threshold high (darker) to put more pixels in the letter when analysing double blackthresholdCount = blackCountStats.getMean() + (2.0 * blackCountStats.getStandardDeviation()); LOG.debug("blackthresholdCount: " + blackthresholdCount); int blackThresholdValue = endWhite; for (int i = endWhite; i >= startBlack; i--) { if (pixelSpread[i] < blackthresholdCount) { blackThresholdValue = i; break; } } LOG.debug("Black threshold value (old): " + blackThresholdValue); blackThreshold = (int) Math.round((blackThresholdValue - blackLimit) * greyscaleMultiplier); LOG.debug("Black threshold (old): " + blackThreshold); blackThresholdValue = (int) Math.round(blackSpread.getPercentile(60.0)); LOG.debug("Black threshold value (new): " + blackThresholdValue); LOG.debug("Black spread 25 percentile: " + (int) Math.round(blackSpread.getPercentile(25.0))); LOG.debug("Black spread 50 percentile: " + (int) Math.round(blackSpread.getPercentile(50.0))); LOG.debug("Black spread 75 percentile: " + (int) Math.round(blackSpread.getPercentile(75.0))); blackThreshold = (int) Math.round((blackThresholdValue - blackLimit) * greyscaleMultiplier); LOG.debug("Black threshold (new): " + blackThreshold); // use mean + 1 sigma to find the separation threshold // we keep threshold low (1 sigma) to encourage letter breaks double separationthresholdCount = blackCountStats.getMean() + (1.0 * blackCountStats.getStandardDeviation()); LOG.debug("Separation threshold value: " + separationthresholdCount); int separationThresholdValue = endWhite; for (int i = endWhite; i >= startBlack; i--) { if (pixelSpread[i] < separationthresholdCount) { separationThresholdValue = i; break; } } LOG.debug("Separation threshold value (old): " + separationThresholdValue); separationThresholdValue = (int) Math.round(blackSpread.getPercentile(75.0)); LOG.debug("Separation threshold value (new): " + separationThresholdValue); LOG.debug("Black spread 25 percentile: " + (int) Math.round(blackSpread.getPercentile(25.0))); LOG.debug("Black spread 50 percentile: " + (int) Math.round(blackSpread.getPercentile(50.0))); LOG.debug("Black spread 75 percentile: " + (int) Math.round(blackSpread.getPercentile(75.0))); separationThreshold = (int) Math.round((separationThresholdValue - blackLimit) * greyscaleMultiplier); LOG.debug("Separation threshold: " + separationThreshold); if (drawPixelSpread) this.drawChart(pixelSpread, countStats, blackCountStats, blackSpread, startWhite, endWhite, startBlack, blackThresholdValue); }
From source file:com.joliciel.jochre.graphics.SourceImageImpl.java
private void drawChart(int[] pixelSpread, DescriptiveStatistics countStats, DescriptiveStatistics blackCountStats, DescriptiveStatistics blackSpread, int startWhite, int endWhite, int startBlack, int blackThresholdValue) { XYSeries xySeries = new XYSeries("Brightness data"); double maxSpread = 0; for (int i = 0; i < 256; i++) { xySeries.add(i, pixelSpread[i]); if (pixelSpread[i] > maxSpread) maxSpread = pixelSpread[i];//from ww w . j a v a2 s .c o m } XYSeries keyValues = new XYSeries("Key values"); XYSeries counts = new XYSeries("Counts"); counts.add(10.0, countStats.getMean()); counts.add(100.0, blackCountStats.getMean()); counts.add(125.0, blackCountStats.getMean() + (1.0 * blackCountStats.getStandardDeviation())); counts.add(150.0, blackCountStats.getMean() + (2.0 * blackCountStats.getStandardDeviation())); counts.add(175.0, blackCountStats.getMean() + (3.0 * blackCountStats.getStandardDeviation())); counts.add(75.0, blackCountStats.getMean() - (1.0 * blackCountStats.getStandardDeviation())); counts.add(50.0, blackCountStats.getMean() - (2.0 * blackCountStats.getStandardDeviation())); counts.add(25.0, blackCountStats.getMean() - (3.0 * blackCountStats.getStandardDeviation())); keyValues.add(startWhite, maxSpread / 4.0); keyValues.add(endWhite, maxSpread / 4.0); keyValues.add(startBlack, maxSpread / 4.0); keyValues.add(blackThresholdValue, maxSpread / 2.0); XYSeriesCollection dataset = new XYSeriesCollection(xySeries); dataset.addSeries(keyValues); dataset.addSeries(counts); final ValueAxis xAxis = new NumberAxis("Brightness"); final ValueAxis yAxis = new NumberAxis("Count"); yAxis.setUpperBound(maxSpread); xAxis.setUpperBound(255.0); //final XYItemRenderer renderer = new XYBarRenderer(); final XYBarRenderer renderer = new XYBarRenderer(); renderer.setShadowVisible(false); final XYPlot plot = new XYPlot(dataset, xAxis, yAxis, renderer); plot.setOrientation(PlotOrientation.VERTICAL); String title = "BrightnessChart"; final JFreeChart chart = new JFreeChart(title, JFreeChart.DEFAULT_TITLE_FONT, plot, true); chart.setBackgroundPaint(Color.white); File file = new File("data/" + title + ".png"); try { ChartUtilities.saveChartAsPNG(file, chart, 600, 600); } catch (IOException e) { e.printStackTrace(); throw new RuntimeException(e); } }
From source file:com.joliciel.talismane.stats.FScoreCalculator.java
/** * Combine the results of n cross validation results into a single f-score file. * @param directory/*www .j a v a 2 s .c om*/ * @param prefix * @param suffix * @param csvFileWriter */ static void combineCrossValidationResults(File directory, String prefix, String suffix, Writer csvFileWriter) { try { File[] files = directory.listFiles(); Map<Integer, Map<String, FScoreStats>> fileStatsMap = new HashMap<Integer, Map<String, FScoreStats>>(); for (File file : files) { if (file.getName().startsWith(prefix) && file.getName().endsWith(suffix)) { int index = Integer.parseInt(file.getName().substring(prefix.length(), prefix.length() + 1)); Map<String, FScoreStats> statsMap = new HashMap<String, FScoreCalculator.FScoreStats>(); fileStatsMap.put(index, statsMap); Scanner scanner = new Scanner( new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"))); boolean firstLine = true; int truePositivePos = -1; while (scanner.hasNextLine()) { String line = scanner.nextLine(); List<String> cells = CSV.getCSVCells(line); if (firstLine) { int i = 0; for (String cell : cells) { if (cell.equals("true+")) { truePositivePos = i; break; } i++; } if (truePositivePos < 0) { throw new JolicielException("Couldn't find true+ on first line"); } firstLine = false; } else { FScoreStats stats = new FScoreStats(); String outcome = cells.get(0); stats.outcome = outcome; if (outcome.equals("AVERAGE")) break; stats.truePos = Integer.parseInt(cells.get(truePositivePos)); stats.falsePos = Integer.parseInt(cells.get(truePositivePos + 1)); stats.falseNeg = Integer.parseInt(cells.get(truePositivePos + 2)); stats.precision = Double.parseDouble(cells.get(truePositivePos + 3)); stats.recall = Double.parseDouble(cells.get(truePositivePos + 4)); stats.fScore = Double.parseDouble(cells.get(truePositivePos + 5)); statsMap.put(outcome, stats); } // firstLine? } // has more lines scanner.close(); } // file in current series } // next file int numFiles = fileStatsMap.size(); if (numFiles == 0) { throw new JolicielException("No files found matching prefix and suffix provided"); } Map<String, DescriptiveStatistics> descriptiveStatsMap = new HashMap<String, DescriptiveStatistics>(); Map<String, FScoreStats> outcomeStats = new HashMap<String, FScoreCalculator.FScoreStats>(); Set<String> outcomes = new TreeSet<String>(); for (Map<String, FScoreStats> statsMap : fileStatsMap.values()) { for (FScoreStats stats : statsMap.values()) { DescriptiveStatistics fScoreStats = descriptiveStatsMap.get(stats.outcome + "fScore"); if (fScoreStats == null) { fScoreStats = new DescriptiveStatistics(); descriptiveStatsMap.put(stats.outcome + "fScore", fScoreStats); } fScoreStats.addValue(stats.fScore); DescriptiveStatistics precisionStats = descriptiveStatsMap.get(stats.outcome + "precision"); if (precisionStats == null) { precisionStats = new DescriptiveStatistics(); descriptiveStatsMap.put(stats.outcome + "precision", precisionStats); } precisionStats.addValue(stats.precision); DescriptiveStatistics recallStats = descriptiveStatsMap.get(stats.outcome + "recall"); if (recallStats == null) { recallStats = new DescriptiveStatistics(); descriptiveStatsMap.put(stats.outcome + "recall", recallStats); } recallStats.addValue(stats.recall); FScoreStats outcomeStat = outcomeStats.get(stats.outcome); if (outcomeStat == null) { outcomeStat = new FScoreStats(); outcomeStat.outcome = stats.outcome; outcomeStats.put(stats.outcome, outcomeStat); } outcomeStat.truePos += stats.truePos; outcomeStat.falsePos += stats.falsePos; outcomeStat.falseNeg += stats.falseNeg; outcomes.add(stats.outcome); } } csvFileWriter.write(CSV.format(prefix + suffix)); csvFileWriter.write("\n"); csvFileWriter.write(CSV.format("outcome")); csvFileWriter.write(CSV.format("true+") + CSV.format("false+") + CSV.format("false-") + CSV.format("tot precision") + CSV.format("avg precision") + CSV.format("dev precision") + CSV.format("tot recall") + CSV.format("avg recall") + CSV.format("dev recall") + CSV.format("tot f-score") + CSV.format("avg f-score") + CSV.format("dev f-score") + "\n"); for (String outcome : outcomes) { csvFileWriter.write(CSV.format(outcome)); FScoreStats outcomeStat = outcomeStats.get(outcome); DescriptiveStatistics fScoreStats = descriptiveStatsMap.get(outcome + "fScore"); DescriptiveStatistics precisionStats = descriptiveStatsMap.get(outcome + "precision"); DescriptiveStatistics recallStats = descriptiveStatsMap.get(outcome + "recall"); outcomeStat.calculate(); csvFileWriter.write(CSV.format(outcomeStat.truePos)); csvFileWriter.write(CSV.format(outcomeStat.falsePos)); csvFileWriter.write(CSV.format(outcomeStat.falseNeg)); csvFileWriter.write(CSV.format(outcomeStat.precision * 100)); csvFileWriter.write(CSV.format(precisionStats.getMean())); csvFileWriter.write(CSV.format(precisionStats.getStandardDeviation())); csvFileWriter.write(CSV.format(outcomeStat.recall * 100)); csvFileWriter.write(CSV.format(recallStats.getMean())); csvFileWriter.write(CSV.format(recallStats.getStandardDeviation())); csvFileWriter.write(CSV.format(outcomeStat.fScore * 100)); csvFileWriter.write(CSV.format(fScoreStats.getMean())); csvFileWriter.write(CSV.format(fScoreStats.getStandardDeviation())); csvFileWriter.write("\n"); csvFileWriter.flush(); } } catch (IOException ioe) { throw new RuntimeException(ioe); } }
From source file:com.joliciel.jochre.graphics.SegmenterImpl.java
/** * Clear out anything found in the right & left margins * @param sourceImage/* w w w . jav a 2s . c o m*/ */ void cleanMargins(SourceImage sourceImage) { LOG.debug("########## cleanMargins #########"); int minCardinalityForMargin = 8; double averageShapeWidth = sourceImage.getAverageShapeWidth(); LOG.debug("Finding right margin"); double rightLimit = (double) sourceImage.getWidth() * 0.67; // first, create a DBScan cluster of all rows near the right-hand side List<RowOfShapes> rightHandRows = new ArrayList<RowOfShapes>(); List<double[]> rightCoordinates = new ArrayList<double[]>(); for (RowOfShapes row : sourceImage.getRows()) { double right = row.getRight(); if (right >= rightLimit) { LOG.trace(row.toString()); LOG.trace( "Right: " + right + " + " + row.getXAdjustment() + " = " + (right - row.getXAdjustment())); right -= row.getXAdjustment(); rightHandRows.add(row); rightCoordinates.add(new double[] { right }); } } DBSCANClusterer<RowOfShapes> rightMarginClusterer = new DBSCANClusterer<RowOfShapes>(rightHandRows, rightCoordinates); Set<Set<RowOfShapes>> rowClusters = rightMarginClusterer.cluster(averageShapeWidth, minCardinalityForMargin, true); TreeSet<Set<RowOfShapes>> orderedRowClusters = new TreeSet<Set<RowOfShapes>>( new CardinalityComparator<RowOfShapes>()); orderedRowClusters.addAll(rowClusters); int i = 0; // find the right-most cluster with sufficient cardinality, and assume it's the right margin DescriptiveStatistics rightMarginStats = null; for (Set<RowOfShapes> cluster : orderedRowClusters) { DescriptiveStatistics rightStats = new DescriptiveStatistics(); for (RowOfShapes row : cluster) rightStats.addValue(row.getRight() - row.getXAdjustment()); LOG.debug("Cluster " + i + ". Cardinality=" + cluster.size()); LOG.debug("Right mean : " + rightStats.getMean()); LOG.debug("Right std dev: " + rightStats.getStandardDeviation()); if (cluster.size() >= minCardinalityForMargin && (rightMarginStats == null || rightMarginStats.getMean() < rightStats.getMean())) { rightMarginStats = rightStats; } i++; } // see how many rows would violate this margin - if too many, assume no margin // these rows are only rows which extend across the margin if (rightMarginStats != null) { LOG.debug("Right margin mean : " + rightMarginStats.getMean()); LOG.debug("Right margin std dev: " + rightMarginStats.getStandardDeviation()); double rightMarginLimit = rightMarginStats.getMean() + sourceImage.getAverageShapeWidth(); LOG.debug("rightMarginLimit: " + rightMarginLimit); int numRowsToChop = 0; for (RowOfShapes row : sourceImage.getRows()) { if (row.getRight() >= rightLimit) { if (row.getRight() - row.getXAdjustment() >= rightMarginLimit && row.getLeft() - row.getXAdjustment() <= rightMarginLimit) { LOG.debug("Found overlapping row : " + row); LOG.debug("Adjusted right : " + (row.getRight() - row.getXAdjustment())); numRowsToChop++; } } } if (numRowsToChop >= 3) { LOG.debug("Too many overlapping rows - ignoring margin"); rightMarginStats = null; } } if (rightMarginStats != null) { double rightMarginLimit = rightMarginStats.getMean() + sourceImage.getAverageShapeWidth(); List<RowOfShapes> rowsToRemove = new ArrayList<RowOfShapes>(); for (RowOfShapes row : sourceImage.getRows()) { double right = row.getRight() - row.getXAdjustment(); LOG.trace(row.toString()); LOG.trace("Adjusted right: " + right); if (right >= rightMarginLimit) { LOG.trace("Has out-of-margin stuff!"); // need to chop off groups to the right of this threshold List<GroupOfShapes> groupsToChop = new ArrayList<GroupOfShapes>(); for (GroupOfShapes group : row.getGroups()) { if (group.getLeft() - row.getXAdjustment() > rightMarginLimit) { groupsToChop.add(group); LOG.debug("Chopping group outside of right margin: " + group); } } for (GroupOfShapes group : groupsToChop) { row.getShapes().removeAll(group.getShapes()); } row.getGroups().removeAll(groupsToChop); if (row.getGroups().size() == 0) { LOG.debug("Removing empty " + row); rowsToRemove.add(row); } else { row.recalculate(); row.assignGuideLines(); } } // does this row extend beyond the margin? } // next row sourceImage.getRows().removeAll(rowsToRemove); } // have a right margin LOG.debug("Finding left margin"); double leftLimit = (double) sourceImage.getWidth() * 0.33; // first, create a DBScan cluster of all rows near the left-hand side List<RowOfShapes> leftHandRows = new ArrayList<RowOfShapes>(); List<double[]> leftCoordinates = new ArrayList<double[]>(); for (RowOfShapes row : sourceImage.getRows()) { double left = row.getLeft(); if (left <= leftLimit) { LOG.trace(row.toString()); LOG.trace("Left: " + left + " - " + row.getXAdjustment() + " = " + (left - row.getXAdjustment())); left -= row.getXAdjustment(); leftHandRows.add(row); leftCoordinates.add(new double[] { left }); } } DBSCANClusterer<RowOfShapes> leftMarginClusterer = new DBSCANClusterer<RowOfShapes>(leftHandRows, leftCoordinates); Set<Set<RowOfShapes>> rowClustersLeft = leftMarginClusterer.cluster(averageShapeWidth, minCardinalityForMargin, true); TreeSet<Set<RowOfShapes>> orderedRowClustersLeft = new TreeSet<Set<RowOfShapes>>( new CardinalityComparator<RowOfShapes>()); orderedRowClustersLeft.addAll(rowClustersLeft); i = 0; // find the left-most cluster with sufficient cardinality, and assume it's the left margin DescriptiveStatistics leftMarginStats = null; for (Set<RowOfShapes> cluster : orderedRowClustersLeft) { DescriptiveStatistics leftStats = new DescriptiveStatistics(); for (RowOfShapes row : cluster) leftStats.addValue(row.getLeft() - row.getXAdjustment()); LOG.debug("Cluster " + i + ". Cardinality=" + cluster.size()); LOG.debug("Left mean : " + leftStats.getMean()); LOG.debug("Left std dev: " + leftStats.getStandardDeviation()); if (cluster.size() >= minCardinalityForMargin && (leftMarginStats == null || leftMarginStats.getMean() > leftStats.getMean())) { leftMarginStats = leftStats; } i++; } // see how many rows would violate this margin - if too many, assume no margin // these rows are only rows which extend across the margin if (leftMarginStats != null) { LOG.debug("Left margin mean : " + leftMarginStats.getMean()); LOG.debug("Left margin std dev: " + leftMarginStats.getStandardDeviation()); double leftMarginLimit = leftMarginStats.getMean() - sourceImage.getAverageShapeWidth(); LOG.debug("leftMarginLimit: " + leftMarginLimit); int numRowsToChop = 0; for (RowOfShapes row : sourceImage.getRows()) { if (row.getLeft() <= leftLimit) { if (row.getLeft() - row.getXAdjustment() <= leftMarginLimit && row.getRight() - row.getXAdjustment() >= leftMarginLimit) { LOG.debug("Found overlapping row : " + row); LOG.debug("Adjusted left : " + (row.getLeft() - row.getXAdjustment())); numRowsToChop++; } } } if (numRowsToChop >= 3) { LOG.debug("Too many overlapping rows - ignoring margin"); leftMarginStats = null; } } if (leftMarginStats != null) { double leftMarginLimit = leftMarginStats.getMean() - sourceImage.getAverageShapeWidth(); List<RowOfShapes> rowsToRemove = new ArrayList<RowOfShapes>(); for (RowOfShapes row : sourceImage.getRows()) { double left = row.getLeft() - row.getXAdjustment(); LOG.trace(row.toString()); LOG.trace("Adjusted left: " + left); if (left <= leftMarginLimit) { LOG.trace("Has out-of-margin stuff!"); // need to chop off groups to the left of this threshold List<GroupOfShapes> groupsToChop = new ArrayList<GroupOfShapes>(); for (GroupOfShapes group : row.getGroups()) { if (group.getRight() - row.getXAdjustment() < leftMarginLimit) { groupsToChop.add(group); LOG.debug("Chopping group outside of left margin: " + group); } } for (GroupOfShapes group : groupsToChop) { row.getShapes().removeAll(group.getShapes()); } row.getGroups().removeAll(groupsToChop); if (row.getGroups().size() == 0) { LOG.debug("Removing empty " + row); rowsToRemove.add(row); } else { row.recalculate(); row.assignGuideLines(); } } // does this row extend beyond the margin? } // next row sourceImage.getRows().removeAll(rowsToRemove); } // have a left margin }
From source file:com.joliciel.jochre.lexicon.LexiconErrorWriter.java
static void mergeCrossValidation(File evalDir, String prefix) { try {/*from w w w .j a v a2 s .c o m*/ File[] files = evalDir.listFiles(new FilenameFilter() { @Override public boolean accept(File dir, String name) { if (name.endsWith(".csv")) return true; else return false; } }); List<String> groupNames = new ArrayList<String>(); Map<String, Writer> writers = new HashMap<String, Writer>(); Map<String, ErrorStatistics> errorMap = new LinkedHashMap<String, ErrorStatistics>(); Map<String, Map<String, DescriptiveStatistics>> statMap = new HashMap<String, Map<String, DescriptiveStatistics>>(); for (File file : files) { String filename = file.getName(); LOG.debug("Processing " + filename); int index = Integer.parseInt(filename.substring(prefix.length(), prefix.length() + 1)); String suffix = filename.substring(prefix.length() + 2, filename.lastIndexOf('_')); String fileType = filename.substring(filename.lastIndexOf('_') + 1, filename.lastIndexOf('.')); LOG.debug("Processing " + filename); LOG.debug("index: " + index); LOG.debug("suffix: " + suffix); LOG.debug("fileType: " + fileType); Writer writer = writers.get(fileType); boolean firstFile = false; if (writer == null) { writer = new BufferedWriter(new OutputStreamWriter( new FileOutputStream( new File(evalDir, prefix + "A_" + suffix + "_" + fileType + ".csv"), false), "UTF8")); writers.put(fileType, writer); firstFile = true; } if (fileType.equals("KEMatrix")) { Scanner scanner = new Scanner(file); int i = 0; List<String> myGroupNames = new ArrayList<String>(); Map<String, Boolean> haveCountMap = new HashMap<String, Boolean>(); while (scanner.hasNextLine()) { String line = scanner.nextLine(); List<String> cells = CSV.getCSVCells(line); if (i == 0) { for (int j = 0; j < cells.size(); j += 5) { String groupName = cells.get(j); if (!errorMap.containsKey(groupName)) { errorMap.put(groupName, new ErrorStatistics()); statMap.put(groupName, new HashMap<String, DescriptiveStatistics>()); groupNames.add(groupName); } myGroupNames.add(groupName); } } else if (i == 1) { // do nothing } else { String rowName = cells.get(0); int j = 0; for (String groupName : myGroupNames) { ErrorStatistics errorStats = errorMap.get(groupName); Map<String, DescriptiveStatistics> stats = statMap.get(groupName); double correctCount = Double.parseDouble(cells.get(j * 5 + 1)); double errorCount = Double.parseDouble(cells.get(j * 5 + 2)); double totalCount = Double.parseDouble(cells.get(j * 5 + 3)); Boolean haveCount = haveCountMap.get(groupName); if (rowName.equals("known")) { errorStats.knownWordCorrectCount += correctCount; errorStats.knownWordErrorCount += errorCount; } else if (rowName.equals("unknown")) { errorStats.unknownWordCorrectCount += correctCount; errorStats.unknownWordErrorCount += errorCount; } else if (rowName.equals("goodSeg")) { errorStats.goodSegCorrectCount += correctCount; errorStats.goodSegErrorCount += errorCount; } else if (rowName.equals("badSeg")) { errorStats.badSegCorrectCount += correctCount; errorStats.badSegErrorCount += errorCount; } else if (rowName.equals("knownLetters")) { errorStats.knownWordCorrectLetterCount += correctCount; errorStats.knownWordErrorLetterCount += errorCount; } else if (rowName.equals("unknownLetters")) { errorStats.unknownWordCorrectLetterCount += correctCount; errorStats.unknownWordErrorLetterCount += errorCount; } else if (rowName.equals("goodSegLetters")) { errorStats.goodSegCorrectLetterCount += correctCount; errorStats.goodSegErrorLetterCount += errorCount; } else if (rowName.equals("badSegLetters")) { errorStats.badSegCorrectLetterCount += correctCount; errorStats.badSegErrorLetterCount += errorCount; } else if (rowName.equals("inBeam")) { errorStats.answerInBeamCorrectCount += correctCount; errorStats.answerInBeamErrorCount += errorCount; } else if (rowName.equals("total")) { haveCountMap.put(groupName, totalCount > 0); } else if (rowName.endsWith("%")) { if (haveCount) { String keyPrefix = rowName.substring(0, rowName.length() - 1); String key = keyPrefix + "|correct"; DescriptiveStatistics correctStat = stats.get(key); if (correctStat == null) { correctStat = new DescriptiveStatistics(); stats.put(key, correctStat); } correctStat.addValue(correctCount); key = keyPrefix + "|error"; DescriptiveStatistics errorStat = stats.get(key); if (errorStat == null) { errorStat = new DescriptiveStatistics(); stats.put(key, errorStat); } errorStat.addValue(errorCount); key = keyPrefix + "|total"; DescriptiveStatistics totalStat = stats.get(key); if (totalStat == null) { totalStat = new DescriptiveStatistics(); stats.put(key, totalStat); } totalStat.addValue(totalCount); } } j++; } } i++; } } else { Scanner scanner = new Scanner(file); boolean firstLine = true; while (scanner.hasNextLine()) { String line = scanner.nextLine(); if (firstLine) { if (firstFile) writer.write(line + "\n"); firstLine = false; } else { writer.write(line + "\n"); } writer.flush(); } } // file type } // next file Writer statsWriter = writers.get("KEMatrix"); writeStats(statsWriter, errorMap); statsWriter.write("\n"); String[] statTypes = new String[] { "known", "unknown", "goodSeg", "badSeg", "inBeam", "total", "knownLetter", "unknownLetter", "goodSegLetter", "badSegLetter", "totalLetter" }; for (String statType : statTypes) { for (String groupName : groupNames) { Map<String, DescriptiveStatistics> statsMap = statMap.get(groupName); DescriptiveStatistics correctStat = statsMap.get(statType + "|correct"); DescriptiveStatistics errorStat = statsMap.get(statType + "|error"); DescriptiveStatistics totalStat = statsMap.get(statType + "|total"); statsWriter.write(CSV.format(statType + "%Avg") + CSV.format(correctStat.getMean()) + CSV.format(errorStat.getMean()) + CSV.format(totalStat.getMean()) + CSV.getCsvSeparator()); } // next group statsWriter.write("\n"); for (String groupName : groupNames) { Map<String, DescriptiveStatistics> statsMap = statMap.get(groupName); DescriptiveStatistics correctStat = statsMap.get(statType + "|correct"); DescriptiveStatistics errorStat = statsMap.get(statType + "|error"); DescriptiveStatistics totalStat = statsMap.get(statType + "|total"); statsWriter.write(CSV.format(statType + "%Dev") + CSV.format(correctStat.getStandardDeviation()) + CSV.format(errorStat.getStandardDeviation()) + CSV.format(totalStat.getStandardDeviation()) + CSV.getCsvSeparator()); } // next group statsWriter.write("\n"); statsWriter.flush(); } statsWriter.close(); } catch (IOException e) { LogUtils.logError(LOG, e); throw new RuntimeException(e); } }
From source file:com.joliciel.csvLearner.CSVLearner.java
private void doCommandEvaluate() throws IOException { if (resultFilePath == null) throw new RuntimeException("Missing argument: resultFile"); if (featureDir == null) throw new RuntimeException("Missing argument: featureDir"); if (testIdFilePath != null) { if (crossValidation) throw new RuntimeException("Cannot combine testIdFile with cross validation"); if (testSegment >= 0) { throw new RuntimeException("Cannot combine testIdFile with test segment"); }/*from ww w . ja v a2 s .co m*/ } if (!crossValidation && testIdFilePath == null) { if (testSegment < 0) throw new RuntimeException("Missing argument: testSegment"); if (testSegment > 9) throw new RuntimeException("testSegment must be an integer between 0 and 9"); } if (outDirPath == null) throw new RuntimeException("Missing argument: outDir"); LOG.info("Generating event list from CSV files..."); CSVEventListReader reader = this.getReader(TrainingSetType.TEST_SEGMENT, false); GenericEvents events = reader.getEvents(); File outDir = new File(outDirPath); outDir.mkdirs(); String fileBase = this.featureDir.replace('/', '_'); fileBase = fileBase.replace(':', '_'); fileBase = fileBase + "_cutoff" + cutoff; if (generateEventFile) { File eventFile = new File(outDir, fileBase + "_events.txt"); this.generateEventFile(eventFile, events); } File fscoreFile = new File(outDir, fileBase + "_fscores.csv"); Writer fscoreFileWriter = new BufferedWriter( new OutputStreamWriter(new FileOutputStream(fscoreFile, false), "UTF8")); File outcomeFile = new File(outDir, fileBase + "_outcomes.csv"); Writer outcomeFileWriter = new BufferedWriter( new OutputStreamWriter(new FileOutputStream(outcomeFile, false), "UTF8")); try { if (!crossValidation) { MaxentModel maxentModel = this.train(events, null); this.evaluate(maxentModel, events, fscoreFileWriter, outcomeFileWriter); } else { DescriptiveStatistics accuracyStats = new DescriptiveStatistics(); Map<String, DescriptiveStatistics[]> outcomeFscoreStats = new TreeMap<String, DescriptiveStatistics[]>(); for (int segment = 0; segment <= 9; segment++) { outcomeFileWriter.write("Run " + segment + ",\n"); fscoreFileWriter.write("Run " + segment + ",\n"); if (balanceOutcomes) { for (String outcome : reader.getOutcomes()) { int i = 0; for (GenericEvent event : events) { if (event.getOutcome().equals(outcome)) { boolean test = i % 10 == segment; event.setTest(test); i++; } } } } else { int i = 0; for (GenericEvent event : events) { boolean test = i % 10 == segment; event.setTest(test); i++; } } MaxentModel maxentModel = this.train(events, null); FScoreCalculator<String> fscoreCalculator = this.evaluate(maxentModel, events, fscoreFileWriter, outcomeFileWriter); accuracyStats.addValue(fscoreCalculator.getTotalFScore()); for (String outcome : fscoreCalculator.getOutcomeSet()) { DescriptiveStatistics[] stats = outcomeFscoreStats.get(outcome); if (stats == null) { stats = new DescriptiveStatistics[3]; stats[0] = new DescriptiveStatistics(); stats[1] = new DescriptiveStatistics(); stats[2] = new DescriptiveStatistics(); outcomeFscoreStats.put(outcome, stats); } stats[0].addValue(fscoreCalculator.getPrecision(outcome)); stats[1].addValue(fscoreCalculator.getRecall(outcome)); stats[2].addValue(fscoreCalculator.getFScore(outcome)); } // next outcome outcomeFileWriter.write("\n"); } // next segment fscoreFileWriter.write( "outcome,precision avg., precision dev., recall avg., recall dev., f-score avg., f-score dev.,\n"); for (String outcome : outcomeFscoreStats.keySet()) { DescriptiveStatistics[] stats = outcomeFscoreStats.get(outcome); fscoreFileWriter .write(CSVFormatter.format(outcome) + "," + CSVFormatter.format(stats[0].getMean()) + "," + CSVFormatter.format(stats[0].getStandardDeviation()) + "," + CSVFormatter.format(stats[1].getMean()) + "," + CSVFormatter.format(stats[1].getStandardDeviation()) + "," + CSVFormatter.format(stats[2].getMean()) + "," + CSVFormatter.format(stats[2].getStandardDeviation()) + "," + "\n"); } fscoreFileWriter.write("TOTAL,,,,," + CSVFormatter.format(accuracyStats.getMean()) + "," + CSVFormatter.format(accuracyStats.getStandardDeviation()) + ",\n"); LOG.info("Accuracy mean: " + accuracyStats.getMean()); LOG.info("Accuracy std dev: " + accuracyStats.getStandardDeviation()); } } finally { fscoreFileWriter.flush(); fscoreFileWriter.close(); outcomeFileWriter.flush(); outcomeFileWriter.close(); } LOG.info("#### Complete ####"); }
From source file:datafu.hourglass.jobs.StagedOutputJob.java
/** * Writes Hadoop counters and other task statistics to a file in the file system. * // w ww . jav a2 s .c o m * @param fs * @throws IOException */ private void writeCounters(final FileSystem fs) throws IOException { final Path actualOutputPath = FileOutputFormat.getOutputPath(this); SimpleDateFormat timestampFormat = new SimpleDateFormat("yyyyMMddHHmmss"); String suffix = timestampFormat.format(new Date()); if (_countersParentPath != null) { if (!fs.exists(_countersParentPath)) { _log.info("Creating counter parent path " + _countersParentPath); fs.mkdirs(_countersParentPath, FsPermission.valueOf("-rwxrwxr-x")); } // make the name as unique as possible in this case because this may be a directory // where other counter files will be dropped _countersPath = new Path(_countersParentPath, ".counters." + suffix); } else { _countersPath = new Path(actualOutputPath, ".counters." + suffix); } _log.info(String.format("Writing counters to %s", _countersPath)); FSDataOutputStream counterStream = fs.create(_countersPath); BufferedOutputStream buffer = new BufferedOutputStream(counterStream, 256 * 1024); OutputStreamWriter writer = new OutputStreamWriter(buffer); for (String groupName : getCounters().getGroupNames()) { for (Counter counter : getCounters().getGroup(groupName)) { writeAndLog(writer, String.format("%s=%d", counter.getName(), counter.getValue())); } } JobID jobID = this.getJobID(); org.apache.hadoop.mapred.JobID oldJobId = new org.apache.hadoop.mapred.JobID(jobID.getJtIdentifier(), jobID.getId()); long minStart = Long.MAX_VALUE; long maxFinish = 0; long setupStart = Long.MAX_VALUE; long cleanupFinish = 0; DescriptiveStatistics mapStats = new DescriptiveStatistics(); DescriptiveStatistics reduceStats = new DescriptiveStatistics(); boolean success = true; JobClient jobClient = new JobClient(this.conf); Map<String, String> taskIdToType = new HashMap<String, String>(); TaskReport[] setupReports = jobClient.getSetupTaskReports(oldJobId); if (setupReports.length > 0) { _log.info("Processing setup reports"); for (TaskReport report : jobClient.getSetupTaskReports(oldJobId)) { taskIdToType.put(report.getTaskID().toString(), "SETUP"); if (report.getStartTime() == 0) { _log.warn("Skipping report with zero start time"); continue; } setupStart = Math.min(setupStart, report.getStartTime()); } } else { _log.error("No setup reports"); } TaskReport[] mapReports = jobClient.getMapTaskReports(oldJobId); if (mapReports.length > 0) { _log.info("Processing map reports"); for (TaskReport report : mapReports) { taskIdToType.put(report.getTaskID().toString(), "MAP"); if (report.getFinishTime() == 0 || report.getStartTime() == 0) { _log.warn("Skipping report with zero start or finish time"); continue; } minStart = Math.min(minStart, report.getStartTime()); mapStats.addValue(report.getFinishTime() - report.getStartTime()); } } else { _log.error("No map reports"); } TaskReport[] reduceReports = jobClient.getReduceTaskReports(oldJobId); if (reduceReports.length > 0) { _log.info("Processing reduce reports"); for (TaskReport report : reduceReports) { taskIdToType.put(report.getTaskID().toString(), "REDUCE"); if (report.getFinishTime() == 0 || report.getStartTime() == 0) { _log.warn("Skipping report with zero start or finish time"); continue; } maxFinish = Math.max(maxFinish, report.getFinishTime()); reduceStats.addValue(report.getFinishTime() - report.getStartTime()); } } else { _log.error("No reduce reports"); } TaskReport[] cleanupReports = jobClient.getCleanupTaskReports(oldJobId); if (cleanupReports.length > 0) { _log.info("Processing cleanup reports"); for (TaskReport report : cleanupReports) { taskIdToType.put(report.getTaskID().toString(), "CLEANUP"); if (report.getFinishTime() == 0) { _log.warn("Skipping report with finish time of zero"); continue; } cleanupFinish = Math.max(cleanupFinish, report.getFinishTime()); } } else { _log.error("No cleanup reports"); } if (minStart == Long.MAX_VALUE) { _log.error("Could not determine map-reduce start time"); success = false; } if (maxFinish == 0) { _log.error("Could not determine map-reduce finish time"); success = false; } if (setupStart == Long.MAX_VALUE) { _log.error("Could not determine setup start time"); success = false; } if (cleanupFinish == 0) { _log.error("Could not determine cleanup finish time"); success = false; } // Collect statistics on successful/failed/killed task attempts, categorized by setup/map/reduce/cleanup. // Unfortunately the job client doesn't have an easier way to get these statistics. Map<String, Integer> attemptStats = new HashMap<String, Integer>(); _log.info("Processing task attempts"); for (TaskCompletionEvent event : getTaskCompletionEvents(jobClient, oldJobId)) { String type = taskIdToType.get(event.getTaskAttemptId().getTaskID().toString()); String status = event.getTaskStatus().toString(); String key = String.format("%s_%s_ATTEMPTS", status, type); if (!attemptStats.containsKey(key)) { attemptStats.put(key, 0); } attemptStats.put(key, attemptStats.get(key) + 1); } if (success) { writeAndLog(writer, String.format("SETUP_START_TIME_MS=%d", setupStart)); writeAndLog(writer, String.format("CLEANUP_FINISH_TIME_MS=%d", cleanupFinish)); writeAndLog(writer, String.format("COMPLETE_WALL_CLOCK_TIME_MS=%d", cleanupFinish - setupStart)); writeAndLog(writer, String.format("MAP_REDUCE_START_TIME_MS=%d", minStart)); writeAndLog(writer, String.format("MAP_REDUCE_FINISH_TIME_MS=%d", maxFinish)); writeAndLog(writer, String.format("MAP_REDUCE_WALL_CLOCK_TIME_MS=%d", maxFinish - minStart)); writeAndLog(writer, String.format("MAP_TOTAL_TASKS=%d", (long) mapStats.getN())); writeAndLog(writer, String.format("MAP_MAX_TIME_MS=%d", (long) mapStats.getMax())); writeAndLog(writer, String.format("MAP_MIN_TIME_MS=%d", (long) mapStats.getMin())); writeAndLog(writer, String.format("MAP_AVG_TIME_MS=%d", (long) mapStats.getMean())); writeAndLog(writer, String.format("MAP_STD_TIME_MS=%d", (long) mapStats.getStandardDeviation())); writeAndLog(writer, String.format("MAP_SUM_TIME_MS=%d", (long) mapStats.getSum())); writeAndLog(writer, String.format("REDUCE_TOTAL_TASKS=%d", (long) reduceStats.getN())); writeAndLog(writer, String.format("REDUCE_MAX_TIME_MS=%d", (long) reduceStats.getMax())); writeAndLog(writer, String.format("REDUCE_MIN_TIME_MS=%d", (long) reduceStats.getMin())); writeAndLog(writer, String.format("REDUCE_AVG_TIME_MS=%d", (long) reduceStats.getMean())); writeAndLog(writer, String.format("REDUCE_STD_TIME_MS=%d", (long) reduceStats.getStandardDeviation())); writeAndLog(writer, String.format("REDUCE_SUM_TIME_MS=%d", (long) reduceStats.getSum())); writeAndLog(writer, String.format("MAP_REDUCE_SUM_TIME_MS=%d", (long) mapStats.getSum() + (long) reduceStats.getSum())); for (Map.Entry<String, Integer> attemptStat : attemptStats.entrySet()) { writeAndLog(writer, String.format("%s=%d", attemptStat.getKey(), attemptStat.getValue())); } } writer.close(); buffer.close(); counterStream.close(); }
From source file:org.bresearch.websec.test.CommonsMathTest.java
public void test1() throws Exception { /* min, max, mean, geometric mean, n, sum, sum of squares, * standard deviation, variance, percentiles, skewness, kurtosis, median */ // Get a DescriptiveStatistics instance using factory method DescriptiveStatistics stats = new DescriptiveStatistics(); final double[] inputArray = { 4, 3, 3, 2 }; // Add the data from the array for (int i = 0; i < inputArray.length; i++) { stats.addValue(inputArray[i]);//www . j av a 2s . c o m } // Compute some statistics double mean = stats.getMean(); double std = stats.getStandardDeviation(); long n = stats.getN(); assertEquals("3.0", "" + mean); assertEquals("0.816496580927726", "" + std); }
From source file:org.bresearch.websec.test.CommonsMathTest.java
public void test2() { final String data2 = (new WordProcessor()).filterOnlyAlphaNumeric( " !!! Hello my name is a person. Hello how are you doing. hello, this is great. What do you think? "); final BotlistStringUtils utils = new BotlistStringUtils(); final List<String> a = utils.buildWordList(data2); assertEquals(27, a.size());/*w ww. j a va 2 s . c o m*/ DescriptiveStatistics stats = new DescriptiveStatistics(); for (int i = 0; i < utils.mapReduceCount(a, -1).length; i++) { stats.addValue(utils.mapReduceCount(a, -1)[i]); } // Compute some statistics double mean = stats.getMean(); double std = stats.getStandardDeviation(); assertEquals("1.2666666666666666", "" + mean); assertEquals("0.5936168397046634", "" + std); long n = stats.getN(); assertEquals("15", "" + n); }
From source file:org.fhaes.analysis.FHInterval.java
/** * Actually perform the analysis./*from w w w . jav a 2 s.c o m*/ */ @SuppressWarnings("deprecation") private void doAnalysis() { log.debug("INPUT PARAMETERS"); log.debug("inputFileArray = " + inputFileArray); log.debug("analyissType = " + analysisType); log.debug("startYear = " + startYear); log.debug("endYear = " + endYear); log.debug("fireFilterType = " + fireFilterType); log.debug("filterValue = " + filterValue); log.debug("includeIncomplete = " + includeIncomplete); log.debug("alphaLevel = " + alphaLevel); boolean highway = true; ArrayList<FHX2FileReader> myReader = new ArrayList<FHX2FileReader>(); Integer minFirstYear = new Integer(9999); Integer maxLastYear = new Integer(0); String savePath = new String(); savePath = inputFileArray[0].getAbsolutePath(); for (int i = 0; i < inputFileArray.length; i++) { myReader.add(new FHX2FileReader(inputFileArray[i])); /* * set the beginning year accounting for the filter */ if (eventTypeToProcess.equals(EventTypeToProcess.FIRE_EVENT)) { // myReader.get(i).makeClimate2d(); if (startYear == 0 && highway) { if (myReader.get(i).getFirstFireYear() < minFirstYear) { minFirstYear = myReader.get(i).getFirstFireYear(); } } else if (startYear != 0 && highway) { if (myReader.get(i).getFirstYear() < minFirstYear) { minFirstYear = myReader.get(i).getFirstYear(); } } if (startYear != 0) { minFirstYear = startYear; // minFirstYear = minFirstYear+1; } } else if (eventTypeToProcess.equals(EventTypeToProcess.INJURY_EVENT)) { // myReader.get(i).makeClimate2dII(); if (startYear == 0 && highway) { if (myReader.get(i).getFirstInjuryYear() < minFirstYear) { minFirstYear = myReader.get(i).getFirstInjuryYear(); } } else if (startYear != 0 && highway) { if (myReader.get(i).getFirstYear() < minFirstYear) { minFirstYear = myReader.get(i).getFirstYear(); } } if (startYear != 0) { minFirstYear = startYear; // minFirstYear = minFirstYear+1; } } else if (eventTypeToProcess.equals(EventTypeToProcess.FIRE_AND_INJURY_EVENT)) { // myReader.get(i).makeClimate2dII(); if (startYear == 0 && highway) { if (myReader.get(i).getFirstIndicatorYear() < minFirstYear) { minFirstYear = myReader.get(i).getFirstIndicatorYear(); } } else if (startYear != 0 && highway) { if (myReader.get(i).getFirstYear() < minFirstYear) { minFirstYear = myReader.get(i).getFirstYear(); } } if (startYear != 0) { minFirstYear = startYear; // minFirstYear = minFirstYear+1; } } else { log.error("Unsupported event type caught"); } /* * Set the last year accounting for the filter */ if (myReader.get(i).getLastYear() > maxLastYear) { maxLastYear = myReader.get(i).getLastYear(); } if (endYear != 0) { maxLastYear = endYear; } } // end of i loop log.debug("the input filelength is " + inputFileArray.length); log.debug("The FIRST FIRE YEAR is " + minFirstYear); log.debug("The LAST YEAR is " + maxLastYear); log.debug("Minimum and Maximum years are " + minFirstYear + " " + maxLastYear); /* * set the format for the output of the numbers to 2 decimal formats */ DecimalFormat twoPlace = new DecimalFormat("0.00"); DecimalFormat threePlace = new DecimalFormat("0.000"); /* * Calculate the listYears the common years where the files will be analyzed */ ArrayList<Integer> listYears = new ArrayList<Integer>(); for (int i = 0; i < maxLastYear - minFirstYear + 1; i++) { listYears.add(minFirstYear + i); } /* * create arraylist need for the Interval Analysis */ ArrayList<ArrayList<Integer>> climateMatrixSite = new ArrayList<ArrayList<Integer>>(); ArrayList<ArrayList<Double>> filterMatrix = new ArrayList<ArrayList<Double>>(); ArrayList<Integer> climateVector = new ArrayList<Integer>(); ArrayList<ArrayList<Double>> climateVectorFilter2 = new ArrayList<ArrayList<Double>>(); // ArrayList<Double> fireintervalspersite = new ArrayList<Double>(); ArrayList<Integer> climateVectorActualSite = null; ArrayList<Double> filterVectorActual = null; ArrayList<Integer> climateYear = new ArrayList<Integer>(); ArrayList<Integer> minSampleFilter = null; ArrayList<Double> percentOfRecordingfilter = null; Double[] Dfireintervalspersite; double[] dfireintervalspersite; String[] statsparam = new String[22]; if (eventTypeToProcess.equals(EventTypeToProcess.FIRE_EVENT)) { statsparam[0] = "Total intervals"; statsparam[1] = "Mean fire interval"; statsparam[2] = "Median fire interval"; statsparam[3] = "Standard deviation"; statsparam[4] = "Fire frequency"; statsparam[5] = "Coefficient of variation"; statsparam[6] = "Skewness"; statsparam[7] = "Kurtosis"; statsparam[8] = "Minimum fire interval"; statsparam[9] = "Maximum fire interval"; statsparam[10] = "Weibull scale parameter"; statsparam[11] = "Weibull shape parameter"; statsparam[12] = "Weibull mean"; statsparam[13] = "Weibull median"; statsparam[14] = "Weibull mode"; statsparam[15] = "Weibull standard deviation"; statsparam[16] = "Weibull fire frequency"; statsparam[17] = "Weibull skewness"; statsparam[18] = "Lower exceedance interval"; statsparam[19] = "Upper exceedance interval"; statsparam[20] = "Significantly short interval upper bound"; statsparam[21] = "Significantly long interval lower bound"; } else if (eventTypeToProcess.equals(EventTypeToProcess.INJURY_EVENT)) { statsparam[0] = "Total intervals"; statsparam[1] = "Mean indicator interval"; statsparam[2] = "Median indicator interval"; statsparam[3] = "Standard deviation"; statsparam[4] = "Indicator frequency"; statsparam[5] = "Coefficient of variation"; statsparam[6] = "Skewness"; statsparam[7] = "Kurtosis"; statsparam[8] = "Minimum fire interval"; statsparam[9] = "Maximum indicator interval"; statsparam[10] = "Weibull scale parameter"; statsparam[11] = "Weibull shape parameter"; statsparam[12] = "Weibull mean"; statsparam[13] = "Weibull median"; statsparam[14] = "Weibull mode"; statsparam[15] = "Weibull standard deviation"; statsparam[16] = "Weibull indicator frequency"; statsparam[17] = "Weibull skewness"; statsparam[18] = "Lower exceedance interval"; statsparam[19] = "Upper exceedance interval"; statsparam[20] = "Significantly short interval upper bound"; statsparam[21] = "Significantly long interval lower bound"; } else if (eventTypeToProcess.equals(EventTypeToProcess.FIRE_AND_INJURY_EVENT)) { statsparam[0] = "Total intervals"; statsparam[1] = "Mean fire and other indicator interval"; statsparam[2] = "Median fire and other indicator interval"; statsparam[3] = "Standard deviation"; statsparam[4] = "Fire and other indicator frequency"; statsparam[5] = "Coefficient of variation"; statsparam[6] = "Skewness"; statsparam[7] = "Kurtosis"; statsparam[8] = "Minimum fire and other indicator interval"; statsparam[9] = "Maximum fire interval"; statsparam[10] = "Weibull scale parameter"; statsparam[11] = "Weibull shape parameter"; statsparam[12] = "Weibull mean"; statsparam[13] = "Weibull median"; statsparam[14] = "Weibull mode"; statsparam[15] = "Weibull standard deviation"; statsparam[16] = "Weibull indicator frequency"; statsparam[17] = "Weibull skewness"; statsparam[18] = "Lower exceedance interval"; statsparam[19] = "Upper exceedance interval"; statsparam[20] = "Significantly short interval upper bound"; statsparam[21] = "Significantly long interval lower bound"; } else { log.error("Unsupported event type caught"); } double[] fixvalt = { 0.999, 0.99, 0.975, 0.95, 0.9, 0.875, 0.8, 0.75, 0.7, 0.667, 0.5, 0.333, 0.3, 0.25, 0.2, 0.125, 0.1, 0.05, 0.025, 0.01, 0.001 }; double[][] ExceeProbcomp = new double[fixvalt.length][myReader.size()]; double[][] ExceeProbsample = new double[fixvalt.length][myReader.size()]; // log.debug("the size of statsparam is " + // statsparam.length); double[][] summaryComp = new double[statsparam.length][myReader.size()]; double[] numberOfintervalscomp = new double[myReader.size()]; // ArrayList<ArrayList<Integer>>(); // ArrayList<ArrayList<Integer>> FIyearperSample = new // ArrayList<ArrayList<Integer>>(); ArrayList<Integer> FyearperSampletemp; ArrayList<Integer> FIyearperSampletemp; // ArrayList<Double> fireintervalspersample = new double[] numberOfintervalssamp = new double[myReader.size()]; double[][] summarySample = new double[statsparam.length][myReader.size()]; Double[] Dfireintervalspersample; double[] dfireintervalspersample; /* * Set up either of the two filters two create the binary matrix on the case of composite analysis there are two possible filters: * Number of fires and percentage of scarred trees. */ Integer firesFilter1 = new Integer(0); Double firesFilter2 = new Double(0); if (fireFilterType.equals(FireFilterType.NUMBER_OF_EVENTS)) { if (filterValue != 1) firesFilter1 = filterValue.intValue(); // log.debug("number of fires is selected is: "+ // firesFilter1); } else if (fireFilterType.equals(FireFilterType.PERCENTAGE_OF_ALL_TREES)) { if (filterValue != 1) firesFilter2 = filterValue / 100.0; // log.debug("percentage of fires is selected is: "+ // firesFilter2); } else if (fireFilterType.equals(FireFilterType.PERCENTAGE_OF_RECORDING)) { if (filterValue != 1) firesFilter2 = filterValue / 100.0; // TODO ELENA TO CHECK } else { log.error("Unknown FireFilterType"); return; } boolean[] enoughIntComp = new boolean[myReader.size()]; boolean[] enoughIntSamp = new boolean[myReader.size()]; // NEW FOR ELENA log.debug("Sample depth filter type = " + sampleDepthFilterType); log.debug("Sample depth value = " + sampleDepthFilterValue); // if (sampleDepthFilterType.equals(SampleDepthFilterType.MIN_NUM_SAMPLES)) // { // // TODO ELENA // } // else if (sampleDepthFilterType.equals(SampleDepthFilterType.MIN_NUM_RECORDER_SAMPLES)) // { // // TODO ELENA // } /* * start processing each file individually: The analysis can be done by either tree (by sample/non-binary) or by site * (composite/binary). by tree the box selected is: jCheckTree. by site the box selected is: */ for (int i = 0; i < myReader.size(); i++) { log.debug(" Starting to Process file : " + myReader.get(i).getName()); /* * get the vector Year containing the vector of year of a given fhx file load it into the array list climateYear. */ climateYear = myReader.get(i).getYearArray(); // new stuff // Create filter based on min number of samples/recorder samples int[] depths = null; if (sampleDepthFilterType.equals(SampleDepthFilterType.MIN_NUM_SAMPLES)) { depths = myReader.get(i).getSampleDepths(); log.debug("MIN_NUM_SAMPLES "); } else if (sampleDepthFilterType.equals(SampleDepthFilterType.MIN_NUM_RECORDER_SAMPLES)) { depths = myReader.get(i).getRecordingDepths(eventTypeToProcess); log.debug(" MIN_NUM_RECORDER_SAMPLES"); } else { log.error("Unknown sample depth filter type."); return; } minSampleFilter = new ArrayList<Integer>(); for (int ij = 0; ij < listYears.size(); ij++) { if (climateYear.indexOf(listYears.get(ij)) == -1) { minSampleFilter.add(-1); } else { // log.debug("the sample depth is " // + myReader.get(i).getSampleDepths()[climateYear.indexOf(listYearsComp.get(ij))]); minSampleFilter.add(new Integer(depths[climateYear.indexOf(listYears.get(ij))])); } // log.debug(" " + minSampleFilter.get(ij)); } // end new stuff /* * get filter matrix for each file. * * filters2d matrix composed of the 3 filters number of fires (total capital letter per row) total number of tree (total lower * case letter plus bars counting only after a fire) percent of scared trees total fires/total trees */ // climateVectorFilter2 = myReader.get(i).getFilterArrays(eventTypeToProcess); /* * More new stuff */ if (filterValue != 1) { /* * get both matrices: * * 2. filters2d matrix composed of the 3 filters number of fires (total capital letter per row) total number of tree (total * lower case letter plus bars counting only after a fire) percent of scared trees total fires/total trees */ climateVectorFilter2 = myReader.get(i).getFilterArrays(eventTypeToProcess); /* * if by tree analysis is selected create two matrices (array list) 1. filterMatrix containing the three filter vectors only * in between common years (so using the listYearComp array list subset of the years vector) 2. climateMatrix 2 dimensional * array list containing binary matrices restricted to the listYear list. */ if (fireFilterType.equals(FireFilterType.PERCENTAGE_OF_RECORDING)) { percentOfRecordingfilter = new ArrayList<Double>(); for (int ij = 0; ij < listYears.size(); ij++) { if (climateYear.indexOf(listYears.get(ij)) == -1) { percentOfRecordingfilter.add(-1.0); } else { if (myReader.get(i).getRecordingDepths(eventTypeToProcess)[climateYear .indexOf(listYears.get(ij))] != 0) { percentOfRecordingfilter.add(new Double( climateVectorFilter2.get(0).get(climateYear.indexOf(listYears.get(ij))) / myReader.get(i).getRecordingDepths(eventTypeToProcess)[climateYear .indexOf(listYears.get(ij))])); } else { percentOfRecordingfilter.add(-99.0); } } log.debug("PERCENTAGE_OF_RECORDING is: " + percentOfRecordingfilter.get(ij)); } } else { for (int ik = 0; ik < 3; ik++) { log.debug("filter number is: " + ik); filterVectorActual = new ArrayList<Double>(); for (int ij = 0; ij < listYears.size(); ij++) { // log.debug(" climateYear.indexOf(listYearsComp.get(j))" + // climateYear.indexOf(listYearsComp.get(ij))); // if(ik==0){log.debug("number of fires // "+climateVectorFilter2.get(0).get(climateYear.indexOf(listYears.get(ij)))+" year // "+listYearsComp.get(ij));} if (climateYear.indexOf(listYears.get(ij)) == -1) { filterVectorActual.add(-1.0); } else { filterVectorActual.add(new Double( climateVectorFilter2.get(ik).get(climateYear.indexOf(listYears.get(ij))))); } if (ik == 2) { log.debug("filteperc " + filterVectorActual.get(ij)); } } // log.debug("size of filterVectorActual is : "+filterVectorActual.size()); filterMatrix.add(filterVectorActual); // if(ik==0){log.debug("filters is: "+filter); } } // end of if-else percentageofrecording // log.debug("size of the FilterMatrix is" + filterMatrix.size()); } // end of if filters not equal to 1 /* * end of more new stuff */ /* * * 1. Create the filterMatrix containing the tree filter vectors only in between common years (so using the listYearComp array * list subset of the years vector) */ // for (int ik = 0; ik < 3; ik++) // { // filterVectorActual = new ArrayList<Double>(); // for (int ij = 0; ij < listYears.size(); ij++) // { // if (climateYear.indexOf(listYears.get(ij)) == -1) // { // filterVectorActual.add(-1.0); // } // else // { // filterVectorActual.add(new Double(climateVectorFilter2.get(ik).get(climateYear.indexOf(listYears.get(ij))))); // } // } /* * ArrayList filterMatrix containes the filter matrix for each of the files */ // filterMatrix.add(filterVectorActual); // }//end of creating the filter matrix. /* * get matrix climate binary matrix by site (binary analysis) if Composite is selected. */ // if ((doComposite)&&(!jTextOfFires.getText().equals("1"))){ if (analysisType.equals(AnalysisType.COMPOSITE)) { log.debug("inside the comp"); // System.out.println("inside the comp " + " working on file "+ myReader.get(i).getName() ); if (eventTypeToProcess.equals(EventTypeToProcess.FIRE_EVENT)) { climateVector = myReader.get(i).getFireEventsArray(); } else if (eventTypeToProcess.equals(EventTypeToProcess.INJURY_EVENT)) { climateVector = myReader.get(i).getOtherInjuriesArray(); } else if (eventTypeToProcess.equals(EventTypeToProcess.FIRE_AND_INJURY_EVENT)) { climateVector = myReader.get(i).getFiresAndInjuriesArray(); } else { log.error("Unsupported event type caught"); } climateVectorActualSite = new ArrayList<Integer>(); for (int j = 0; j < listYears.size(); j++) { if (climateYear.indexOf(listYears.get(j)) == -1) { climateVectorActualSite.add(-1); } else { if (minSampleFilter.get(j).intValue() >= sampleDepthFilterValue.intValue()) { if (filterValue != 1) { if (fireFilterType.equals(FireFilterType.NUMBER_OF_EVENTS)) { // log.debug("fire filter: "+firesFilter1+" year is: "+listYears.get(j) // +" fires: "+filterMatrix.get(3*i).get(j)+" climatevector: // "+climateVector.get(climateYear.indexOf(listYears.get(j)))); if ((filterMatrix.get(3 * i).get(j) < firesFilter1) && (climateVector.get(climateYear.indexOf(listYears.get(j)))) != -1.0) { climateVectorActualSite.add(0); } else { climateVectorActualSite .add(climateVector.get(climateYear.indexOf(listYears.get(j)))); } } else if (fireFilterType.equals(FireFilterType.PERCENTAGE_OF_ALL_TREES)) { // log.debug("percent of fires is selected is: "+ // firesFilter2+" "+climateVector.get(climateYear.indexOf(listYearsComp.get(j)))); // log.debug("the filter percent of fires is"+filterMatrix.get((3*i+2)).get(j)); if ((filterMatrix.get(3 * i + 2).get(j) == -99)) { climateVectorActualSite.add(-1); } else { if ((filterMatrix.get(3 * i + 2).get(j) < firesFilter2) && ((climateVector .get(climateYear.indexOf(listYears.get(j)))) != -1.0)) { climateVectorActualSite.add(0); } else { climateVectorActualSite .add(climateVector.get(climateYear.indexOf(listYears.get(j)))); } } } else if (fireFilterType.equals(FireFilterType.PERCENTAGE_OF_RECORDING)) { // TODO // ELENA TO IMPLEMENT if (percentOfRecordingfilter.get(j) == -99) { climateVectorActualSite.add(-1); } else { if ((percentOfRecordingfilter.get(j) < firesFilter2) && ((climateVector .get(climateYear.indexOf(listYears.get(j)))) != -1.0)) { climateVectorActualSite.add(0); } else { climateVectorActualSite .add(climateVector.get(climateYear.indexOf(listYears.get(j)))); } } } else { log.error("Unknown FireFilterType"); return; } } // end of if filter not equal to 1 else { climateVectorActualSite .add(climateVector.get(climateYear.indexOf(listYears.get(j)))); } // end of else of if filter not equal to 1 } // end of if the filter minsampedepth else { // log.debug("j is " + j + "minSampleFilter is " + minSampleFilter.get(j)); climateVectorActualSite.add(-1); } } // end else 645 to 721 } // end of j loop listyears (420-459) /* * climateMatrixSite has the composite information taking in consideration both the filters and common years */ climateMatrixSite.add(climateVectorActualSite); /* * Must get the years with Fires from the climateMatrixSite which has been filter already */ ArrayList<Double> fireintervalspersite = new ArrayList<Double>(); ArrayList<Integer> yearsWithFires = new ArrayList<Integer>(); for (int ij = 0; ij < listYears.size(); ij++) { if (climateMatrixSite.get(i).get(ij) == 1) { yearsWithFires.add(listYears.get(ij)); log.debug("year with fires " + listYears.get(ij)); } } /* * check that the number of years with fires is bigger of equal than 3 if so make the fire intervals else the test can not * be run and report NA */ // new swich if (yearsWithFires.size() != 0) { if (includeIncomplete) { if (maxLastYear.compareTo(yearsWithFires.get(yearsWithFires.size() - 1)) != 0) { log.debug("here"); numberOfintervalscomp[i] = yearsWithFires.size(); } else { numberOfintervalscomp[i] = yearsWithFires.size() - 1; } } else { numberOfintervalscomp[i] = yearsWithFires.size() - 1; } } log.debug("number of invervlas in comp is: " + numberOfintervalscomp[i]); // end of new switch if (numberOfintervalscomp[i] >= 3) { enoughIntComp[i] = true; ArrayList<Integer> fireIntervals = generateFireIntervals(yearsWithFires); for (int ij = 0; ij < fireIntervals.size(); ij++) { // log.debug("fire intervals are: "+ // test1.getFireIntervals().get(ij)); fireintervalspersite.add(fireIntervals.get(ij) * 1.0); } /* * Add extra interval if include incomplete is selected. the interval goes from the last fire scar year to the last year * of the fire history. */ if (includeIncomplete) { double includeinterval = maxLastYear - yearsWithFires.get(yearsWithFires.size() - 1); if (includeinterval > 0) { fireintervalspersite.add(includeinterval); System.out.println("the last year is " + maxLastYear + "the last year with fire is " + yearsWithFires.get(yearsWithFires.size() - 1)); log.debug("the included interval is " + includeinterval); } } log.debug("FireintervalsPerSite =" + fireintervalspersite); /* * Get the normal statistics for the fire intervals add the values to the stats and then call them for the stats */ DescriptiveStatistics stats = new DescriptiveStatistics(); Dfireintervalspersite = new Double[fireintervalspersite.size()]; Dfireintervalspersite = fireintervalspersite.toArray(Dfireintervalspersite); dfireintervalspersite = new double[fireintervalspersite.size()]; // summaryComp = new // double[statsparam.length][myReader.size()]; for (int ik = 0; ik < fireintervalspersite.size(); ik++) { stats.addValue(Dfireintervalspersite[ik].doubleValue()); dfireintervalspersite[ik] = Dfireintervalspersite[ik].doubleValue(); } /* * load the Summary Analysis for the Composite fire intervals */ summaryComp[0][i] = fireintervalspersite.size(); // double mean = stats.getMean(); summaryComp[1][i] = stats.getMean(); // double median = // StatUtils.percentile(dfireintervalspersite, 50); summaryComp[2][i] = StatUtils.percentile(dfireintervalspersite, 50); // double std = stats.getStandardDeviation(); summaryComp[3][i] = stats.getStandardDeviation(); // double skew = stats.getSkewness(); summaryComp[4][i] = 1.0 / summaryComp[1][i]; summaryComp[5][i] = summaryComp[3][i] / summaryComp[1][i]; summaryComp[6][i] = stats.getSkewness(); // double kurt = stats.getKurtosis(); if (numberOfintervalscomp[i] == 3) { summaryComp[7][i] = -99; } else { summaryComp[7][i] = stats.getKurtosis(); } // log.debug("nomean \t\t nostd \t\t nokurt \t noskew \t\t nomedian"); // log.debug(twoPlace.format(mean)+"\t\t"+twoPlace.format(std)+"\t\t"+twoPlace.format(kurt)+"\t\t"+twoPlace.format(skew)+"\t\t"+twoPlace.format(median)); Weibull weibull = new Weibull(fireintervalspersite); ArrayList<Double> weibullProb = weibull.getWeibullProbability(fireintervalspersite); ArrayList<Double> siglonglowbound = new ArrayList<Double>(); ArrayList<Double> sigshortupbound = new ArrayList<Double>(); log.debug("the weibull probability of first element is " + weibullProb.get(0)); log.debug("the index the size of the interval is " + weibullProb.indexOf(weibullProb.get(0))); for (int ij = 0; ij < weibullProb.size() - 1; ij++) { if (weibullProb.get(ij) <= alphaLevel) { siglonglowbound.add(fireintervalspersite.get(ij)); } if (weibullProb.get(ij) >= (1 - alphaLevel)) { sigshortupbound.add(fireintervalspersite.get(ij)); } } summaryComp[10][i] = weibull.getScale(); summaryComp[11][i] = weibull.getShape(); summaryComp[12][i] = weibull.getMean(); summaryComp[13][i] = weibull.getMedian(); summaryComp[14][i] = weibull.getMode(); summaryComp[15][i] = weibull.getSigma(); summaryComp[16][i] = 1.0 / summaryComp[13][i]; summaryComp[17][i] = weibull.getSkew(); summaryComp[18][i] = weibull.getExceedenceProbability2()[0]; summaryComp[19][i] = weibull.getExceedenceProbability2()[1]; Collections.sort(sigshortupbound); log.debug("siglonglowbound is " + siglonglowbound); try { summaryComp[20][i] = sigshortupbound.get(sigshortupbound.size() - 1); } catch (Exception e) { summaryComp[20][i] = Double.NaN; } Collections.sort(siglonglowbound); try { summaryComp[21][i] = siglonglowbound.get(0); } catch (Exception e) { summaryComp[21][i] = Double.NaN; } log.debug("sigshortupbound is " + sigshortupbound); Collections.sort(fireintervalspersite); summaryComp[8][i] = fireintervalspersite.get(0); summaryComp[9][i] = fireintervalspersite.get(fireintervalspersite.size() - 1); // log.debug("shape \t\t scale \t\t median "); // log.debug(twoPlace.format(test1.Weibull_Parameters(fireintervalspersite)[0])+"\t\t"+twoPlace.format(test1.Weibull_Parameters(fireintervalspersite)[1])+"\t\t"+twoPlace.format(test1.weibull_median(test1.Weibull_Parameters(fireintervalspersite)))); // log.debug("mean \t\t sigma \t\t mode \t\t skewness"); // log.debug(twoPlace.format(test1.weibull_mean(test1.Weibull_Parameters(fireintervalspersite)))+"\t\t"+twoPlace.format(test1.weibull_sigma(test1.Weibull_Parameters(fireintervalspersite)))+"\t\t"+twoPlace.format(test1.weibull_mode(test1.Weibull_Parameters(fireintervalspersite)))+"\t\t"+twoPlace.format(test1.weibull_skew(test1.Weibull_Parameters(fireintervalspersite)))); // log.debug("maxhazard \t\t lei \t\t uei "); // log.debug(twoPlace.format(test1.maxhazard_int(test1.Weibull_Parameters(fireintervalspersite)))+"\t\t"+twoPlace.format(test1.weibull_lowuppexcint(test1.Weibull_Parameters(fireintervalspersite))[0])+"\t\t"+twoPlace.format(test1.weibull_lowuppexcint(test1.Weibull_Parameters(fireintervalspersite))[1])); // log.debug("the size of YearWith Fires is "+YearsWithFires.size()); System.out.println("the size of the prb exdc is " + weibull.getExceedenceProbability().length); for (int kk = 0; kk < weibull.getExceedenceProbability().length; kk++) { ExceeProbcomp[kk][i] = weibull.getExceedenceProbability()[kk]; // log.debug("file "+i+"Exce probability "+ // ExceeProbcomp[kk][i]); } } // end of if enoughIntComp else { enoughIntComp[i] = false; } } // end the if composite is selected /* * starting the process for the sample mode. */ if (analysisType.equals(AnalysisType.SAMPLE)) { log.debug("I am in sample "); ArrayList<Double> fireintervalspersample = new ArrayList<Double>(); FIyearperSampletemp = new ArrayList<Integer>(); // FyearperSampletemp = new ArrayList<Integer>(); for (int k = 0; k < myReader.get(i).getNumberOfSeries(); k++) { log.debug("Parsing file index " + i + ", series number " + (k + 1)); FyearperSampletemp = new ArrayList<Integer>(); // log.debug("the size of the years of the file is:"+ // myReader.get(i).getYear().size()); // log.debug("years with fires in sample "+k + // "years are "); // for (int j = 0; j < myReader.get(i).getYearArray().size(); j++) for (int j = 0; j < listYears.size(); j++) { // log.debug("the size climate2d is "+myReader.get(i).getClimate2d().get(k).get(j)); if (eventTypeToProcess.equals(EventTypeToProcess.FIRE_EVENT)) { if (climateYear.indexOf(listYears.get(j)) != -1) { if (myReader.get(i).getClimate2d().get(k) .get(climateYear.indexOf(listYears.get(j))) == 1) { // FyearperSampletemp.add((j + myReader.get(i).getFirstYear())); FyearperSampletemp.add(listYears.get(j)); } } } // { // if ((myReader.get(i).getClimate2d().get(k).get(j) == 1)) // { // / log.debug("I here inside ==1 "+ // / j+" "+myReader.get(i).getFirstYear()); // / int temp=j+myReader.get(i).getFirstYear(); // / log.debug((j+myReader.get(i).getFirstYear())); // /// FyearperSampletemp.add((j + myReader.get(i).getFirstYear())); // } // } else if (eventTypeToProcess.equals(EventTypeToProcess.INJURY_EVENT)) { if (climateYear.indexOf(listYears.get(j)) != -1) { if (myReader.get(i).getClimate2dII().get(k) .get(climateYear.indexOf(listYears.get(j))) == 1) { FyearperSampletemp.add(listYears.get(j)); } } // if ((myReader.get(i).getClimate2dII().get(k).get(j) == 1)) // { // FyearperSampletemp.add((j + myReader.get(i).getFirstYear())); // } } else if (eventTypeToProcess.equals(EventTypeToProcess.FIRE_AND_INJURY_EVENT)) { if (climateYear.indexOf(listYears.get(j)) != -1) { if (myReader.get(i).getClimate2dIII().get(k) .get(climateYear.indexOf(listYears.get(j))) == 1) { FyearperSampletemp.add(listYears.get(j)); } } // if ((myReader.get(i).getClimate2dIII().get(k).get(j) == 1)) // { // FyearperSampletemp.add((j + myReader.get(i).getFirstYear())); // } } else { log.error("Unsupported event type caught"); } } // / end of the loop for listYears in common (finish loading the fire year per sample log.debug( "series number " + (k + 1) + " FyearperSampletemp.size() " + FyearperSampletemp.size()); if (FyearperSampletemp.size() != 0) { if (includeIncomplete) { if (maxLastYear.compareTo(FyearperSampletemp.get(FyearperSampletemp.size() - 1)) != 0) { numberOfintervalssamp[i] = numberOfintervalssamp[i] + FyearperSampletemp.size(); } } else { numberOfintervalssamp[i] = numberOfintervalssamp[i] + (FyearperSampletemp.size() - 1); } } log.debug("series number: " + (k + 1) + " number of intervals " + numberOfintervalssamp[i]); // new if ((FyearperSampletemp.size() == 1) && (includeIncomplete)) { log.debug("last index per sample is " + myReader.get(i).getLastYearIndexPerSample()[k]); log.debug("first year per sample is " + myReader.get(i).getFirstYear()); log.debug("maxLastyear is " + maxLastYear); // if (maxLastYear != FyearperSampletemp.get(FyearperSampletemp.size() - 1)) if (maxLastYear.compareTo(FyearperSampletemp.get(FyearperSampletemp.size() - 1)) != 0) { log.debug("I am in not equal "); log.debug( "last year in the sample is " + (myReader.get(i).getLastYearIndexPerSample()[k] + myReader.get(i).getFirstYear())); log.debug("maxLastyear is " + maxLastYear); log.debug("the last fire year in the sample " + FyearperSampletemp.get(FyearperSampletemp.size() - 1)); if (maxLastYear <= (myReader.get(i).getLastYearIndexPerSample()[k] + myReader.get(i).getFirstYear())) { Integer temp = ((maxLastYear) - FyearperSampletemp.get(FyearperSampletemp.size() - 1)); // int temp1 = maxLastYear.intValue() - FyearperSampletemp.get(FyearperSampletemp.size() - 1).intValue(); log.debug("in less than or equal to "); // temp = (maxLastYear) - FyearperSampletemp.get(FyearperSampletemp.size() - 1); log.debug("the resta temp is " + temp); // FIyearperSampletemp.add(((maxLastYear) - FyearperSampletemp.get(FyearperSampletemp.size() - 1))); if ((maxLastYear) - FyearperSampletemp.get(FyearperSampletemp.size() - 1) > 0) { FIyearperSampletemp.add( (maxLastYear) - FyearperSampletemp.get(FyearperSampletemp.size() - 1)); log.debug("the fire intervals for sample " + k + " is " + FIyearperSampletemp.get(0)); } // FIyearperSampletemp.add(temp); // log.debug("the fire intervals for sample " + k + " is " + FIyearperSampletemp.get(0)); } else { log.debug("in else "); FIyearperSampletemp.add((myReader.get(i).getLastYearIndexPerSample()[k] + myReader.get(i).getFirstYear()) - FyearperSampletemp.get(FyearperSampletemp.size() - 1)); log.debug("fire intervals for sample " + k + " is " + FIyearperSampletemp.get(0)); } // FIyearperSampletemp.add((myReader.get(i).getFirstYear() + myReader.get(i).getLastYearIndexPerSample()[k]) // - FyearperSampletemp.get(FyearperSampletemp.size() - 1)); } // log.debug("fire intervals for sample " + k + " is " + FIyearperSampletemp.get(0)); } // end of if one fire year and includelastyear so we have at least one interval in a given series. // endofnew if ((FyearperSampletemp.size() >= 2)) { log.debug("Series number is " + (k + 1)); for (int jk = 0; jk < FyearperSampletemp.size() - 1; jk++) { // FIyearperSampletemp.add(FyearperSample.get(k).get(jk+1) // - FyearperSample.get(k).get(jk)); log.debug("FyearperSampletemp is " + FyearperSampletemp.get(jk)); if ((FyearperSampletemp.get(jk + 1) - FyearperSampletemp.get(jk)) > 0) { FIyearperSampletemp .add(FyearperSampletemp.get(jk + 1) - FyearperSampletemp.get(jk)); } // FIyearperSampletemp.add(FyearperSampletemp.get(jk+1) // - FyearperSampletemp.get(jk)); log.debug("fire intervals for sample " + k + " is " + FIyearperSampletemp.get(jk)); // fisumtemp= fisumtemp + // FIyearperSampletemp.get(jk).intValue(); } if (includeIncomplete) { // if (maxLastYear != FyearperSampletemp.get(FyearperSampletemp.size() - 1)) if (maxLastYear.compareTo(FyearperSampletemp.get(FyearperSampletemp.size() - 1)) != 0) // if ((myReader.get(i).getLastYearIndexPerSample()[k] + myReader.get(i).getFirstYear()) != FyearperSampletemp // .get(FyearperSampletemp.size() - 1)) { if (maxLastYear <= (myReader.get(i).getLastYearIndexPerSample()[k] + myReader.get(i).getFirstYear())) { if (((maxLastYear) - FyearperSampletemp.get(FyearperSampletemp.size() - 1)) > 0) { FIyearperSampletemp.add(((maxLastYear) - FyearperSampletemp.get(FyearperSampletemp.size() - 1))); } } else { FIyearperSampletemp.add((myReader.get(i).getLastYearIndexPerSample()[k] + myReader.get(i).getFirstYear()) - FyearperSampletemp.get(FyearperSampletemp.size() - 1)); } // log.debug("the sample number is "+k+ // " the size of the fyearpersampletemp is "+ // FyearperSampletemp.size() ); // log.debug("the last year per sample is " // + (myReader.get(i).getLastYearIndexPerSample()[k] + myReader.get(i).getFirstYear())); // log.debug(" the last fire year per sample " + FyearperSampletemp.get(FyearperSampletemp.size() - 1)); // FIyearperSampletemp.add((maxLastYear) - FyearperSampletemp.get(FyearperSampletemp.size() - 1)); // log.debug("the last intrval in included is on is " // + FIyearperSampletemp.get(FIyearperSampletemp.size() - 1)); } } } // end of if at least 2 fier years so we have at least one interval in a given series. log.debug("size of FIyearperSampletemp " + FIyearperSampletemp.size() + " at series is :" + (k + 1)); // FIyearperSampletemp.size()+ // " X "+FIyearperSampletemp.get(0).size()); } // end of the loop for number of series. // log.debug("size of FIyearperSample "+ // FIyearperSampletemp.size()); for (int j = 0; j < FIyearperSampletemp.size(); j++) { fireintervalspersample.add(FIyearperSampletemp.get(j) * 1.0); } /* * Get the normal statistics for the fire intervals add the values to the stats and then call them for the stats */ if (fireintervalspersample.size() >= 3) { enoughIntSamp[i] = true; DescriptiveStatistics stasample = new DescriptiveStatistics(); Dfireintervalspersample = new Double[fireintervalspersample.size()]; Dfireintervalspersample = fireintervalspersample.toArray(Dfireintervalspersample); dfireintervalspersample = new double[fireintervalspersample.size()]; // summarySample = new // double[statsparam.length][myReader.size()]; for (int ik = 0; ik < fireintervalspersample.size(); ik++) { stasample.addValue(Dfireintervalspersample[ik].doubleValue()); dfireintervalspersample[ik] = Dfireintervalspersample[ik].doubleValue(); log.debug("the " + ik + " fire interval is " + dfireintervalspersample[ik]); } log.debug("the size for dfireintervalspersample is " + dfireintervalspersample.length); // ADDED BY PETE if (dfireintervalspersample.length == 0) continue; /* * load the Summary Analysis for the Sample fire intervals */ summarySample[0][i] = fireintervalspersample.size(); // double mean = stats.getMean(); summarySample[1][i] = stasample.getMean(); log.debug("mean sample is " + stasample.getMean()); // double median = // StatUtils.percentile(dfireintervalspersite, 50); summarySample[2][i] = StatUtils.percentile(dfireintervalspersample, 50); log.debug("summarySample[2][] " + i + " " + summarySample[2][i]); // double std = stats.getStandardDeviation(); summarySample[3][i] = stasample.getStandardDeviation(); log.debug("summarySample[3][] " + i + " " + summarySample[3][i]); // double skew = stats.getSkewness(); summarySample[4][i] = 1.0 / summarySample[1][i]; log.debug("summarySample[4][] " + i + " " + summarySample[4][i]); summarySample[5][i] = summarySample[3][i] / summarySample[1][i]; log.debug("summarySample[5][] " + i + " " + summarySample[5][i]); summarySample[6][i] = stasample.getSkewness(); log.debug("summarySample[6][] " + i + " " + summarySample[6][i]); // double kurt = stats.getKurtosis(); if (numberOfintervalssamp[i] == 3) { summarySample[7][i] = -99; } else { summarySample[7][i] = stasample.getKurtosis(); } // summarySample[7][i] = stasample.getKurtosis(); log.debug("summarySample[7][] " + i + " " + summarySample[7][i]); // log.debug("nomean \t\t nostd \t\t nokurt \t noskew \t\t nomedian"); // log.debug(twoPlace.format(mean)+"\t\t"+twoPlace.format(std)+"\t\t"+twoPlace.format(kurt)+"\t\t"+twoPlace.format(skew)+"\t\t"+twoPlace.format(median)); Weibull weibull = new Weibull(fireintervalspersample); // ArrayList<Double> weibullProb = weibull.getWeibullProbability(fireintervalspersample); ArrayList<Double> siglonglowbound = new ArrayList<Double>(); ArrayList<Double> sigshortupbound = new ArrayList<Double>(); log.debug("the weibull probability of first element is " + weibullProb.get(0)); log.debug("the index the size of the interval is " + weibullProb.indexOf(weibullProb.get(0))); for (int ij = 0; ij < weibullProb.size() - 1; ij++) { if (weibullProb.get(ij) <= alphaLevel) { siglonglowbound.add(fireintervalspersample.get(ij)); } if (weibullProb.get(ij) >= (1 - alphaLevel)) { sigshortupbound.add(fireintervalspersample.get(ij)); } } // summarySample[10][i] = weibull.getScale(); log.debug("summarySample[10][] " + i + " " + summarySample[10][i]); summarySample[11][i] = weibull.getShape(); log.debug("summarySample[11][] " + i + " " + summarySample[11][i]); summarySample[12][i] = weibull.getMean(); summarySample[13][i] = weibull.getMedian(); summarySample[14][i] = weibull.getMode(); summarySample[15][i] = weibull.getSigma(); summarySample[16][i] = 1.0 / summarySample[13][i]; summarySample[17][i] = weibull.getSkew(); summarySample[18][i] = weibull.getExceedenceProbability2()[0]; summarySample[19][i] = weibull.getExceedenceProbability2()[1]; Collections.sort(sigshortupbound); log.debug("siglonglowbound is " + siglonglowbound); try { summarySample[20][i] = sigshortupbound.get(sigshortupbound.size() - 1); } catch (Exception e) { summarySample[20][i] = Double.NaN; } Collections.sort(siglonglowbound); try { summarySample[21][i] = siglonglowbound.get(0); } catch (Exception e) { summarySample[21][i] = Double.NaN; } log.debug("sigshortupbound is " + sigshortupbound); Collections.sort(fireintervalspersample); try { summarySample[8][i] = fireintervalspersample.get(0); } catch (Exception ex) { log.error("Index out of bounds exception caught: "); log.error(" summarySample[8][i] = fireintervalspersample.get(0)"); ex.printStackTrace(); } summarySample[9][i] = fireintervalspersample.get(fireintervalspersample.size() - 1); // log.debug("shape \t\t scale \t\t median "); // log.debug(twoPlace.format(test2.Weibull_Parameters(fireintervalspersample)[0])+"\t\t"+twoPlace.format(test2.Weibull_Parameters(fireintervalspersample)[1])+"\t\t"+twoPlace.format(test2.weibull_median(test1.Weibull_Parameters(fireintervalspersample)))); // log.debug("mean \t\t sigma \t\t mode \t\t skewness"); // log.debug(twoPlace.format(test1.weibull_mean(test2.Weibull_Parameters(fireintervalspersample)))+"\t\t"+twoPlace.format(test1.weibull_sigma(test2.Weibull_Parameters(fireintervalspersample)))+"\t\t"+twoPlace.format(test2.weibull_mode(test1.Weibull_Parameters(fireintervalspersample)))+"\t\t"+twoPlace.format(test1.weibull_skew(test2.Weibull_Parameters(fireintervalspersample)))); // log.debug("maxhazard \t\t lei \t\t uei "); // log.debug(twoPlace.format(test2.maxhazard_int(test2.Weibull_Parameters(fireintervalspersample)))+"\t\t"+twoPlace.format(test2.weibull_lowuppexcint(test2.Weibull_Parameters(fireintervalspersample))[0])+"\t\t"+twoPlace.format(test2.weibull_lowuppexcint(test2.Weibull_Parameters(fireintervalspersample))[1])); // log.debug("the size of YearWith Fires is "+YearsWithFires.size()); // log.debug("the size of the prb exdc is // "+test2.weibull_Exprob(test2.Weibull_Parameters(fireintervalspersample)).length); System.out.println( "the size of the prb exdc sample is " + weibull.getExceedenceProbability().length); for (int kk = 0; kk < weibull.getExceedenceProbability().length; kk++) { ExceeProbsample[kk][i] = weibull.getExceedenceProbability()[kk]; log.debug("file " + i + " Exce probability " + ExceeProbsample[kk][i]); // log.debug("the size is "+ExceeProbsample.length); } } // end of if at least 4 fireintervals else { enoughIntSamp[i] = false; } } // end of if jRadioSample selected. // log.debug("the size of exceeprobsample is "ExceeProbsample.length+" X "+ExceeProbsample[0].length); } // end of i readering each file loop do loop (354-1185) /* * */ // log.debug("size of the climateMatrixSite is "+climateMatrixSite.size()+" X "+climateMatrixSite.get(0).size()); // for (int j = 0; j < listYears.size(); j++){ // log.debug(climateMatrixSite.get(0).get(j) + " " + // listYears.get(j)); // } // setCursor(Cursor.getDefaultCursor()); /* * create JFileChooser object to generate a browsing capabilities */ JFileChooser fileBrowse = new JFileChooser(); fileBrowse = new JFileChooser(savePath.substring(0, savePath.lastIndexOf(File.separator))); /* * set multiselect on (even though we don't need it) */ fileBrowse.setMultiSelectionEnabled(true); /* * set file and folder directive */ fileBrowse.setFileSelectionMode(JFileChooser.FILES_AND_DIRECTORIES); /* * set file type: coma delimited file csv */ // FileFilter filter1 = new CSVFileFilter(); fileBrowse.setFileFilter(filter1); /* * set dialog text: select the name and location of the matrix files */ fileBrowse.setDialogTitle("Select the name and location of the Stats Summary file:"); /* * create the writer object for each of the files to be created */ Writer wr; Writer wrWDE; Writer wrSample; Writer wrWDESample; /* * set delimiter in this case we are using comas "," */ String delim = ","; /* * Start writing information into the files */ try { if (analysisType.equals(AnalysisType.COMPOSITE)) { wr = new BufferedWriter(new FileWriter(summaryFile)); wrWDE = new BufferedWriter(new FileWriter(exceedenceFile)); /* * write the heading to the files */ String buffer = ""; buffer = buffer + "Composite Parameters" + delim; for (int i = 0; i < inputFileArray.length; i++) { buffer = buffer + inputFileArray[i].getLabel() + delim; } ; wr.write(buffer.substring(0, buffer.length() - 1) + System.getProperty("line.separator")); buffer = ""; for (int j = 0; j < statsparam.length; j++) { buffer = buffer + statsparam[j] + delim; for (int k = 0; k < inputFileArray.length; k++) { if (j == 0) { if (numberOfintervalscomp[k] < 3) { buffer = buffer + twoPlace.format(numberOfintervalscomp[k]) + delim; } else { buffer = buffer + twoPlace.format(summaryComp[0][k]) + delim; } } else { if (enoughIntComp[k]) { if (summaryComp[j][k] == -99) { buffer = buffer + "" + delim; } else { buffer = buffer + twoPlace.format(summaryComp[j][k]) + delim; } } else { buffer = buffer + "" + delim; } } } // end of k loop filearray wr.write(buffer.substring(0, buffer.length() - 1) + System.getProperty("line.separator")); buffer = ""; } // end of j loop Stats // wr.close(); // // // wrWDE = new BufferedWriter(new // FileWriter(outputWDExceeTable)); /* * write the heading to the files */ buffer = ""; wrWDE.write("Exceedence Prob" + delim); for (int i = 0; i < inputFileArray.length; i++) { buffer = buffer + inputFileArray[i].getLabel() + delim; } wrWDE.write(buffer.substring(0, buffer.length() - 1) + System.getProperty("line.separator")); buffer = ""; for (int j = 0; j < fixvalt.length; j++) { buffer = buffer + threePlace.format(fixvalt[j]) + delim; for (int k = 0; k < inputFileArray.length; k++) { if (enoughIntComp[k]) { buffer = buffer + twoPlace.format(ExceeProbcomp[j][k]) + delim; } else { buffer = buffer + "" + delim; } } wrWDE.write(buffer.substring(0, buffer.length() - 1) + System.getProperty("line.separator")); buffer = ""; } wr.close(); wrWDE.close(); } // end of if jRadioComp is selecte if (analysisType.equals(AnalysisType.SAMPLE)) { wrSample = new BufferedWriter(new FileWriter(summaryFile)); wrWDESample = new BufferedWriter(new FileWriter(exceedenceFile)); /* * write the heading to the files */ wrSample.write("Sample Parameters" + delim); for (int i = 0; i < inputFileArray.length; i++) { wrSample.write(inputFileArray[i].getLabel() + delim); } wrSample.write(System.getProperty("line.separator")); for (int j = 0; j < statsparam.length; j++) { wrSample.write(statsparam[j] + delim); for (int k = 0; k < inputFileArray.length; k++) { if (j == 0) { if (numberOfintervalssamp[k] < 3) { wrSample.write(twoPlace.format(numberOfintervalssamp[k]) + delim); } else { wrSample.write(twoPlace.format(summarySample[0][k]) + delim); } } else { if (enoughIntSamp[k]) { if (summarySample[j][k] == -99) { wrSample.write("" + delim); } else { wrSample.write(twoPlace.format(summarySample[j][k]) + delim); } } else { wrSample.write("" + delim); } } } // end of k loop file array wrSample.write(System.getProperty("line.separator")); } // end of loop j loop stats // wrSample.close(); // // // log.debug("the size is "+fixvalt.length+" X "+inputFile.length); // wrWDESample = new BufferedWriter(new // FileWriter(outputWDExceeTablesample)); /* * write the heading to the files */ wrWDESample.write("Exceedence Prob" + delim); for (int i = 0; i < inputFileArray.length; i++) { wrWDESample.write(inputFileArray[i].getLabel() + delim); } wrWDESample.write(System.getProperty("line.separator")); for (int j = 0; j < fixvalt.length; j++) { wrWDESample.write(threePlace.format(fixvalt[j]) + delim); for (int k = 0; k < inputFileArray.length; k++) { // System.out.print(ExceeProbcomp[j][k]+delim); if (enoughIntSamp[k]) { wrWDESample.write(twoPlace.format(ExceeProbsample[j][k]) + delim); } else { wrWDESample.write("" + delim); } } // System.out.print(System.getProperty("line.separator")); wrWDESample.write(System.getProperty("line.separator")); } wrSample.close(); wrWDESample.close(); } // end of jradiosample } // end of Try catch (IOException ex) { ex.printStackTrace(); } finally { } }