List of usage examples for org.apache.commons.math3.stat.descriptive SummaryStatistics SummaryStatistics
public SummaryStatistics()
From source file:ca.mcgill.cs.creco.logic.ScoredAttribute.java
private void setNumericStats(ArrayList<TypedValue> pValues) { SummaryStatistics ss = new SummaryStatistics(); double min = DEFAULT_MIN; double max = DEFAULT_MAX; ArrayList<Double> values = new ArrayList<Double>(pValues.size()); for (TypedValue tv : pValues) { if (tv.isNumeric()) { ss.addValue(tv.getNumeric()); if (tv.getNumeric() < min) { min = tv.getNumeric();/*from ww w . ja v a 2s . c om*/ } if (tv.getNumeric() > max) { max = tv.getNumeric(); } values.add(tv.getNumeric()); } } //Set bounds if (Double.isNaN(min)) { LOG.error("Min value is NaN: " + aAttributeID + ", " + aAttributeName + ", " + aCategory.getId()); } if (Double.isNaN(max)) { LOG.error("Max value is NaN: " + aAttributeID + ", " + aAttributeName + ", " + aCategory.getId()); } aMin = new TypedValue(min); aMax = new TypedValue(max); double mean = ss.getGeometricMean(); double variance = ss.getStandardDeviation() * ss.getStandardDeviation(); //Calculate Entropy double entropy = 0; for (TypedValue tv : pValues) { if (tv.isNumeric()) { double prob = computeNormalProbability(tv, mean, variance); entropy = entropy - prob * (Math.log(prob)); } } aDefaultValue = new TypedValue(mean); if (!Double.isNaN(entropy)) { aEntropy = entropy; } else { aEntropy = 0; } //Get the correlation NumericCorrelator ac = new NumericCorrelator(aCategory); aCorrelation = ac.computeCorrelation(aAttributeID, CONSIDERATION_THRESHOLD); if (aIsPrice) { aDirection = Direction.LESS_IS_BETTER; } else { aDirection = ac.computeAttributeDirection(aAttributeID, CONSIDERATION_THRESHOLD); } //Calculate Ranking if (aDirection == Direction.LESS_IS_BETTER) { Collections.sort(values); } else { Collections.sort(values, Collections.reverseOrder()); } setRank(values); }
From source file:net.recommenders.rival.evaluation.statistics.ConfidenceInterval.java
/** * Method that takes only one metric as parameter. It is useful when * comparing more than two metrics (so that a confidence interval is * computed for each of them), as suggested in [Sakai, 2014] * * @param alpha probability of incorrectly rejecting the null hypothesis (1 * - confidence_level)//from w ww .j a v a 2 s . c o m * @param metricValuesPerDimension one value of the metric for each * dimension * @return array with the confidence interval: [mean - margin of error, mean * + margin of error] */ public double[] getConfidenceInterval(final double alpha, final Map<?, Double> metricValuesPerDimension) { SummaryStatistics differences = new SummaryStatistics(); for (Double d : metricValuesPerDimension.values()) { differences.addValue(d); } return getConfidenceInterval(alpha, (int) differences.getN() - 1, (int) differences.getN(), differences.getStandardDeviation(), differences.getMean()); }
From source file:model.scenario.CompetitiveScenarioTest.java
@Test public void rightPriceAndQuantityLearningInventory() { for (int competitors = 4; competitors <= 7; competitors++) { System.out.println("FORCED COMPETITIVE FIRMS: " + (competitors + 1)); for (int i = 0; i < 5; i++) { final MacroII macroII = new MacroII(System.currentTimeMillis()); final TripolistScenario scenario1 = new TripolistScenario(macroII); scenario1.setSalesDepartmentType(SalesDepartmentOneAtATime.class); scenario1.setAskPricingStrategy(SalesControlWithFixedInventoryAndPID.class); scenario1.setControlType(// ww w. j av a 2 s . c o m MonopolistScenario.MonopolistScenarioIntegratedControlEnum.MARGINAL_PLANT_CONTROL); scenario1.setAdditionalCompetitors(competitors); scenario1.setWorkersToBeRehiredEveryDay(true); scenario1.setDemandIntercept(102); //assign scenario macroII.setScenario(scenario1); macroII.start(); while (macroII.schedule.getTime() < 8000) { macroII.schedule.step(macroII); /* System.out.println("sales: " + scenario1.getCompetitors().get(0).getSalesDepartment(GoodType.GENERIC). getLatestObservation(SalesDataType.OUTFLOW) +"," + scenario1.getCompetitors().get(1).getSalesDepartment(GoodType.GENERIC). getLatestObservation(SalesDataType.OUTFLOW)); */ } SummaryStatistics prices = new SummaryStatistics(); SummaryStatistics quantities = new SummaryStatistics(); SummaryStatistics target = new SummaryStatistics(); for (int j = 0; j < 500; j++) { macroII.schedule.step(macroII); assert !Float.isNaN(macroII.getMarket(UndifferentiatedGoodType.GENERIC).getTodayAveragePrice()); prices.addValue(macroII.getMarket(UndifferentiatedGoodType.GENERIC).getTodayAveragePrice()); quantities.addValue(macroII.getMarket(UndifferentiatedGoodType.GENERIC).getTodayVolume()); for (EconomicAgent agent : macroII.getMarket(UndifferentiatedGoodType.GENERIC).getSellers()) { SalesDepartment department = ((Firm) agent) .getSalesDepartment(UndifferentiatedGoodType.GENERIC); target.addValue(macroII.getMarket(UndifferentiatedGoodType.GENERIC).getTodayVolume()); } } System.out.println(prices.getMean() + " - " + quantities.getMean() + "/" + target.getMean() + "----" + macroII.seed() + " | " + macroII.getMarket(UndifferentiatedGoodType.GENERIC).getLastDaysAveragePrice()); System.out.println("standard deviations: price : " + prices.getStandardDeviation() + " , quantity: " + quantities.getStandardDeviation()); OneSectorStatics.printSlopes(scenario1); if (competitors >= 4) { assertEquals(prices.getMean(), 58, 5); // assertTrue(prices.getStandardDeviation() < 5.5); assertEquals(quantities.getMean(), 44, 5); // assertTrue(quantities.getStandardDeviation() < 5.5); } macroII.finish(); } } }
From source file:RealFunctionValidation.java
public static SummaryStatistics assessAccuracy(final Method method, final DataInputStream in, final DataOutputStream out) throws IOException, IllegalAccessException, IllegalArgumentException, InvocationTargetException { if (method.getReturnType() != Double.TYPE) { throw new IllegalArgumentException("method must return a double"); }/*from w w w . j a v a2 s . c o m*/ final Class<?>[] types = method.getParameterTypes(); for (int i = 0; i < types.length; i++) { if (!types[i].isPrimitive()) { final StringBuilder builder = new StringBuilder(); builder.append("argument #").append(i + 1).append(" of method ").append(method.getName()) .append("must be of primitive of type"); throw new IllegalArgumentException(builder.toString()); } } final SummaryStatistics stat = new SummaryStatistics(); final Object[] parameters = new Object[types.length]; while (true) { try { for (int i = 0; i < parameters.length; i++) { parameters[i] = readAndWritePrimitiveValue(in, out, types[i]); } final double expected = in.readDouble(); if (FastMath.abs(expected) > 1E-16) { final Object value = method.invoke(null, parameters); final double actual = ((Double) value).doubleValue(); final double err = FastMath.abs(actual - expected); final double ulps = err / FastMath.ulp(expected); out.writeDouble(expected); out.writeDouble(actual); out.writeDouble(ulps); stat.addValue(ulps); } } catch (EOFException e) { break; } } return stat; }
From source file:ijfx.ui.plugin.panel.OverlayPanel.java
protected XYChart.Series<Double, Double> getOverlayHistogram(Overlay overlay) { Timer timer = timerService.getTimer(this.getClass()); timer.start();/* w w w .jav a2 s.com*/ Double[] valueList = statsService.getValueList(currentDisplay(), overlay); timer.elapsed("Getting the stats"); SummaryStatistics sumup = new SummaryStatistics(); for (Double v : valueList) { sumup.addValue(v); } timer.elapsed("Building the sumup"); double min = sumup.getMin(); double max = sumup.getMax(); double range = max - min; int bins = 100;//new Double(max - min).intValue(); EmpiricalDistribution distribution = new EmpiricalDistribution(bins); double[] values = ArrayUtils.toPrimitive(valueList); Arrays.parallelSort(values); distribution.load(values); timer.elapsed("Sort and distrubution repartition up"); XYChart.Series<Double, Double> serie = new XYChart.Series<>(); ArrayList<Data<Double, Double>> data = new ArrayList<>(bins); double k = min; for (SummaryStatistics st : distribution.getBinStats()) { data.add(new Data<Double, Double>(k, new Double(st.getN()))); k += range / bins; } serie.getData().clear(); serie.getData().addAll(data); timer.elapsed("Creating charts"); return serie; }
From source file:model.experiments.tuningRuns.CompetitiveAveragingGridSearch.java
public static CompetitiveAveragingResult weightedRun(int hrDays, int salesDays, boolean decoratedHr, boolean decoratedSales) { SummaryStatistics averageResultingPrice = new SummaryStatistics(); SummaryStatistics averageResultingQuantity = new SummaryStatistics(); SummaryStatistics averageStandardDeviation = new SummaryStatistics(); for (int i = 0; i < 5; i++) { final MacroII macroII = new MacroII(System.currentTimeMillis()); final TripolistScenario scenario1 = new TripolistScenario(macroII); scenario1.setSalesDepartmentType(SalesDepartmentOneAtATime.class); scenario1.setAskPricingStrategy(SalesControlWithFixedInventoryAndPID.class); scenario1.setControlType(//from www . j a v a2s .c om MonopolistScenario.MonopolistScenarioIntegratedControlEnum.MARGINAL_PLANT_CONTROL); scenario1.setAdditionalCompetitors(4); scenario1.setWorkersToBeRehiredEveryDay(true); scenario1.setDemandIntercept(102); scenario1.setSalesPricePreditorStrategy(FixedDecreaseSalesPredictor.class); //assign scenario macroII.setScenario(scenario1); macroII.start(); macroII.schedule.step(macroII); for (Firm firm : scenario1.getCompetitors()) { for (HumanResources hr : firm.getHRs()) { hr.setPredictor(new FixedIncreasePurchasesPredictor(0)); PriceAverager priceAverager = new WeightedPriceAverager(hrDays); if (decoratedHr) priceAverager = new NoTradingOverrideAveragerDecorator(priceAverager); hr.setPriceAverager(priceAverager); } PriceAverager priceAverager = new WeightedPriceAverager(salesDays); if (decoratedSales) priceAverager = new NoTradingOverrideAveragerDecorator(priceAverager); firm.getSalesDepartment(UndifferentiatedGoodType.GENERIC).setPriceAverager(priceAverager); firm.getSalesDepartment(UndifferentiatedGoodType.GENERIC) .setPredictorStrategy(new FixedDecreaseSalesPredictor(0)); } while (macroII.schedule.getTime() < 10000) { macroII.schedule.step(macroII); } SummaryStatistics prices = new SummaryStatistics(); SummaryStatistics quantities = new SummaryStatistics(); for (int j = 0; j < 500; j++) { macroII.schedule.step(macroII); prices.addValue(macroII.getMarket(UndifferentiatedGoodType.GENERIC).getTodayAveragePrice()); quantities.addValue(macroII.getMarket(UndifferentiatedGoodType.GENERIC).getTodayVolume()); } //okay? averageResultingPrice.addValue(prices.getMean()); averageResultingQuantity.addValue(quantities.getMean()); averageStandardDeviation.addValue(prices.getStandardDeviation()); } //okay? return new CompetitiveAveragingResult(averageResultingPrice.getMean(), averageResultingQuantity.getMean(), averageStandardDeviation.getMean()); }
From source file:io.tilt.minka.business.leader.Shepherd.java
private boolean checkHealth(long now, long normalDelay, List<Heartbeat> onTime) { SummaryStatistics stat = new SummaryStatistics(); // there's some hope long lastCreation = onTime.get(0).getCreation().getMillis(); long biggestDistance = 0; for (Heartbeat hb : onTime) { long creation = hb.getCreation().getMillis(); long arriveDelay = now - creation; long distance = creation - lastCreation; stat.addValue(distance);/*from w w w . j a v a2 s . co m*/ lastCreation = creation; biggestDistance = distance > biggestDistance ? distance : biggestDistance; if (logger.isDebugEnabled()) { logger.debug("{}: HB SeqID: {}, Arrive Delay: {}ms, Distance: {}ms, Creation: {}", getClass().getSimpleName(), hb.getSequenceId(), arriveDelay, distance, hb.getCreation()); } } long stdDeviationDelay = (long) Precision.round(stat.getStandardDeviation(), 2); long permittedStdDeviationDistance = (normalDelay * (long) (config.getShepherdHeartbeatMaxDistanceStandardDeviation() * 10d) / 100); /* long permittedBiggestDelay = (normalDelay * (long)(config.getShepherdHeartbeatMaxBiggestDistanceFactor()*10d) / 100); */ final NetworkShardID shardId = onTime.get(0).getShardId(); boolean healthly = stdDeviationDelay < permittedStdDeviationDistance;// && biggestDelay < permittedBiggestDelay; if (logger.isInfoEnabled()) { logger.debug("{}: Shard: {}, {} Standard deviation distance: {}/{}", getClass().getSimpleName(), shardId, healthly ? HEALTH_UP : HEALTH_DOWN, stdDeviationDelay, permittedStdDeviationDistance); } return healthly; }
From source file:com.itemanalysis.psychometrics.irt.estimation.MarginalMaximumLikelihoodEstimation.java
/** * *//*from w w w. java2 s. c o m*/ public void computeG2ItemFit(int nbins, int minExpectedCount) { IrtExaminee irtExaminee = new IrtExaminee("", irm); double[] eapEstimate = new double[responseVector.length]; SummaryStatistics stats = new SummaryStatistics(); //First loop over response vectors for (int i = 0; i < responseVector.length; i++) { //EAP estimate of ability irtExaminee.setResponseVector(responseVector[i]); eapEstimate[i] = irtExaminee.eapEstimate(latentDistribution); for (int N = 0; N < responseVector[i].getFrequency(); N++) {//Expand response vectors stats.addValue(eapEstimate[i]); } } DefaultLinearTransformation linearTransformation = new DefaultLinearTransformation(stats.getMean(), latentDistribution.getMean(), stats.getStandardDeviation(), latentDistribution.getStandardDeviation()); //Create fit statistic objects itemFit = new ItemFitG2[nItems]; double lower = linearTransformation.transform(stats.getMin()) - .01;//Subtract a small number to ensure lowest theta is counted double upper = linearTransformation.transform(stats.getMax()) + .01;//Add a small number to ensure largest theta is counted Cut thetaCut = new Cut(lower, upper, nbins); for (int j = 0; j < nItems; j++) { itemFit[j] = new ItemFitG2(irm[j], thetaCut, minExpectedCount); } //Second loop over response vectors //Increment fit statistics double A = linearTransformation.getScale(); for (int i = 0; i < responseVector.length; i++) { //Estimate EAP standard deviation should be the same as the standard deviation //of the latent distribution used to estimate the item parameters. eapEstimate[i] = eapEstimate[i] * A; for (int N = 0; N < responseVector[i].getFrequency(); N++) {//Expand table for (int j = 0; j < nItems; j++) { ((ItemFitG2) itemFit[j]).increment(eapEstimate[i], responseVector[i].getResponseAt(j)); } } } // System.out.println(thetaCut.toString()); //Compute Item Fit for (int j = 0; j < nItems; j++) { itemFit[j].compute(); } }
From source file:model.scenario.OneLinkSupplyChainResult.java
public static OneLinkSupplyChainResult beefMonopolistFixedProductionsOneRun(long seed, float divideMonopolistGainsByThis, int monopolistSpeed, final boolean foodLearned, File csvFileToWrite) {/*from www . j ava 2 s .c o m*/ final MacroII macroII = new MacroII(seed); final OneLinkSupplyChainScenarioCheatingBuyPriceAndForcedMonopolist scenario1 = new OneLinkSupplyChainScenarioCheatingBuyPriceAndForcedMonopolist( macroII, OneLinkSupplyChainScenario.INPUT_GOOD) { @Override public void buildFoodPurchasesPredictor(PurchasesDepartment department) { if (foodLearned) department.setPredictor(new FixedIncreasePurchasesPredictor(0)); } @Override protected SalesDepartment createSalesDepartment(Firm firm, Market goodmarket) { SalesDepartment department = super.createSalesDepartment(firm, goodmarket); //To change body of overridden methods use File | Settings | File Templates. if (foodLearned && goodmarket.getGoodType().equals(OneLinkSupplyChainScenario.OUTPUT_GOOD)) department.setPredictorStrategy(new FixedDecreaseSalesPredictor(0)); return department; } @Override protected HumanResources createPlant(Blueprint blueprint, Firm firm, Market laborMarket) { HumanResources hr = super.createPlant(blueprint, firm, laborMarket); //To change body of overridden methods use File | Settings | File Templates. if (foodLearned && !blueprint.getOutputs().containsKey(OneLinkSupplyChainScenario.INPUT_GOOD)) hr.setPredictor(new FixedIncreasePurchasesPredictor(0)); return hr; } }; scenario1.setControlType(MarginalMaximizer.class); scenario1.setSalesDepartmentType(SalesDepartmentOneAtATime.class); //use standard PID parameters scenario1.setDivideProportionalGainByThis(divideMonopolistGainsByThis); scenario1.setDivideIntegrativeGainByThis(divideMonopolistGainsByThis); //100 days delay scenario1.setBeefPricingSpeed(monopolistSpeed); //no need for filter with the cheating price scenario1.setBeefPriceFilterer(null); scenario1.setBeefTargetInventory(100); scenario1.setFoodTargetInventory(100); //add csv writer if needed if (csvFileToWrite != null) DailyStatCollector.addDailyStatCollectorToModel(csvFileToWrite, macroII); macroII.setScenario(scenario1); macroII.start(); SummaryStatistics averageFoodPrice = new SummaryStatistics(); SummaryStatistics averageBeefPrice = new SummaryStatistics(); SummaryStatistics averageBeefTraded = new SummaryStatistics(); while (macroII.schedule.getTime() < 15000) { macroII.schedule.step(macroII); if (macroII.schedule.getTime() >= 14500) { averageFoodPrice.addValue(macroII.getMarket(OneLinkSupplyChainScenario.OUTPUT_GOOD) .getLatestObservation(MarketDataType.AVERAGE_CLOSING_PRICE)); averageBeefPrice.addValue(macroII.getMarket(OneLinkSupplyChainScenario.INPUT_GOOD) .getLatestObservation(MarketDataType.AVERAGE_CLOSING_PRICE)); averageBeefTraded .addValue(macroII.getMarket(OneLinkSupplyChainScenario.INPUT_GOOD).getYesterdayVolume()); } } macroII.finish(); System.out.println("done with price: " + averageBeefPrice.getMean() + ", and standard deviation : " + averageBeefPrice.getStandardDeviation()); System.out.println("seed: " + macroII.seed()); System.out.println(); //the beef price is in the ballpark return new OneLinkSupplyChainResult(averageBeefPrice.getMean(), averageFoodPrice.getMean(), averageBeefTraded.getMean(), macroII); }
From source file:com.civprod.writerstoolbox.OpenNLP.training.ThoughtAndSpeechTrainer.java
private void cmdTrainActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_cmdTrainActionPerformed final ThoughtAndSpeechTrainer tempThis = this; new Thread(() -> { textTestResults.setText(""); Charset charset = Charset.forName("UTF-8"); //create TokenizerFactory part of the training context ThoughtAndSpeechParserFactory myTokenizerFactory = new ThoughtAndSpeechParserFactory("EN", this.saidWordsDictionary, this.thoughtWordsDictionary); /*ThoughtAndSpeechParser stdTokenizer = null; try {//from w w w .j a va2 s . com stdTokenizer = OpenNLPUtils.createTokenizer(); } catch (IOException ex) { Logger.getLogger(TokenizerTrainer.class.getName()).log(Level.SEVERE, null, ex); }*/ List<FileSplit> FileSplits = FileSplit.generateFileSplitsLOO(mFileCollectionListModel); File trainingFile = new File("en-ThoughtAndSpeech.train"); File testFile = new File("en-ThoughtAndSpeech.test"); SummaryStatistics curFStats = new SummaryStatistics(); SummaryStatistics curRecallStats = new SummaryStatistics(); SummaryStatistics curPrecisionStats = new SummaryStatistics(); SummaryStatistics stdFStats = new SummaryStatistics(); SummaryStatistics stdRecallStats = new SummaryStatistics(); SummaryStatistics stdPrecisionStats = new SummaryStatistics(); java.io.BufferedOutputStream trainingFileWriter = null; for (FileSplit curFileSplit : FileSplits) { try { //create training file trainingFileWriter = new java.io.BufferedOutputStream( new java.io.FileOutputStream(trainingFile)); for (File curTrainingFile : curFileSplit.getTrainingFiles()) { java.io.BufferedInputStream curTrainingFileReader = null; try { curTrainingFileReader = new java.io.BufferedInputStream( new java.io.FileInputStream(curTrainingFile)); while (curTrainingFileReader.available() > 0) { trainingFileWriter.write(curTrainingFileReader.read()); } } catch (IOException ex) { Logger.getLogger(SentenceDetectorTrainer.class.getName()).log(Level.SEVERE, null, ex); } finally { if (curTrainingFileReader != null) { curTrainingFileReader.close(); } } } trainingFileWriter.write('\n'); } catch (IOException ex) { Logger.getLogger(SentenceDetectorTrainer.class.getName()).log(Level.SEVERE, null, ex); } finally { if (trainingFileWriter != null) { try { trainingFileWriter.close(); } catch (IOException ex) { Logger.getLogger(SentenceDetectorTrainer.class.getName()).log(Level.SEVERE, null, ex); } } } //create test file java.io.BufferedOutputStream testFileWriter = null; try { //create training file testFileWriter = new java.io.BufferedOutputStream(new java.io.FileOutputStream(testFile)); for (File curTrainingFile : curFileSplit.getTestFiles()) { String testingFileName = curTrainingFile.getCanonicalPath(); textTestResults .setText(textTestResults.getText() + "testing with " + testingFileName + "\n"); java.io.BufferedInputStream curTrainingFileReader = null; try { curTrainingFileReader = new java.io.BufferedInputStream( new java.io.FileInputStream(curTrainingFile)); while (curTrainingFileReader.available() > 0) { int read = curTrainingFileReader.read(); testFileWriter.write(read); } } catch (IOException ex) { Logger.getLogger(SentenceDetectorTrainer.class.getName()).log(Level.SEVERE, null, ex); } finally { if (curTrainingFileReader != null) { curTrainingFileReader.close(); } } } testFileWriter.write('\n'); } catch (IOException ex) { Logger.getLogger(SentenceDetectorTrainer.class.getName()).log(Level.SEVERE, null, ex); } finally { if (testFileWriter != null) { try { testFileWriter.close(); } catch (IOException ex) { Logger.getLogger(SentenceDetectorTrainer.class.getName()).log(Level.SEVERE, null, ex); } } } //create and train model ObjectStream<String> trainingLineStream = null; ThoughtAndSpeechModel train = null; try { trainingLineStream = new PlainTextByLineStream(new MarkableFileInputStreamFactory(trainingFile), charset); ObjectStream<ThoughtAndSpeechSample> sampleStream = null; try { sampleStream = new ThoughtAndSpeechSampleStream(trainingLineStream); train = ThoughtAndSpeechParserME.train("en", sampleStream, myTokenizerFactory, TrainingParameters.defaultParams()); } catch (IOException ex) { Logger.getLogger(SentenceDetectorTrainer.class.getName()).log(Level.SEVERE, null, ex); } finally { if (sampleStream != null) { try { sampleStream.close(); } catch (IOException ex) { Logger.getLogger(SentenceDetectorTrainer.class.getName()).log(Level.SEVERE, null, ex); } } } } catch (IOException ex) { Logger.getLogger(SentenceDetectorTrainer.class.getName()).log(Level.SEVERE, null, ex); } finally { if (trainingLineStream != null) { try { trainingLineStream.close(); } catch (IOException ex) { Logger.getLogger(SentenceDetectorTrainer.class.getName()).log(Level.SEVERE, null, ex); } } } if (train != null) { ObjectStream<String> testingLineStream = null; try { testingLineStream = new PlainTextByLineStream(new MarkableFileInputStreamFactory(testFile), charset); ObjectStream<ThoughtAndSpeechSample> sampleStream = null; try { sampleStream = new ThoughtAndSpeechSampleStream(testingLineStream); ThoughtAndSpeechParserME testDetector = new ThoughtAndSpeechParserME(train); ThoughtAndSpeechEvaluator evaluator = new ThoughtAndSpeechEvaluator(testDetector); evaluator.evaluate(sampleStream); FMeasure testFMeasure = evaluator.getFMeasure(); curFStats.addValue(testFMeasure.getFMeasure()); curRecallStats.addValue(testFMeasure.getRecallScore()); curPrecisionStats.addValue(testFMeasure.getPrecisionScore()); textTestResults.setText(textTestResults.getText() + testFMeasure.getFMeasure() + " " + testFMeasure.getPrecisionScore() + " " + testFMeasure.getRecallScore() + "\n"); /*if (stdTokenizer != null) { testingLineStream = new PlainTextByLineStream(new FileInputStream(testFile), charset); sampleStream = new TokenSampleStream(testingLineStream); TokenizerEvaluator stdEvaluator = new TokenizerEvaluator(stdTokenizer); stdEvaluator.evaluate(sampleStream); FMeasure stdFMeasure = stdEvaluator.getFMeasure(); stdFStats.addValue(stdFMeasure.getFMeasure()); stdRecallStats.addValue(stdFMeasure.getRecallScore()); stdPrecisionStats.addValue(stdFMeasure.getPrecisionScore()); textTestResults.setText(textTestResults.getText() + " " + stdFMeasure.getFMeasure() + " " + stdFMeasure.getPrecisionScore() + " " + stdFMeasure.getRecallScore() + "\n"); }*/ textTestResults.setText(textTestResults.getText() + "\n"); } catch (IOException ex) { Logger.getLogger(SentenceDetectorTrainer.class.getName()).log(Level.SEVERE, null, ex); } finally { if (sampleStream != null) { try { sampleStream.close(); } catch (IOException ex) { Logger.getLogger(SentenceDetectorTrainer.class.getName()).log(Level.SEVERE, null, ex); } } } } catch (IOException ex) { Logger.getLogger(SentenceDetectorTrainer.class.getName()).log(Level.SEVERE, null, ex); } finally { if (testingLineStream != null) { try { testingLineStream.close(); } catch (IOException ex) { Logger.getLogger(SentenceDetectorTrainer.class.getName()).log(Level.SEVERE, null, ex); } } } } } textTestResults.setText(textTestResults.getText() + "\n"); textTestResults.setText(textTestResults.getText() + "test model\n"); textTestResults.setText(textTestResults.getText() + "f score mean " + curFStats.getMean() + " stdDev " + curFStats.getStandardDeviation() + "\n"); textTestResults.setText(textTestResults.getText() + "recall mean " + curRecallStats.getMean() + " stdDev " + curRecallStats.getStandardDeviation() + "\n"); textTestResults.setText(textTestResults.getText() + "precision score mean " + curPrecisionStats.getMean() + " stdDev " + curPrecisionStats.getStandardDeviation() + "\n"); textTestResults.setText(textTestResults.getText() + "std model\n"); textTestResults.setText(textTestResults.getText() + "f score mean " + stdFStats.getMean() + " stdDev " + stdFStats.getStandardDeviation() + "\n"); textTestResults.setText(textTestResults.getText() + "recall mean " + stdRecallStats.getMean() + " stdDev " + stdRecallStats.getStandardDeviation() + "\n"); textTestResults.setText(textTestResults.getText() + "precision score mean " + stdPrecisionStats.getMean() + " stdDev " + stdPrecisionStats.getStandardDeviation() + "\n"); //create combinded training file trainingFileWriter = null; try { trainingFileWriter = new java.io.BufferedOutputStream(new java.io.FileOutputStream(trainingFile)); for (File curTrainingFile : mFileCollectionListModel) { java.io.BufferedInputStream curTrainingFileReader = null; try { curTrainingFileReader = new java.io.BufferedInputStream( new java.io.FileInputStream(curTrainingFile)); while (curTrainingFileReader.available() > 0) { trainingFileWriter.write(curTrainingFileReader.read()); } } catch (IOException ex) { Logger.getLogger(SentenceDetectorTrainer.class.getName()).log(Level.SEVERE, null, ex); } finally { if (curTrainingFileReader != null) { curTrainingFileReader.close(); } } } trainingFileWriter.write('\n'); } catch (IOException ex) { Logger.getLogger(SentenceDetectorTrainer.class.getName()).log(Level.SEVERE, null, ex); } finally { if (trainingFileWriter != null) { try { trainingFileWriter.close(); } catch (IOException ex) { Logger.getLogger(SentenceDetectorTrainer.class.getName()).log(Level.SEVERE, null, ex); } } } //create and train model ObjectStream<String> lineStream = null; this.createdObject = null; try { lineStream = new PlainTextByLineStream(new MarkableFileInputStreamFactory(trainingFile), charset); ObjectStream<ThoughtAndSpeechSample> sampleStream = null; try { sampleStream = new ThoughtAndSpeechSampleStream(lineStream); this.createdObject = ThoughtAndSpeechParserME.train("en", sampleStream, myTokenizerFactory, TrainingParameters.defaultParams()); } catch (IOException ex) { Logger.getLogger(SentenceDetectorTrainer.class.getName()).log(Level.SEVERE, null, ex); } finally { if (sampleStream != null) { try { sampleStream.close(); } catch (IOException ex) { Logger.getLogger(SentenceDetectorTrainer.class.getName()).log(Level.SEVERE, null, ex); } } } } catch (IOException ex) { Logger.getLogger(SentenceDetectorTrainer.class.getName()).log(Level.SEVERE, null, ex); } finally { if (lineStream != null) { try { lineStream.close(); } catch (IOException ex) { Logger.getLogger(SentenceDetectorTrainer.class.getName()).log(Level.SEVERE, null, ex); } } } if (createdObject != null) { OutputStream modelOut = null; File modelFile = new File("en-ThoughtAndSpeech-token.bin"); try { modelOut = new BufferedOutputStream(new FileOutputStream(modelFile)); createdObject.serialize(modelOut); } catch (IOException ex) { Logger.getLogger(SentenceDetectorTrainer.class.getName()).log(Level.SEVERE, null, ex); } finally { if (modelOut != null) { try { modelOut.close(); } catch (IOException ex) { Logger.getLogger(SentenceDetectorTrainer.class.getName()).log(Level.SEVERE, null, ex); } } } } textTestResults.setText(textTestResults.getText() + "done"); }).start(); }