List of usage examples for org.apache.commons.io FileUtils readLines
public static List readLines(File file) throws IOException
From source file:hu.bme.mit.sette.common.tasks.RunResultParser.java
private SnippetInputsXml parseSnippet(final Snippet snippet) throws Exception { // TODO validation? SnippetInputsXml inputsXml = new SnippetInputsXml(); inputsXml.setToolName(getTool().getName()); inputsXml.setSnippetProjectElement(/*from w ww . j av a 2 s .c om*/ new SnippetProjectElement(getSnippetProjectSettings().getBaseDirectory().getCanonicalPath())); inputsXml.setSnippetElement( new SnippetElement(snippet.getContainer().getJavaClass().getName(), snippet.getMethod().getName())); // TODO needs more documentation File infoFile = RunnerProjectUtils.getSnippetInfoFile(getRunnerProjectSettings(), snippet); if (!infoFile.exists()) { inputsXml.setResultType(ResultType.NA); } else { List<String> lines = FileUtils.readLines(infoFile); if (lines.get(2).startsWith("Destroyed")) { if (lines.get(2).startsWith("Destroyed: yes")) { inputsXml.setResultType(ResultType.TM); } } else { // TODO error handling System.err.println("FORMAT PROBLEM"); } } if (inputsXml.getResultType() == null) { parseSnippet(snippet, inputsXml); } return inputsXml; }
From source file:edu.cmu.cs.lti.discoursedb.io.prosolo.blog.converter.BlogConverter.java
@Override public void run(String... args) throws Exception { if (args.length < 3) { logger.error(//from w w w. ja v a 2 s. c o m "USAGE: BlogConverterApplication <DiscourseName> <DataSetName> <blogDump> <userMapping (optional)> <dumpIsWrappedInJsonArray (optional, default=false)>"); throw new RuntimeException("Incorrect number of launch parameters."); } final String discourseName = args[0]; final String dataSetName = args[1]; if (dataSourceService.dataSourceExists(dataSetName)) { logger.warn("Dataset " + dataSetName + " has already been imported into DiscourseDB. Terminating..."); return; } final String forumDumpFileName = args[2]; File blogDumpFile = new File(forumDumpFileName); if (!blogDumpFile.exists() || !blogDumpFile.isFile() || !blogDumpFile.canRead()) { logger.error("Forum dump file does not exist or is not readable."); throw new RuntimeException("Can't read file " + forumDumpFileName); } //parse the optional fourth and fifth parameter String userMappingFileName = null; String jsonarray = null; if (args.length == 4) { if (args[3].equalsIgnoreCase("true") || args[3].equalsIgnoreCase("false")) { jsonarray = args[3]; } else { userMappingFileName = args[3]; } } else { if (args[3].equalsIgnoreCase("true") || args[3].equalsIgnoreCase("false")) { jsonarray = args[3]; userMappingFileName = args[4]; } else { jsonarray = args[4]; userMappingFileName = args[3]; } } //read the blog author to edX user mapping, if available if (userMappingFileName != null) { logger.trace("Reading user mapping from " + userMappingFileName); File userMappingFile = new File(userMappingFileName); if (!userMappingFile.exists() || !userMappingFile.isFile() || !userMappingFile.canRead()) { logger.error("User mappiong file does not exist or is not readable."); throw new RuntimeException("Can't read file " + userMappingFileName); } List<String> lines = FileUtils.readLines(userMappingFile); lines.remove(0); //remove header for (String line : lines) { String[] blogToedx = line.split(MAPPING_SEPARATOR); //if the current line contained a valid mapping, add it to the map if (blogToedx.length == 2 && blogToedx[0] != null && !blogToedx[0].isEmpty() && blogToedx[1] != null && !blogToedx[1].isEmpty()) { blogToedxMap.put(blogToedx[0], blogToedx[1]); } } } if (jsonarray != null && jsonarray.equalsIgnoreCase(("true"))) { logger.trace("Set reader to expect a json array rather than regular json input."); this.dumpWrappedInJsonArray = true; } /* * Map data to DiscourseDB */ logger.info("Mapping blog posts and comments to DiscourseDB"); try (InputStream in = new FileInputStream(blogDumpFile)) { if (dumpWrappedInJsonArray) { //if the json dump is wrapped in a top-level array @SuppressWarnings("unchecked") List<ProsoloBlogPost> posts = (List<ProsoloBlogPost>) new ObjectMapper() .readValues(new JsonFactory().createParser(in), new TypeReference<List<ProsoloBlogPost>>() { }).next(); posts.stream().forEach(p -> converterService.mapPost(p, discourseName, dataSetName, blogToedxMap)); } else { //if the json dump is NOT wrapped in a top-level array Iterator<ProsoloBlogPost> pit = new ObjectMapper().readValues(new JsonFactory().createParser(in), ProsoloBlogPost.class); Iterable<ProsoloBlogPost> iterable = () -> pit; StreamSupport.stream(iterable.spliterator(), false) .forEach(p -> converterService.mapPost(p, discourseName, dataSetName, blogToedxMap)); } } logger.info("All done."); }
From source file:com.cj.restspecs.mojo.ConcatenateMojoTest.java
@Test public void testHeaderAndFooter() throws Exception { // GIVEN//ww w .j a v a 2s.co m FilesystemScenario mavenProject = new FilesystemScenario(); File destinationFile = new File(mavenProject.targetDir, "RestSpec.js"); ConcatenateMojo mojo = new ConcatenateMojo(); mojo.directories.add(relativePath(mavenProject.root, mavenProject.srcDir)); mojo.basedir = mavenProject.root; mojo.destinationFile = destinationFile; // WHEN mojo.execute(); // THEN String[] actualLines = FileUtils.readLines(destinationFile).toArray(new String[] {}); String[] expectedHeaderLines = { "/*jslint newcap: false*/", "/*global RestSpec:true */", "", "/*THIS FILE HAS BEEN AUTOMATICALLY GENERATED*/", "", "var RestSpec = [" }; String expectedFooter = "];"; for (int i = 0; i < expectedHeaderLines.length; i++) { assertEquals(expectedHeaderLines[i], actualLines[i]); } //check there are 2 lines which are just commas int commaCount = 0; for (String line : actualLines) { if (line.equals(",")) { commaCount++; } } assertEquals("There should be two commas between the rest specs", 2, commaCount); //check the last line assertEquals("The rest spec array should end with ']'", expectedFooter, actualLines[actualLines.length - 1]); }
From source file:de.tudarmstadt.ukp.similarity.experiments.coling2012.util.Evaluator.java
@SuppressWarnings("unchecked") public static void runEvaluationMetric(WekaClassifier wekaClassifier, EvaluationMetric metric, Dataset dataset) throws IOException { StringBuilder sb = new StringBuilder(); List<String> gold = ColingUtils.readGoldstandard(dataset); List<String> exp = FileUtils.readLines( new File(OUTPUT_DIR + "/" + dataset.toString() + "/" + wekaClassifier.toString() + "/output.csv")); if (metric.equals(EvaluationMetric.Accuracy)) { double acc = 0.0; for (int i = 0; i < gold.size(); i++) { if (gold.get(i).equals(exp.get(i))) acc++;// ww w . j a va2 s . c o m } acc /= gold.size(); sb.append(acc); } else if (metric.equals(EvaluationMetric.AverageF1)) { // Get all classes Set<String> classesSet = new HashSet<String>(); for (String cl : gold) classesSet.add(cl); // Order the classes List<String> classes = new ArrayList<String>(classesSet); // Initialize confusion matrix // exp\class A B // A x1 x2 // B x3 x4 int[][] matrix = new int[classes.size()][classes.size()]; // Initialize matrix for (int i = 0; i < classes.size(); i++) for (int j = 0; j < classes.size(); j++) matrix[i][j] = 0; // Construct confusion matrix for (int i = 0; i < gold.size(); i++) { int goldIndex = classes.indexOf(gold.get(i)); int expIndex = classes.indexOf(exp.get(i)); matrix[goldIndex][expIndex] += 1; } // Compute precision and recall per class double[] prec = new double[classes.size()]; double[] rec = new double[classes.size()]; for (int i = 0; i < classes.size(); i++) { double tp = matrix[i][i]; double fp = 0.0; double fn = 0.0; // FP for (int j = 0; j < classes.size(); j++) { if (i == j) continue; fp += matrix[j][i]; } // FN for (int j = 0; j < classes.size(); j++) { if (i == j) continue; fn += matrix[i][j]; } // Save prec[i] = tp / (tp + fp); rec[i] = tp / (tp + fn); } // Compute average F1 score across all classes double f1 = 0.0; for (int i = 0; i < classes.size(); i++) { double f1PerClass = (2 * prec[i] * rec[i]) / (prec[i] + rec[i]); f1 += f1PerClass; } f1 = f1 / classes.size(); // Output sb.append(f1); } FileUtils.writeStringToFile(new File(OUTPUT_DIR + "/" + dataset.toString() + "/" + wekaClassifier.toString() + "/" + metric.toString() + ".txt"), sb.toString()); }
From source file:com.ibm.watson.developer_cloud.professor_languo.pipeline.PipelineDriverTest.java
private void test_results_are_written_to_file() throws IOException { assertTrue(ResultWriter.class.getSimpleName() + " output file is missing", resultOutputFile.exists()); assertEquals("Test results file contains incorrect number of rows", 31 * com.ibm.watson.developer_cloud.professor_languo.pipeline.PipelineDriverTest.DummyQuestionAnswerer.DummyAnswerGenerator.NUM_ANSWERS + 1, FileUtils.readLines(resultOutputFile).size()); }
From source file:ml.shifu.dtrain.NNTest.java
@Test public void testNNApp() throws IOException { Properties props = new Properties(); LOG.info("Set property for Guagua driver"); props.setProperty(GuaguaConstants.MASTER_COMPUTABLE_CLASS, NNMaster.class.getName()); props.setProperty(GuaguaConstants.WORKER_COMPUTABLE_CLASS, NNWorker.class.getName()); props.setProperty(GuaguaConstants.GUAGUA_ITERATION_COUNT, "30"); props.setProperty(GuaguaConstants.GUAGUA_MASTER_RESULT_CLASS, NNParams.class.getName()); props.setProperty(GuaguaConstants.GUAGUA_WORKER_RESULT_CLASS, NNParams.class.getName()); props.setProperty(GuaguaConstants.GUAGUA_INPUT_DIR, getClass().getResource("/data/wdbc/wdbc.normalized").toString()); props.setProperty(GuaguaConstants.GUAGUA_MASTER_INTERCEPTERS, NNOutput.class.getName()); props.setProperty(DtrainConstants.GUAGUA_NN_OUTPUT, OUTPUT); props.setProperty(DtrainConstants.NN_PROGRESS_FILE, PROGRESS_FILE_STRING); props.setProperty(DtrainConstants.NN_TRAINER_ID, "#1"); props.setProperty(DtrainConstants.NN_TMP_MODELS_FOLDER, TMP_MODELS_FOLDER); // Since many parameter setting in NNMaster/NNWorker lack default value. So we have to // specify all these parameters, or the master/worker won't work properly. Settings // below contains all indispensable and optional parameters. LOG.info("Set property for NN trainer"); props.setProperty(DtrainConstants.NN_DATA_DELIMITER, ","); props.setProperty(DtrainConstants.SHIFU_DTRAIN_BAGGING_NUM, "1"); props.setProperty(DtrainConstants.SHIFU_DTRAIN_IS_TRAIN_ON_DISK, "false"); props.setProperty(DtrainConstants.SHIFU_DTRAIN_BAGGING_SAMPLE_RATE, "1.0"); props.setProperty(DtrainConstants.SHIFU_DTRAIN_CROSS_VALIDATION_RATE, "0.2"); props.setProperty(DtrainConstants.SHIFU_DTRAIN_NN_INPUT_NODES, "30"); props.setProperty(DtrainConstants.SHIFU_DTRAIN_NN_OUTPUT_NODES, "1"); props.setProperty(DtrainConstants.SHIFU_DTRAIN_NN_HIDDEN_LAYERS, "2"); props.setProperty(DtrainConstants.SHIFU_DTRAIN_NN_HIDDEN_NODES, "30,20"); props.setProperty(DtrainConstants.SHIFU_DTRAIN_NN_ACT_FUNCS, DtrainConstants.NN_SIGMOID + "," + DtrainConstants.NN_SIGMOID); props.setProperty(DtrainConstants.SHIFU_DTRAIN_NN_PROPAGATION, NNUtils.QUICK_PROPAGATION); props.setProperty(DtrainConstants.SHIFU_DTRAIN_NN_LEARNING_RATE, "0.2"); props.setProperty(DtrainConstants.SHIFU_DTRAIN_PARALLEL, "true"); GuaguaUnitDriver<NNParams, NNParams> driver = new GuaguaMRUnitDriver<NNParams, NNParams>(props); driver.run();/*from w ww.j a va 2s. c o m*/ // Check output files exist. File finalModel = new File(NN_TEST); Assert.assertTrue(finalModel.exists()); File progressFile = new File(PROGRESS_FILE_STRING); Assert.assertTrue(progressFile.exists()); // Check final output error less than threshold. List<String> errorList = FileUtils.readLines(progressFile); String errorLine = errorList.get(errorList.size() - 1); Pattern resultPattern = Pattern .compile("Train\\s+Error:(\\d+\\.\\d+)\\s+Validation\\s+Error:(\\d+\\.\\d+)"); Matcher errorMatcher = resultPattern.matcher(errorLine); Assert.assertTrue(errorMatcher.find()); double trainErr = Double.parseDouble(errorMatcher.group(1)); double testErr = Double.parseDouble(errorMatcher.group(2)); double threshold = 0.2; Assert.assertTrue(Double.compare((trainErr + testErr) / 2, threshold) <= 0); // Check final model. // Here only simply check the output weight size. BasicNetwork model = (BasicNetwork) EncogDirectoryPersistence.loadObject(new File(OUTPUT)); Assert.assertEquals(model.getFlat().getWeights().length, 31 * 30 + 31 * 20 + 21); }
From source file:de.tudarmstadt.ukp.dkpro.wsd.si.twsi.TwsiSenseInventoryBase.java
public List<String[]> getSubstitutions(File file) throws IOException { List<String[]> substitutions = new ArrayList<String[]>(); for (String line : FileUtils.readLines(file)) { substitutions.add(line.split("\t")); }/*w w w. j av a 2 s .co m*/ return substitutions; }
From source file:net.jperf.logback.AppenderTest.java
public void testCsvRenderer() throws Exception { LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory(); JoranConfigurator configurator = new JoranConfigurator(); configurator.setContext(lc);//w w w . ja va2 s. co m // the context was probably already configured by default configuration // rules lc.reset(); configurator.doConfigure(getClass().getResource("logbackWCsv.xml")); Logger logger = lc.getLogger("net.jperf.CsvAppenderTest"); for (int i = 0; i < 20; i++) { StopWatch stopWatch = new Slf4JStopWatch(logger); Thread.sleep(i * 10); stopWatch.stop("csvTest"); } //close the appender logger.getAppender("coalescingStatistics").stop(); //verify the statisticsLog.csv file DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); for (Object line : FileUtils.readLines(new File("./target/statisticsLogback.csv"))) { String[] values = line.toString().split(","); //first column is the tag assertEquals("\"csvTest\"", values[0]); //next 2 columns are the dates - ensure they can be parsed assertTrue(dateFormat.parse(values[1]).before(dateFormat.parse(values[2]))); //next 3 columns are mean, min and max double mean = Double.parseDouble(values[3]); int min = Integer.parseInt(values[4]); int max = Integer.parseInt(values[5]); assertTrue(mean >= min && mean <= max); //next column is stddev - ust make sure it's parseable Double.parseDouble(values[6]); //next column is count assertTrue(Integer.parseInt(values[7]) < 20); //final column is TPS - just make sure it's parseable Double.parseDouble(values[8]); } //verify the pivotedStatisticsLog.csv file for (Object line : FileUtils.readLines(new File("./target/pivotedStatisticsLog.csv"))) { String[] values = line.toString().split(","); //first 2 columns are the dates - ensure they can be parsed assertTrue(dateFormat.parse(values[0]).before(dateFormat.parse(values[1]))); //next column is mean, ensure it can be parsed Double.parseDouble(values[2]); //last column should be empty, so make sure last char on string is comma assertEquals(',', line.toString().charAt(line.toString().length() - 1)); } }
From source file:de.unidue.ltl.flextag.core.reports.CvAvgPerWordClassReport.java
private Map<String, WordClass> getWcPerformances(File locateKey) throws IOException { Map<String, WordClass> wcp = new HashMap<>(); List<String> lines = FileUtils.readLines(locateKey); Map<String, String> labels = getLabels(lines); for (String l : lines) { if (l.startsWith("#")) { continue; }//from www. ja v a 2s. c o m String[] entry = splitAtFirstEqualSignRightHandSide(l); String pg = entry[1]; String[] split = pg.split(";"); if (split.length < 2) { System.out.println("ERROR\t" + l); continue; } String prediction = labels.get(split[0]); String gold = labels.get(split[1]); WordClass wordClass = wcp.get(gold); if (wordClass == null) { wordClass = new WordClass(); } if (gold.equals(prediction)) { wordClass.incrementCorrect(); } else { wordClass.incrementIncorrect(); } wcp.put(gold, wordClass); } return wcp; }
From source file:edu.isistan.carcha.util.Utils.java
/** * Extracts crosscutting concerns(ccc) from a text file where each ccc is<br> * <code>//from w ww.j a va2s . c om * (classification) \t\t (label) * <code> * @param fileName The file name * @return The entity list * @throws IOException */ @SuppressWarnings("unchecked") public static List<Entity> extractCrosscuttingConcernsFromTextFile(String fileName) throws IOException { List<Entity> ccc = Utils.transformedList(FileUtils.readLines(new File(fileName)), new String2Concern(true)); return ccc; }