List of usage examples for org.apache.commons.io FileUtils lineIterator
public static LineIterator lineIterator(File file, String encoding) throws IOException
File. From source file:org.opensextant.examples.RegexTest.java
/** * The main method./*from www. j av a 2s . com*/ * * @param args * the arguments */ public static void main(String[] args) { // file containing the tab separated test data String testFileName = args[0]; // the file containing the regex definintions String patternFileName = args[1]; // file into which we will write the results String resultFileName = args[2]; File testFile = new File(testFileName); URL patternFile = null; BufferedWriter resWriter = null; // setup the output File resFile = new File(resultFileName); try { resWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(resFile), "UTF-8")); } catch (UnsupportedEncodingException e) { LOGGER.error("Couldnt write to " + resFile.getName() + ":" + e.getMessage(), e); return; } catch (FileNotFoundException e) { LOGGER.error("Couldnt write to " + resFile.getName() + ":" + e.getMessage(), e); return; } // write the results header try { resWriter.write( "Entity Type\tPos\\Neg\tTest Input\tScore\tComplete\tCount\tTypes Found\tRules Matched\tAnnotations Found"); resWriter.newLine(); } catch (IOException e) { LOGGER.error("Couldnt write to " + resFile.getName() + ":" + e.getMessage(), e); } // get the pattern file as a URL try { patternFile = new File(patternFileName).toURI().toURL(); } catch (MalformedURLException e) { LOGGER.error("Couldn't use pattern file " + patternFileName + ":" + e.getMessage(), e); } // initialize the regex matcher RegexMatcher reger = new RegexMatcher(patternFile); LOGGER.info("Loaded " + reger.getRules().size() + " rules " + " for types " + reger.getTypes()); LOGGER.info("Writing results to " + resFile.getAbsolutePath()); // loop over the lines of the test file LineIterator iter = null; try { iter = FileUtils.lineIterator(testFile, "UTF-8"); } catch (IOException e) { LOGGER.error("Couldnt read from " + testFile.getName() + ":" + e.getMessage(), e); } int lineCount = 0; while (iter.hasNext()) { // get next line String line = iter.next(); // skip comments and blank lines if (line == null || line.startsWith("#") || line.trim().length() == 0) { continue; } lineCount++; // get the fields of the line String[] pieces = line.split("[\t\\s]+", 3); String entityType = pieces[0]; String posOrNeg = pieces[1]; String testText = pieces[2]; // send the test text to regex matcher List<RegexAnnotation> annos = reger.match(testText); // examine the results and return a line to be sent to the results // file String results = score(entityType, posOrNeg, testText, annos); annos.clear(); try { // write the original line and the results to the results file resWriter.write(line + "\t" + results); resWriter.newLine(); } catch (IOException e) { LOGGER.error("Couldn't write to " + resFile.getName() + ":" + e.getMessage(), e); } } iter.close(); LOGGER.info("Tagged and scored " + lineCount + " test lines"); // cleanup try { resWriter.flush(); resWriter.close(); } catch (IOException e) { LOGGER.error("Couldn't close " + resFile.getName() + ":" + e.getMessage(), e); } }
From source file:org.opensextant.matching.DataLoader.java
private static String flatten(String currentPath) { File topDir = new File(currentPath).getParentFile(); File input = new File(currentPath); Map<File, String> index = new HashMap<File, String>(); // read the index file into the index Map // loop over the lines of the index file LineIterator indexIter = null;/*from w w w.j a v a2 s . c o m*/ try { indexIter = FileUtils.lineIterator(input, "UTF-8"); } catch (IOException e) { LOGGER.error("Couldnt read from " + input.getName() + ":", e); return null; } if (indexIter != null) { while (indexIter.hasNext()) { // get next line String line = indexIter.next(); String[] pieces = line.split(":"); File subFile = new File(topDir, pieces[0]); String tmpVal = pieces[1]; if (pieces.length >= 3) { tmpVal = tmpVal + ":" + pieces[2]; } index.put(subFile, tmpVal); } } File tmp = null; try { tmp = File.createTempFile("vocab", "txt"); } catch (IOException e) { LOGGER.error("Could not create temp file when flattening vocab:", e); return null; } // loop over the files mentioned in the index and write to temp file int indexID = 0; for (File in : index.keySet()) { String[] catAndTax = index.get(in).split(":"); String cat = catAndTax[0]; String tax = ""; if (catAndTax.length > 1) { tax = catAndTax[1]; } else { tax = "NONE"; } // loop over the lines of the subfiles file // write the new flat contents to the temp file LineIterator contentIter = null; try { contentIter = FileUtils.lineIterator(in, "UTF-8"); } catch (IOException e) { LOGGER.error("Couldnt read from " + in.getName(), e); return null; } if (contentIter != null) { while (contentIter.hasNext()) { // get next line String line = contentIter.next(); // concat the pieces String out = indexID + "\t" + line + "\t" + cat + "\t" + tax + "\n"; // write all pieces to temp try { FileUtils.writeStringToFile(tmp, out, "UTF-8", true); } catch (IOException e) { LOGGER.error("Could not write to temp file when flattening vocab:", e); } indexID++; } } } LOGGER.info("Flattened " + indexID + " vocabulary entries to temp file"); // return temp file path return tmp.getAbsolutePath(); }
From source file:org.proninyaroslav.libretorrent.core.IPFilterParser.java
public static boolean parseDATFilterFile(String path, ip_filter filter) { if (path == null || filter == null) { return false; }/*from w ww. ja v a2 s.co m*/ File file = new File(path); if (!file.exists()) { return false; } LineIterator it = null; try { it = FileUtils.lineIterator(file, "UTF-8"); } catch (IOException e) { Log.e(TAG, Log.getStackTraceString(e)); } if (it == null) { return false; } long lineNum = 0; long badLineNum = 0; try { while (it.hasNext()) { ++lineNum; String line = it.nextLine(); line = line.trim(); if (line.isEmpty()) { continue; } /* Ignoring commented lines */ if (line.startsWith("#") || line.startsWith("//")) { continue; } /* Line should be split by commas */ String[] parts = line.split(","); long elementNum = parts.length; /* IP Range should be split by a dash */ String[] ips = parts[0].split("-"); if (ips.length != 2) { Log.w(TAG, "parseDATFilterFile: line " + lineNum + " is malformed."); Log.w(TAG, "Line was " + line); ++badLineNum; continue; } String startIp = cleanupIPAddress(ips[0]); if (startIp == null || startIp.isEmpty()) { Log.w(TAG, "parseDATFilterFile: line " + lineNum + " is malformed."); Log.w(TAG, "Start IP of the range is malformated: " + ips[0]); ++badLineNum; continue; } error_code error = new error_code(); address startAddr = address.from_string(startIp, error); if (error.value() > 0) { Log.w(TAG, "parseDATFilterFile: line " + lineNum + " is malformed."); Log.w(TAG, "Start IP of the range is malformated:" + ips[0]); ++badLineNum; continue; } String endIp = cleanupIPAddress(ips[1]); if (endIp == null || endIp.isEmpty()) { Log.w(TAG, "parseDATFilterFile: line " + lineNum + " is malformed."); Log.w(TAG, "End IP of the range is malformated: " + ips[1]); ++badLineNum; continue; } address endAddr = address.from_string(endIp, error); if (error.value() > 0) { Log.w(TAG, "parseDATFilterFile: line " + lineNum + " is malformed."); Log.w(TAG, "End IP of the range is malformated:" + ips[1]); ++badLineNum; continue; } if (startAddr.is_v4() != endAddr.is_v4()) { Log.w(TAG, "parseDATFilterFile: line " + lineNum + " is malformed."); Log.w(TAG, "One IP is IPv4 and the other is IPv6!"); ++badLineNum; continue; } /* Check if there is an access value (apparently not mandatory) */ int accessNum = 0; if (elementNum > 1) { /* There is possibly one */ accessNum = Integer.parseInt(parts[1].trim()); } /* Ignoring this rule because access value is too high */ if (accessNum > 127) { continue; } try { filter.add_rule(startAddr, endAddr, ip_filter.access_flags.blocked.swigValue()); } catch (Exception e) { Log.w(TAG, "parseDATFilterFile: line " + lineNum + " is malformed."); Log.w(TAG, "Line was " + line); ++badLineNum; } } } finally { it.close(); } return badLineNum < lineNum; }
From source file:org.proninyaroslav.libretorrent.core.IPFilterParser.java
public static boolean parseP2PFilterFile(String path, ip_filter filter) { if (path == null || filter == null) { return false; }//from ww w . j a v a2s . co m File file = new File(path); if (!file.exists()) { return false; } LineIterator it = null; try { it = FileUtils.lineIterator(file, "UTF-8"); } catch (IOException e) { Log.e(TAG, Log.getStackTraceString(e)); } if (it == null) { return false; } long lineNum = 0; long badLineNum = 0; try { while (it.hasNext()) { ++lineNum; String line = it.nextLine(); line = line.trim(); if (line.isEmpty()) { continue; } /* Ignoring commented lines */ if (line.startsWith("#") || line.startsWith("//")) { continue; } /* Line should be split by ':' */ String[] parts = line.split(":"); if (parts.length < 2) { Log.w(TAG, "parseP2PFilterFile: line " + lineNum + " is malformed."); ++badLineNum; continue; } /* IP Range should be split by a dash */ String[] ips = parts[1].split("-"); if (ips.length != 2) { Log.w(TAG, "parseP2PFilterFile: line " + lineNum + " is malformed."); Log.w(TAG, "Line was " + line); ++badLineNum; continue; } String startIp = cleanupIPAddress(ips[0]); if (startIp == null || startIp.isEmpty()) { Log.w(TAG, "parseP2PFilterFile: line " + lineNum + " is malformed."); Log.w(TAG, "Start IP of the range is malformated: " + ips[0]); ++badLineNum; continue; } error_code error = new error_code(); address startAddr = address.from_string(startIp, error); if (error.value() > 0) { Log.w(TAG, "parseP2PFilterFile: line " + lineNum + " is malformed."); Log.w(TAG, "Start IP of the range is malformated:" + ips[0]); ++badLineNum; continue; } String endIp = cleanupIPAddress(ips[1]); if (endIp == null || endIp.isEmpty()) { Log.w(TAG, "parseP2PFilterFile: line " + lineNum + " is malformed."); Log.w(TAG, "End IP of the range is malformated: " + ips[1]); ++badLineNum; continue; } address endAddr = address.from_string(endIp, error); if (error.value() > 0) { Log.w(TAG, "parseP2PFilterFile: line " + lineNum + " is malformed."); Log.w(TAG, "End IP of the range is malformated:" + ips[1]); ++badLineNum; continue; } if (startAddr.is_v4() != endAddr.is_v4()) { Log.w(TAG, "parseP2PFilterFile: line " + lineNum + " is malformed."); Log.w(TAG, "One IP is IPv4 and the other is IPv6!"); ++badLineNum; continue; } try { filter.add_rule(startAddr, endAddr, ip_filter.access_flags.blocked.swigValue()); } catch (Exception e) { Log.w(TAG, "parseP2PFilterFile: line " + lineNum + " is malformed."); Log.w(TAG, "Line was " + line); ++badLineNum; } } } finally { it.close(); } return badLineNum < lineNum; }
From source file:org.schedulesdirect.api.LineupTest.java
static private void initJsonData() throws Exception { try {/*from w ww .j a v a 2 s .com*/ if (!SampleData.updateFor(SampleType.LINEUPS, false)) LOG.info("Using current sample data because it's less than a week old."); } catch (Exception e) { if (!SampleData.exists(SampleType.LINEUPS)) throw new IOException("No sample data available!", e); else LOG.warn("Failed to download fresh sample data; using existing data instead!"); } LineIterator itr = FileUtils.lineIterator(SampleData.locate(SampleType.LINEUPS), "UTF-8"); while (itr.hasNext()) SAMPLE_DATA.add(itr.nextLine()); }
From source file:org.sonar.dev.TrimMojo.java
private void trimDirectory() throws MojoExecutionException { File[] files = scanFiles();/*w ww.jav a 2 s .co m*/ for (File file : files) { StringBuilder sb = new StringBuilder(); try { LineIterator lines = FileUtils.lineIterator(file, sourceEncoding); while (lines.hasNext()) { String line = lines.nextLine(); if (line != null && !"".equals(line.trim())) { sb.append(line.trim()); sb.append(IOUtils.LINE_SEPARATOR); } } FileUtils.writeStringToFile(file, sb.toString(), sourceEncoding); } catch (IOException e) { throw new MojoExecutionException("Can not trim the file " + file, e); } } getLog().info("Trimmed files: " + files.length); }
From source file:org.sonar.plugins.codesize.LineCounter.java
private int countFile(File file) { int lines = 0; LineIterator lineIterator = null;/*from www. j av a2 s . c o m*/ try { lineIterator = FileUtils.lineIterator(file, defaultCharset.name()); for (; lineIterator.hasNext(); lineIterator.next()) { lines++; } } catch (IOException e) { LOG.error("Could not open file", e); } finally { LineIterator.closeQuietly(lineIterator); } LOG.debug(file.getName() + ": " + lines); return lines; }
From source file:org.tanaguru.referentiel.creator.CodeGeneratorMojo.java
/** * * @return/* w w w . j ava 2 s. co m*/ */ private Iterable<CSVRecord> getCsv() { // we parse the csv file to extract the first line and get the headers LineIterator lineIterator; try { lineIterator = FileUtils.lineIterator(dataFile, Charset.defaultCharset().name()); } catch (IOException ex) { Logger.getLogger(CodeGeneratorMojo.class.getName()).log(Level.SEVERE, null, ex); lineIterator = null; } String[] csvHeaders = lineIterator.next().split(String.valueOf(delimiter)); isCriterionPresent = extractCriterionFromCsvHeader(csvHeaders); try { extractAvailableLangsFromCsvHeader(csvHeaders); } catch (I18NLanguageNotFoundException ex) { Logger.getLogger(CodeGeneratorMojo.class.getName()).log(Level.SEVERE, null, ex); return null; } // from here we just add each line to a build to re-create the csv content // without the first line. StringBuilder strb = new StringBuilder(); while (lineIterator.hasNext()) { strb.append(lineIterator.next()); strb.append("\n"); } Reader in; try { in = new StringReader(strb.toString()); CSVFormat csvf = CSVFormat.newFormat(delimiter).withHeader(csvHeaders); return csvf.parse(in); } catch (FileNotFoundException ex) { Logger.getLogger(CodeGeneratorMojo.class.getName()).log(Level.SEVERE, null, ex); return null; } catch (IOException ex) { Logger.getLogger(CodeGeneratorMojo.class.getName()).log(Level.SEVERE, null, ex); return null; } }
From source file:org.verwandlung.voj.judger.core.Comparator.java
/** * ?.// www. j a v a 2s .c om * @param standardOutputFilePath - * @param outputFilePath - * @return ?? */ public boolean isOutputTheSame(String standardOutputFilePath, String outputFilePath) throws IOException { File stdFile = new File(standardOutputFilePath); File file = new File(outputFilePath); LineIterator stdFileItr = FileUtils.lineIterator(stdFile, "UTF-8"); LineIterator fileItr = FileUtils.lineIterator(file, "UTF-8"); boolean isFileOutputTheSame = isFileOutputTheSame(stdFileItr, fileItr); LineIterator.closeQuietly(stdFileItr); LineIterator.closeQuietly(fileItr); return isFileOutputTheSame; }
From source file:org.wikimedia.analytics.varnishkafka.Cli.java
private Integer writeEscapedOutput() { int n = 0;//from www. j a v a 2 s . c o m OutputStream out = null; BufferedOutputStream bos = null; try { LineIterator it = FileUtils.lineIterator(inputFile, "UTF-8"); File outputFile = new File(cwd.getPath(), "test." + getFormat()); outputFile.delete(); log.info("Output file path: " + outputFile.toString()); out = new FileOutputStream(outputFile); bos = new BufferedOutputStream(out); setStart(System.nanoTime()); while (it.hasNext()) { n++; String line = it.nextLine(); String[] fields = line.split("\\t"); String ua = fields[14].replace("\\t", " ").replace("\\n", " "); fields[14] = ua; for (String field : fields) { bos.write(field.getBytes()); bos.write("\t".getBytes()); } bos.write("\n".getBytes()); } setEnd(System.nanoTime()); } catch (IOException e) { e.printStackTrace(); } finally { try { out.close(); bos.close(); } catch (IOException e) { e.printStackTrace(); } } return n; }