Example usage for org.apache.commons.io FileUtils lineIterator

List of usage examples for org.apache.commons.io FileUtils lineIterator

  1. HOME
  2. Java
  3. org.apache.commons
  4. org.apache.commons.io.*
  5. FileUtils
  6. lineIterator

Introduction

In this page you can find the example usage for org.apache.commons.io FileUtils lineIterator.

Prototype

public static LineIterator lineIterator(File file) throws IOException

Source Link

Document

Returns an Iterator for the lines in a File using the default encoding for the VM.

Usage

From source file:org.jcvi.ometa.engine.LoadingEngine.java

public void batchLoad() throws Exception {
    String userName = usage.getUsername();
    String passWord = usage.getPassword();
    String serverUrl = usage.getServerUrl();

    String eventFileName = usage.getInputFilename();
    String eventName = usage.getEventName();
    String projectName = usage.getProjectName();
    String outputPath = usage.getOutputLocation();

    int batchSizeInt = 1;
    String batchSize = usage.getBatchSize();
    if (batchSize != null && !batchSize.isEmpty()) {
        batchSizeInt = Integer.parseInt(batchSize);
    }//from   w  ww.  j av a  2s .  c  om

    File eventFile = new File(eventFileName);

    Long timeStamp = new Date().getTime();
    File logFile = new File(outputPath + File.separator + "ometa.log");
    FileWriter logWriter = new FileWriter(logFile, false);

    int successCount = 0;
    File processedFile = new File(outputPath + File.separator + "ometa_processed.csv");
    FileWriter processedWriter = new FileWriter(processedFile, false);
    int failedCount = 0;
    File failedFile = new File(outputPath + File.separator + "ometa_failed.csv");
    FileWriter failedWriter = new FileWriter(failedFile, false);

    // Must break this file up, and deposit it into a temporary output directory.
    String userBase = System.getProperty("user.home");
    ScratchUtils.setScratchBaseLocation(userBase + "/" + Constants.SCRATCH_BASE_LOCATION);
    File scratchLoc = ScratchUtils.getScratchLocation(timeStamp, "LoadingEngine__" + eventFile.getName());

    int processedLineCount = 0;
    try {
        BeanWriter writer = new BeanWriter(serverUrl, userName, passWord);

        LineIterator lineIterator = FileUtils.lineIterator(eventFile);
        int lineCount = 0;

        String headerLine = null;
        while (lineIterator.hasNext()) {
            ++lineCount;
            String currLine = lineIterator.nextLine();

            if (lineCount == 1) {
                headerLine = currLine;
                processedWriter.write(currLine + "\n");
                failedWriter.write(currLine + "\n");
                continue;
            } else if (lineCount == 2 && (currLine.startsWith("#") || currLine.startsWith("\"#"))) { //skip comment line
                continue;
            } else {
                File singleEventFile = new File(scratchLoc.getAbsoluteFile() + File.separator + "temp.csv");
                List<String> lines = new ArrayList<String>(2);
                lines.add(headerLine);
                lines.add(currLine);
                FileUtils.writeLines(singleEventFile, lines);

                try {
                    String eventTarget = writer.writeEvent(singleEventFile, eventName, projectName, true);
                    logWriter.write(String.format("[%d] loaded event for %s\n", lineCount, eventTarget));
                    processedWriter.write(currLine + "\n");
                    successCount++;
                } catch (Exception ex) {
                    failedWriter.write(currLine + "\n");
                    logWriter.write(String.format("[%d] failed :\n", lineCount));
                    logWriter.write(ex.getMessage() + "\n");
                    failedCount++;
                }
                processedLineCount++;
            }
        }
    } catch (IOException ioe) {
        System.err.println("IOException: " + ioe.getMessage());
    } catch (Exception ex) {
        String exceptionString = ex.getCause() == null ? ex.getMessage() : ex.getCause().getMessage();
        logWriter.write(exceptionString);
        throw ex;
    } finally {
        if (scratchLoc != null && scratchLoc.exists()) {
            File cleanupDir = scratchLoc;
            String parentDirName = cleanupDir.getParentFile().getName();
            try {
                Long.parseLong(parentDirName);
                cleanupDir = cleanupDir.getParentFile();
            } catch (NumberFormatException nf) {
            }
            FileUtils.forceDelete(cleanupDir);
        }

        logWriter.close();
        processedWriter.close();
        failedWriter.close();
        System.out.printf("total: %d, processed: %d, failed: %d\n", processedLineCount, successCount,
                failedCount);
        System.out.println("log file: " + logFile.getAbsolutePath());
    }
}

From source file:org.jdev.emg.sonar.cci.CCIXmlMetricsDecorator.java

@Override
public void decorate(Resource resource, DecoratorContext context) {
    if (!Qualifiers.isFile(resource)) {
        return;//from  w w w .j  a  v a2  s.  c om
    }
    ProjectFileSystem fileSystem = context.getProject().getFileSystem();
    File file = lookup(resource, fileSystem);

    try {
        if (readFirstByte(file) != '<') {
            return;
        }
    } catch (IOException e) {
        throw new SonarException(e);
    }

    int numCommentLines;
    CCICountCommentParser commentCounter = new CCICountCommentParser();
    try {
        numCommentLines = commentCounter.countLinesOfComment(FileUtils.openInputStream(file));
        if (numCommentLines == -1) {
            return;
        }
    } catch (IOException e) {
        throw new SonarException(e);
    }

    LineIterator iterator = null;
    int numLines = 0;
    int numBlankLines = 0;
    try {
        Charset charset = fileSystem.getSourceCharset();
        iterator = charset == null ? FileUtils.lineIterator(file)
                : FileUtils.lineIterator(file, charset.name());
        while (iterator.hasNext()) {
            String line = iterator.nextLine();
            numLines++;
            if (line.trim().isEmpty()) {
                numBlankLines++;
            }
        }
    } catch (IOException e) {
        LOG.warn("error reading " + file + " to collect metrics", e);
    } finally {
        LineIterator.closeQuietly(iterator);
    }

    context.saveMeasure(CoreMetrics.LINES, (double) numLines); // Lines
    context.saveMeasure(CoreMetrics.COMMENT_LINES, (double) numCommentLines); // Non Commenting Lines of Code
    context.saveMeasure(CoreMetrics.NCLOC, (double) numLines - numBlankLines - numCommentLines); // Comment Lines
}

From source file:org.jibenakka.sample.mapreduce.wordcount.worker.Mapper.java

/**
 *
 * @param work/*from   ww w  .ja  v  a2  s .  c  o  m*/
 */
protected void extractFileMapResult(InitialMapWork work) {
    LinkedList<Future<Map<String, Integer>>> futureResults = new LinkedList<Future<Map<String, Integer>>>();
    LineIterator lineIterator = null;
    try {
        File fileToCount = work.getFileToCount();
        lineIterator = FileUtils.lineIterator(fileToCount);

        while (lineIterator.hasNext()) {
            // All work including special character handling done at worker
            // level
            String line = lineIterator.nextLine();
            // key is just the file name - initial mapping is easy
            // hard part comes with partitioning and reduction
            // we assume that we have unique file names in the dir
            MapWork newWork = new MapWork(fileToCount.getName(), line);

            Future<Map<String, Integer>> future = this.workRouter.sendRequestReplyFuture(newWork, 30000,
                    getContext());
            future.await();
            futureResults.add(future);
        }

        // FinalMapResult result = new FinalMapResult(
        // (LinkedList<Future<Map<String, Integer>>>) futureResults);
        getContext().channel().sendOneWay(futureResults);
    } catch (IOException e) {
        e.printStackTrace();
    } finally {
        if (lineIterator != null) {
            lineIterator.close();
        }
    }
}

From source file:org.kuali.kfs.gl.batch.PreScrubberStep.java

@Override
protected CustomBatchExecutor getCustomBatchExecutor() {
    return new CustomBatchExecutor() {
        public boolean execute() {
            StopWatch stopWatch = new StopWatch();
            stopWatch.start();/*from   w  w  w  .  j a va  2 s  .c o m*/

            String inputFile = batchFileDirectoryName + File.separator
                    + GeneralLedgerConstants.BatchFileSystem.BACKUP_FILE
                    + GeneralLedgerConstants.BatchFileSystem.EXTENSION;
            String outputFile = batchFileDirectoryName + File.separator
                    + GeneralLedgerConstants.BatchFileSystem.PRE_SCRUBBER_FILE
                    + GeneralLedgerConstants.BatchFileSystem.EXTENSION;

            PreScrubberReportData preScrubberReportData = null;
            LineIterator oeIterator = null;
            try {
                oeIterator = FileUtils.lineIterator(new File(inputFile));
                preScrubberReportData = preScrubberService.preprocessOriginEntries(oeIterator, outputFile);
            } catch (IOException e) {
                LOG.error("IO exception occurred during pre scrubbing.", e);
                throw new RuntimeException("IO exception occurred during pre scrubbing.", e);
            } finally {
                LineIterator.closeQuietly(oeIterator);
            }

            if (preScrubberReportData != null) {
                new PreScrubberReport().generateReport(preScrubberReportData, preScrubberReportWriterService);
            }

            stopWatch.stop();
            if (LOG.isDebugEnabled()) {
                LOG.debug("scrubber step of took " + (stopWatch.getTotalTimeSeconds() / 60.0)
                        + " minutes to complete");
            }
            return true;
        }
    };
}

From source file:org.kuali.kfs.gl.batch.service.impl.ScrubberProcessImpl.java

/**
 * Scrub this single group read only. This will only output the scrubber report. It won't output any other groups.
 *
 * @param group the origin entry group that should be scrubbed
 * @param the document number of any specific entries to scrub
 *///from  ww w.  ja  va2 s .c o  m
@Override
public void scrubGroupReportOnly(String fileName, String documentNumber) {
    LOG.debug("scrubGroupReportOnly() started");
    String unsortedFile = fileName;
    this.inputFile = fileName + ".sort";
    this.validFile = batchFileDirectoryName + File.separator
            + GeneralLedgerConstants.BatchFileSystem.SCRUBBER_VALID_OUTPUT_FILE
            + GeneralLedgerConstants.BatchFileSystem.EXTENSION;
    this.errorFile = batchFileDirectoryName + File.separator
            + GeneralLedgerConstants.BatchFileSystem.SCRUBBER_ERROR_OUTPUT_FILE
            + GeneralLedgerConstants.BatchFileSystem.EXTENSION;
    this.expiredFile = batchFileDirectoryName + File.separator
            + GeneralLedgerConstants.BatchFileSystem.SCRUBBER_EXPIRED_OUTPUT_FILE
            + GeneralLedgerConstants.BatchFileSystem.EXTENSION;
    String prescrubOutput = batchFileDirectoryName + File.separator
            + GeneralLedgerConstants.BatchFileSystem.PRE_SCRUBBER_FILE
            + GeneralLedgerConstants.BatchFileSystem.EXTENSION;
    this.ledgerSummaryReport = new LedgerSummaryReport();
    runDate = calculateRunDate(dateTimeService.getCurrentDate());

    PreScrubberReportData preScrubberReportData = null;

    // run pre-scrubber on the raw input into the sort process
    LineIterator inputEntries = null;
    try {
        inputEntries = FileUtils.lineIterator(new File(unsortedFile));
        preScrubberReportData = preScrubberService.preprocessOriginEntries(inputEntries, prescrubOutput);
    } catch (IOException e1) {
        LOG.error("Error encountered trying to prescrub GLCP/LLCP document", e1);
        throw new RuntimeException("Error encountered trying to prescrub GLCP/LLCP document", e1);
    } finally {
        LineIterator.closeQuietly(inputEntries);
    }
    if (preScrubberReportData != null) {
        preScrubberReportWriterService.setDocumentNumber(documentNumber);
        ((WrappingBatchService) preScrubberReportWriterService).initialize();
        try {
            new PreScrubberReport().generateReport(preScrubberReportData, preScrubberReportWriterService);
        } finally {
            ((WrappingBatchService) preScrubberReportWriterService).destroy();
        }
    }
    BatchSortUtil.sortTextFileWithFields(prescrubOutput, inputFile, new ScrubberSortComparator());

    scrubEntries(true, documentNumber);

    // delete files
    File deleteSortFile = new File(inputFile);
    File deleteValidFile = new File(validFile);
    File deleteErrorFile = new File(errorFile);
    File deleteExpiredFile = new File(expiredFile);
    try {
        deleteSortFile.delete();
        deleteValidFile.delete();
        deleteErrorFile.delete();
        deleteExpiredFile.delete();
    } catch (Exception e) {
        LOG.error("scrubGroupReportOnly delete output files process Stopped: " + e.getMessage());
        throw new RuntimeException(
                "scrubGroupReportOnly delete output files process Stopped: " + e.getMessage(), e);
    }
}

From source file:org.kuali.kfs.module.ld.batch.LaborPreScrubberStep.java

/**
 * @see org.kuali.kfs.sys.batch.AbstractWrappedBatchStep#getCustomBatchExecutor()
 *///www . ja v a 2 s  .  c o m
@Override
protected CustomBatchExecutor getCustomBatchExecutor() {
    return new CustomBatchExecutor() {

        /**
         * @see org.kuali.kfs.sys.batch.service.WrappedBatchExecutorService.CustomBatchExecutor#execute()
         */
        public boolean execute() {
            StopWatch stopWatch = new StopWatch();
            stopWatch.start();

            String inputFile = batchFileDirectoryName + File.separator
                    + LaborConstants.BatchFileSystem.BACKUP_FILE
                    + GeneralLedgerConstants.BatchFileSystem.EXTENSION;
            String outputFile = batchFileDirectoryName + File.separator
                    + LaborConstants.BatchFileSystem.PRE_SCRUBBER_FILE
                    + GeneralLedgerConstants.BatchFileSystem.EXTENSION;

            PreScrubberReportData preScrubberReportData = null;
            LineIterator oeIterator = null;
            try {
                oeIterator = FileUtils.lineIterator(new File(inputFile));
                preScrubberReportData = laborPreScrubberService.preprocessOriginEntries(oeIterator, outputFile);
            } catch (IOException e) {
                LOG.error("IO exception occurred during pre scrubbing.", e);
                throw new RuntimeException("IO exception occurred during pre scrubbing.", e);
            } finally {
                LineIterator.closeQuietly(oeIterator);
            }
            if (preScrubberReportData != null) {
                ((WrappingBatchService) laborPreScrubberReportWriterService).initialize();
                new PreScrubberReport().generateReport(preScrubberReportData,
                        laborPreScrubberReportWriterService);
                ((WrappingBatchService) laborPreScrubberReportWriterService).destroy();
            }

            stopWatch.stop();
            if (LOG.isDebugEnabled()) {
                LOG.debug("labor pre-scrubber scrubber step took " + (stopWatch.getTotalTimeSeconds() / 60.0)
                        + " minutes to complete");
            }
            return true;
        }

    };
}

From source file:org.kuali.kfs.module.ld.batch.service.impl.LaborScrubberProcess.java

/**
 * Scrub this single group read only. This will only output the scrubber report. It won't output any other groups.
 *
 * @param group//from   w  w w .j  a  va2s .  c o  m
 */
public void scrubGroupReportOnly(String fileName, String documentNumber) {
    String unsortedFile = fileName;
    this.inputFile = fileName + ".sort";
    this.validFile = batchFileDirectoryName + File.separator
            + LaborConstants.BatchFileSystem.SCRUBBER_VALID_OUTPUT_FILE
            + GeneralLedgerConstants.BatchFileSystem.EXTENSION;
    this.errorFile = batchFileDirectoryName + File.separator
            + LaborConstants.BatchFileSystem.SCRUBBER_ERROR_OUTPUT_FILE
            + GeneralLedgerConstants.BatchFileSystem.EXTENSION;
    this.expiredFile = batchFileDirectoryName + File.separator
            + LaborConstants.BatchFileSystem.SCRUBBER_EXPIRED_OUTPUT_FILE
            + GeneralLedgerConstants.BatchFileSystem.EXTENSION;
    String prescrubOutput = batchFileDirectoryName + File.separator
            + LaborConstants.BatchFileSystem.PRE_SCRUBBER_FILE
            + GeneralLedgerConstants.BatchFileSystem.EXTENSION;

    PreScrubberReportData preScrubberReportData = null;
    // run pre-scrubber on the raw input into the sort process
    LineIterator inputEntries = null;
    try {
        inputEntries = FileUtils.lineIterator(new File(unsortedFile));
        preScrubberReportData = laborPreScrubberService.preprocessOriginEntries(inputEntries, prescrubOutput);
    } catch (IOException e1) {
        LOG.error("Error encountered trying to prescrub GLCP/LLCP document", e1);
        throw new RuntimeException("Error encountered trying to prescrub GLCP/LLCP document", e1);
    } finally {
        LineIterator.closeQuietly(inputEntries);
    }
    if (preScrubberReportData != null) {
        laborPreScrubberReportWriterService.setDocumentNumber(documentNumber);
        ((WrappingBatchService) laborPreScrubberReportWriterService).initialize();
        try {
            new PreScrubberReport().generateReport(preScrubberReportData, laborPreScrubberReportWriterService);
        } finally {
            ((WrappingBatchService) laborPreScrubberReportWriterService).destroy();
        }
    }
    BatchSortUtil.sortTextFileWithFields(prescrubOutput, inputFile, new LaborScrubberSortComparator());

    scrubEntries(true, documentNumber);

    File deleteSortFile = new File(inputFile);
    File deleteValidFile = new File(validFile);
    File deleteErrorFile = new File(errorFile);
    File deleteExpiredFile = new File(expiredFile);
    try {
        deleteSortFile.delete();
        deleteValidFile.delete();
        deleteErrorFile.delete();
        deleteExpiredFile.delete();
    } catch (Exception e) {
        LOG.error("scrubGroupReportOnly delete output files process Stopped: " + e.getMessage());
        throw new RuntimeException(
                "scrubGroupReportOnly delete output files process Stopped: " + e.getMessage(), e);
    }
}

From source file:org.kuali.rice.krad.demo.uif.components.ComponentLibraryView.java

/**
 * Process xml source code to be consumed by the exhibit component
 *
 * @param sourceCode list of sourceCode to be filled in, in order the group exhibit examples appear
 *//*from  ww  w .  j a  v  a2 s  .c o m*/
private void processXmlSource(List<String> sourceCode) {
    Map<String, String> idSourceMap = new HashMap<String, String>();
    if (xmlFilePath != null) {
        try {
            //Get the source file
            URL fileUrl = ComponentLibraryView.class.getClassLoader().getResource(xmlFilePath);
            File file = new File(fileUrl.toURI());
            Pattern examplePattern = Pattern.compile("ex:(.*?)(\\s|(-->))");

            boolean readingSource = false;
            String currentSource = "";
            String currentId = "";

            LineIterator lineIt = FileUtils.lineIterator(file);
            while (lineIt.hasNext()) {
                String line = lineIt.next();
                if (line.contains("ex:") && !readingSource) {
                    //found a ex: tag and are not already reading source
                    readingSource = true;

                    Matcher matcher = examplePattern.matcher(line);
                    if (matcher.find()) {
                        currentId = matcher.group(1);
                    }

                    currentSource = idSourceMap.get(currentId) != null ? idSourceMap.get(currentId) : "";

                    if (!currentSource.isEmpty()) {
                        currentSource = currentSource + "\n";
                    }
                } else if (line.contains("ex:") && readingSource) {
                    //stop reading source on second ex tag
                    readingSource = false;
                    idSourceMap.put(currentId, currentSource);
                } else if (readingSource) {
                    //when reading source just continue to add it
                    currentSource = currentSource + line + "\n";
                }

            }
        } catch (Exception e) {
            throw new RuntimeException(
                    "file not found or error while reading: " + xmlFilePath + " for source reading", e);
        }
    }

    for (Group demoGroup : demoGroups) {
        //add source to the source list by order that demo groups appear
        String groupId = demoGroup.getId();
        String source = idSourceMap.get(groupId);
        if (source != null) {
            //translate the source to something that can be displayed
            sourceCode.add(translateSource(source));
        }
    }
}

From source file:org.movsim.MovsimCoreMainWithExperiments.java

private static void appendFiles(File src, File dst, boolean writeFirstLine) throws IOException {

    LineIterator lIter = FileUtils.lineIterator(src);
    RandomAccessFile rFile = new RandomAccessFile(dst, "rw");

    rFile.seek(dst.length());//www  . ja  v a2  s. c o m

    long lineCount = 1;
    while (lIter.hasNext()) {
        String line = lIter.next();

        if (lineCount > 1 || writeFirstLine) {
            rFile.write((line + "\n").getBytes());
        }
        lineCount++;
    }
    lIter.close();
    rFile.close();
}

From source file:org.opennms.upgrade.implementations.JettyConfigMigratorOffline.java

@Override
public void execute() throws OnmsUpgradeException {
    String jettySSL = getMainProperties().getProperty("org.opennms.netmgt.jetty.https-port", null);
    String jettyAJP = getMainProperties().getProperty("org.opennms.netmgt.jetty.ajp-port", null);
    boolean sslWasFixed = false;
    boolean ajpWasFixed = false;
    try {/*from w w  w.j  a  v a  2s  . c o m*/
        log("SSL Enabled ? %s\n", jettySSL != null);
        log("AJP Enabled ? %s\n", jettyAJP != null);
        if (jettySSL != null || jettyAJP != null) {
            File jettyXmlExample = new File(getHomeDirectory(),
                    "etc" + File.separator + "examples" + File.separator + "jetty.xml");
            File jettyXml = new File(getHomeDirectory(), "etc" + File.separator + "jetty.xml");

            if (!jettyXml.exists() && !jettyXmlExample.exists()) {
                throw new FileNotFoundException("The required file doesn't exist: " + jettyXmlExample);
            }

            if (!jettyXml.exists()) {
                log("Copying %s into %s\n", jettyXmlExample, jettyXml);
                FileUtils.copyFile(jettyXmlExample, jettyXml);
            }

            log("Creating %s\n", jettyXml);
            File tempFile = new File(jettyXml.getAbsoluteFile() + ".tmp");
            FileWriter w = new FileWriter(tempFile);
            LineIterator it = FileUtils.lineIterator(jettyXmlExample);

            boolean startSsl = false;
            boolean startAjp = false;
            while (it.hasNext()) {
                String line = it.next();
                if (startAjp) {
                    if (line.matches("^\\s+[<][!]--\\s*$")) {
                        continue;
                    }
                    if (line.matches("^\\s+--[>]\\s*$")) {
                        startAjp = false;
                        ajpWasFixed = true;
                        continue;
                    }
                }
                if (startSsl) {
                    if (line.matches("^\\s+[<][!]--\\s*$")) {
                        continue;
                    }
                    if (line.matches("^\\s+--[>]\\s*$")) {
                        startSsl = false;
                        sslWasFixed = true;
                        continue;
                    }
                }
                w.write(line + "\n");
                if (startAjp == false && line.contains("<!-- Add AJP support -->") && jettyAJP != null) {
                    startAjp = true;
                    log("Enabling AjpConnector\n");
                }
                if (startSsl == false && line.contains("<!-- Add HTTPS support -->") && jettySSL != null) {
                    startSsl = true;
                    log("Enabling SslSelectChannelConnector\n");
                }
            }
            LineIterator.closeQuietly(it);
            w.close();
            FileUtils.copyFile(tempFile, jettyXml);
            FileUtils.deleteQuietly(tempFile);
        } else {
            log("Neither SSL nor AJP are enabled.\n");
        }
    } catch (Exception e) {
        throw new OnmsUpgradeException("Can't fix Jetty configuration because " + e.getMessage(), e);
    }
    if (jettyAJP != null && !ajpWasFixed) {
        throw new OnmsUpgradeException(
                "Can't enable APJ, please manually edit jetty.xml and uncomment the section where org.eclipse.jetty.ajp.Ajp13SocketConnector is defined.");
    }
    if (jettySSL != null && !sslWasFixed) {
        throw new OnmsUpgradeException(
                "Can't enable SSL, please manually edit jetty.xml and uncomment the section where org.eclipse.jetty.server.ssl.SslSelectChannelConnector is defined.");
    }
}