List of usage examples for org.apache.commons.io LineIterator closeQuietly
public static void closeQuietly(LineIterator iterator)
From source file:org.kuali.kfs.gl.batch.service.impl.ScrubberProcessImpl.java
/** * Scrub this single group read only. This will only output the scrubber report. It won't output any other groups. * * @param group the origin entry group that should be scrubbed * @param the document number of any specific entries to scrub *//* w w w. j a va 2 s. c o m*/ @Override public void scrubGroupReportOnly(String fileName, String documentNumber) { LOG.debug("scrubGroupReportOnly() started"); String unsortedFile = fileName; this.inputFile = fileName + ".sort"; this.validFile = batchFileDirectoryName + File.separator + GeneralLedgerConstants.BatchFileSystem.SCRUBBER_VALID_OUTPUT_FILE + GeneralLedgerConstants.BatchFileSystem.EXTENSION; this.errorFile = batchFileDirectoryName + File.separator + GeneralLedgerConstants.BatchFileSystem.SCRUBBER_ERROR_OUTPUT_FILE + GeneralLedgerConstants.BatchFileSystem.EXTENSION; this.expiredFile = batchFileDirectoryName + File.separator + GeneralLedgerConstants.BatchFileSystem.SCRUBBER_EXPIRED_OUTPUT_FILE + GeneralLedgerConstants.BatchFileSystem.EXTENSION; String prescrubOutput = batchFileDirectoryName + File.separator + GeneralLedgerConstants.BatchFileSystem.PRE_SCRUBBER_FILE + GeneralLedgerConstants.BatchFileSystem.EXTENSION; this.ledgerSummaryReport = new LedgerSummaryReport(); runDate = calculateRunDate(dateTimeService.getCurrentDate()); PreScrubberReportData preScrubberReportData = null; // run pre-scrubber on the raw input into the sort process LineIterator inputEntries = null; try { inputEntries = FileUtils.lineIterator(new File(unsortedFile)); preScrubberReportData = preScrubberService.preprocessOriginEntries(inputEntries, prescrubOutput); } catch (IOException e1) { LOG.error("Error encountered trying to prescrub GLCP/LLCP document", e1); throw new RuntimeException("Error encountered trying to prescrub GLCP/LLCP document", e1); } finally { LineIterator.closeQuietly(inputEntries); } if (preScrubberReportData != null) { preScrubberReportWriterService.setDocumentNumber(documentNumber); ((WrappingBatchService) preScrubberReportWriterService).initialize(); try { new PreScrubberReport().generateReport(preScrubberReportData, preScrubberReportWriterService); } finally { ((WrappingBatchService) preScrubberReportWriterService).destroy(); } } BatchSortUtil.sortTextFileWithFields(prescrubOutput, inputFile, new ScrubberSortComparator()); scrubEntries(true, documentNumber); // delete files File deleteSortFile = new File(inputFile); File deleteValidFile = new File(validFile); File deleteErrorFile = new File(errorFile); File deleteExpiredFile = new File(expiredFile); try { deleteSortFile.delete(); deleteValidFile.delete(); deleteErrorFile.delete(); deleteExpiredFile.delete(); } catch (Exception e) { LOG.error("scrubGroupReportOnly delete output files process Stopped: " + e.getMessage()); throw new RuntimeException( "scrubGroupReportOnly delete output files process Stopped: " + e.getMessage(), e); } }
From source file:org.kuali.kfs.module.ld.batch.LaborPreScrubberStep.java
/** * @see org.kuali.kfs.sys.batch.AbstractWrappedBatchStep#getCustomBatchExecutor() *///from w w w . ja v a2s . c o m @Override protected CustomBatchExecutor getCustomBatchExecutor() { return new CustomBatchExecutor() { /** * @see org.kuali.kfs.sys.batch.service.WrappedBatchExecutorService.CustomBatchExecutor#execute() */ public boolean execute() { StopWatch stopWatch = new StopWatch(); stopWatch.start(); String inputFile = batchFileDirectoryName + File.separator + LaborConstants.BatchFileSystem.BACKUP_FILE + GeneralLedgerConstants.BatchFileSystem.EXTENSION; String outputFile = batchFileDirectoryName + File.separator + LaborConstants.BatchFileSystem.PRE_SCRUBBER_FILE + GeneralLedgerConstants.BatchFileSystem.EXTENSION; PreScrubberReportData preScrubberReportData = null; LineIterator oeIterator = null; try { oeIterator = FileUtils.lineIterator(new File(inputFile)); preScrubberReportData = laborPreScrubberService.preprocessOriginEntries(oeIterator, outputFile); } catch (IOException e) { LOG.error("IO exception occurred during pre scrubbing.", e); throw new RuntimeException("IO exception occurred during pre scrubbing.", e); } finally { LineIterator.closeQuietly(oeIterator); } if (preScrubberReportData != null) { ((WrappingBatchService) laborPreScrubberReportWriterService).initialize(); new PreScrubberReport().generateReport(preScrubberReportData, laborPreScrubberReportWriterService); ((WrappingBatchService) laborPreScrubberReportWriterService).destroy(); } stopWatch.stop(); if (LOG.isDebugEnabled()) { LOG.debug("labor pre-scrubber scrubber step took " + (stopWatch.getTotalTimeSeconds() / 60.0) + " minutes to complete"); } return true; } }; }
From source file:org.kuali.kfs.module.ld.batch.service.impl.LaborScrubberProcess.java
/** * Scrub this single group read only. This will only output the scrubber report. It won't output any other groups. * * @param group//from w w w .ja v a 2 s . co m */ public void scrubGroupReportOnly(String fileName, String documentNumber) { String unsortedFile = fileName; this.inputFile = fileName + ".sort"; this.validFile = batchFileDirectoryName + File.separator + LaborConstants.BatchFileSystem.SCRUBBER_VALID_OUTPUT_FILE + GeneralLedgerConstants.BatchFileSystem.EXTENSION; this.errorFile = batchFileDirectoryName + File.separator + LaborConstants.BatchFileSystem.SCRUBBER_ERROR_OUTPUT_FILE + GeneralLedgerConstants.BatchFileSystem.EXTENSION; this.expiredFile = batchFileDirectoryName + File.separator + LaborConstants.BatchFileSystem.SCRUBBER_EXPIRED_OUTPUT_FILE + GeneralLedgerConstants.BatchFileSystem.EXTENSION; String prescrubOutput = batchFileDirectoryName + File.separator + LaborConstants.BatchFileSystem.PRE_SCRUBBER_FILE + GeneralLedgerConstants.BatchFileSystem.EXTENSION; PreScrubberReportData preScrubberReportData = null; // run pre-scrubber on the raw input into the sort process LineIterator inputEntries = null; try { inputEntries = FileUtils.lineIterator(new File(unsortedFile)); preScrubberReportData = laborPreScrubberService.preprocessOriginEntries(inputEntries, prescrubOutput); } catch (IOException e1) { LOG.error("Error encountered trying to prescrub GLCP/LLCP document", e1); throw new RuntimeException("Error encountered trying to prescrub GLCP/LLCP document", e1); } finally { LineIterator.closeQuietly(inputEntries); } if (preScrubberReportData != null) { laborPreScrubberReportWriterService.setDocumentNumber(documentNumber); ((WrappingBatchService) laborPreScrubberReportWriterService).initialize(); try { new PreScrubberReport().generateReport(preScrubberReportData, laborPreScrubberReportWriterService); } finally { ((WrappingBatchService) laborPreScrubberReportWriterService).destroy(); } } BatchSortUtil.sortTextFileWithFields(prescrubOutput, inputFile, new LaborScrubberSortComparator()); scrubEntries(true, documentNumber); File deleteSortFile = new File(inputFile); File deleteValidFile = new File(validFile); File deleteErrorFile = new File(errorFile); File deleteExpiredFile = new File(expiredFile); try { deleteSortFile.delete(); deleteValidFile.delete(); deleteErrorFile.delete(); deleteExpiredFile.delete(); } catch (Exception e) { LOG.error("scrubGroupReportOnly delete output files process Stopped: " + e.getMessage()); throw new RuntimeException( "scrubGroupReportOnly delete output files process Stopped: " + e.getMessage(), e); } }
From source file:org.mskcc.cbio.importer.io.internal.FileUtilsImpl.java
/** * Reads the precomputed md5 digest out of a .md5 file (firehose). * Assume the file only contains one line wit checksum. * * @param file File/*w w w. j a v a 2 s . co m*/ * @return String * @throws Exception */ @Override public String getPrecomputedMD5Digest(File file) throws Exception { if (LOG.isInfoEnabled()) { LOG.info("getPrecomputedMD5Digest(): " + file.getCanonicalPath()); } String toReturn = ""; LineIterator it = org.apache.commons.io.FileUtils.lineIterator(file); try { while (it.hasNext()) { String content = it.nextLine(); if (content.split(" ").length == 2) { toReturn = content.split(" ")[0].toUpperCase(); } } } finally { LineIterator.closeQuietly(it); } // outta here return toReturn; }
From source file:org.mskcc.cbio.importer.io.internal.FileUtilsImpl.java
/** * Helper function to create DataMatrix. * * @param data InputStream//from w w w. jav a 2 s . co m * @return DataMatrix */ private DataMatrix getDataMatrix(InputStream data) throws Exception { // iterate over all lines in byte[] List<String> columnNames = null; List<LinkedList<String>> rowData = null; LineIterator it = IOUtils.lineIterator(data, null); try { int count = -1; while (it.hasNext()) { // first row is our column heading, create column vector if (++count == 0) { columnNames = new LinkedList(Arrays.asList(it.nextLine().split(Converter.VALUE_DELIMITER, -1))); } // all other rows are rows in the table else { rowData = (rowData == null) ? new LinkedList<LinkedList<String>>() : rowData; rowData.add(new LinkedList(Arrays.asList(it.nextLine().split(Converter.VALUE_DELIMITER, -1)))); } } } finally { LineIterator.closeQuietly(it); } // problem reading from data? if (columnNames == null || rowData == null) { if (LOG.isInfoEnabled()) { LOG.info( "getDataMatrix(), problem creating DataMatrix from file, data file probably missing data, returning null"); } return null; } // made it here, we can create DataMatrix if (LOG.isInfoEnabled()) { LOG.info("creating new DataMatrix(), from file data"); } // outta here return new DataMatrix(rowData, columnNames); }
From source file:org.nd4j.linalg.factory.Nd4j.java
/** * Read line via input streams//from ww w .j a v a 2 s . c o m * * @param ndarray the input stream ndarray * @param sep character, defaults to "," * @return NDArray */ public static INDArray readTxtString(InputStream ndarray, String sep) { /* We could dump an ndarray to a file with the tostring (since that is valid json) and use put/get to parse it as json But here we leverage our information of the tostring method to be more efficient With our current toString format we use tads along dimension (rank-1,rank-2) to write to the array in two dimensional chunks at a time. This is more efficient than setting each value at a time with putScalar. This also means we can read the file one line at a time instead of loading the whole thing into memory Future work involves enhancing the write json method to provide more features to make the load more efficient */ int lineNum = 0; int rowNum = 0; int tensorNum = 0; char theOrder = 'c'; int[] theShape = { 1, 1 }; int rank = 0; double[][] subsetArr = { { 0.0, 0.0 }, { 0.0, 0.0 } }; INDArray newArr = Nd4j.zeros(2, 2); BufferedReader reader = new BufferedReader(new InputStreamReader(ndarray)); LineIterator it = IOUtils.lineIterator(reader); DecimalFormat format = (DecimalFormat) NumberFormat.getInstance(Locale.US); format.setParseBigDecimal(true); try { while (it.hasNext()) { String line = it.nextLine(); lineNum++; line = line.replaceAll("\\s", ""); if (line.equals("") || line.equals("}")) continue; // is it from dl4j? if (lineNum == 2) { String[] lineArr = line.split(":"); String fileSource = lineArr[1].replaceAll("\\W", ""); if (!fileSource.equals("dl4j")) return null; } // parse ordering if (lineNum == 3) { String[] lineArr = line.split(":"); theOrder = lineArr[1].replaceAll("\\W", "").charAt(0); continue; } // parse shape if (lineNum == 4) { String[] lineArr = line.split(":"); String dropJsonComma = lineArr[1].split("]")[0]; String[] shapeString = dropJsonComma.replace("[", "").split(","); rank = shapeString.length; theShape = new int[rank]; for (int i = 0; i < rank; i++) { try { theShape[i] = Integer.parseInt(shapeString[i]); } catch (NumberFormatException nfe) { } ; } subsetArr = new double[theShape[rank - 2]][theShape[rank - 1]]; newArr = Nd4j.zeros(theShape, theOrder); continue; } //parse data if (lineNum > 5) { String[] entries = line.replace("\\],", "").replaceAll("\\[", "").replaceAll("\\],", "") .replaceAll("\\]", "").split(sep); for (int i = 0; i < theShape[rank - 1]; i++) { try { BigDecimal number = (BigDecimal) format.parse(entries[i]); subsetArr[rowNum][i] = number.doubleValue(); } catch (ParseException e) { e.printStackTrace(); } } rowNum++; if (rowNum == theShape[rank - 2]) { INDArray subTensor = Nd4j.create(subsetArr); newArr.tensorAlongDimension(tensorNum, rank - 1, rank - 2).addi(subTensor); rowNum = 0; tensorNum++; } } } } finally { LineIterator.closeQuietly(it); } return newArr; }
From source file:org.objectstyle.woproject.maven2.wobootstrap.utils.WebObjectsUtils.java
/** * Retrieves the version of installed WebObjects. It uses a * <code>WebObjectsLocator</code> to find the WebObjects version file. * /* w ww.j av a 2 s . co m*/ * @param locator * The WebObjects locator * @return Returns the WebObjects version or <code>null</code> if cannot * discover the WebObjects version */ public static String getWebObjectsVersion(WebObjectsLocator locator) { if (locator == null) { return null; } File versionFile = locator.getWebObjectsVersionFile(); if (versionFile == null || !versionFile.exists()) { return null; } String version = null; LineIterator iterator = null; try { iterator = FileUtils.lineIterator(versionFile, null); while (iterator.hasNext()) { String line = iterator.nextLine(); if ("<key>CFBundleShortVersionString</key>".equals(line.trim())) { String versionLine = iterator.nextLine(); version = versionLine.trim().replaceAll("</?string>", ""); break; } } } catch (IOException exception) { // TODO: hprange, write an info to log instead exception.printStackTrace(); } finally { if (iterator != null) { LineIterator.closeQuietly(iterator); } } return version; }
From source file:org.opennms.upgrade.implementations.JettyConfigMigratorOffline.java
@Override public void execute() throws OnmsUpgradeException { String jettySSL = getMainProperties().getProperty("org.opennms.netmgt.jetty.https-port", null); String jettyAJP = getMainProperties().getProperty("org.opennms.netmgt.jetty.ajp-port", null); boolean sslWasFixed = false; boolean ajpWasFixed = false; try {/* ww w. j a v a2 s . c o m*/ log("SSL Enabled ? %s\n", jettySSL != null); log("AJP Enabled ? %s\n", jettyAJP != null); if (jettySSL != null || jettyAJP != null) { File jettyXmlExample = new File(getHomeDirectory(), "etc" + File.separator + "examples" + File.separator + "jetty.xml"); File jettyXml = new File(getHomeDirectory(), "etc" + File.separator + "jetty.xml"); if (!jettyXml.exists() && !jettyXmlExample.exists()) { throw new FileNotFoundException("The required file doesn't exist: " + jettyXmlExample); } if (!jettyXml.exists()) { log("Copying %s into %s\n", jettyXmlExample, jettyXml); FileUtils.copyFile(jettyXmlExample, jettyXml); } log("Creating %s\n", jettyXml); File tempFile = new File(jettyXml.getAbsoluteFile() + ".tmp"); FileWriter w = new FileWriter(tempFile); LineIterator it = FileUtils.lineIterator(jettyXmlExample); boolean startSsl = false; boolean startAjp = false; while (it.hasNext()) { String line = it.next(); if (startAjp) { if (line.matches("^\\s+[<][!]--\\s*$")) { continue; } if (line.matches("^\\s+--[>]\\s*$")) { startAjp = false; ajpWasFixed = true; continue; } } if (startSsl) { if (line.matches("^\\s+[<][!]--\\s*$")) { continue; } if (line.matches("^\\s+--[>]\\s*$")) { startSsl = false; sslWasFixed = true; continue; } } w.write(line + "\n"); if (startAjp == false && line.contains("<!-- Add AJP support -->") && jettyAJP != null) { startAjp = true; log("Enabling AjpConnector\n"); } if (startSsl == false && line.contains("<!-- Add HTTPS support -->") && jettySSL != null) { startSsl = true; log("Enabling SslSelectChannelConnector\n"); } } LineIterator.closeQuietly(it); w.close(); FileUtils.copyFile(tempFile, jettyXml); FileUtils.deleteQuietly(tempFile); } else { log("Neither SSL nor AJP are enabled.\n"); } } catch (Exception e) { throw new OnmsUpgradeException("Can't fix Jetty configuration because " + e.getMessage(), e); } if (jettyAJP != null && !ajpWasFixed) { throw new OnmsUpgradeException( "Can't enable APJ, please manually edit jetty.xml and uncomment the section where org.eclipse.jetty.ajp.Ajp13SocketConnector is defined."); } if (jettySSL != null && !sslWasFixed) { throw new OnmsUpgradeException( "Can't enable SSL, please manually edit jetty.xml and uncomment the section where org.eclipse.jetty.server.ssl.SslSelectChannelConnector is defined."); } }
From source file:org.opennms.upgrade.implementations.JmxRrdMigratorOffline.java
/** * Fixes a JMX configuration file./* w w w. ja v a 2s . c om*/ * * @param jmxConfigFile the JMX configuration file * @throws OnmsUpgradeException the OpenNMS upgrade exception */ private void fixJmxConfigurationFile(File jmxConfigFile) throws OnmsUpgradeException { try { log("Updating JMX metric definitions on %s\n", jmxConfigFile); zipFile(jmxConfigFile); backupFiles.add(new File(jmxConfigFile.getAbsolutePath() + ZIP_EXT)); File outputFile = new File(jmxConfigFile.getCanonicalFile() + ".temp"); FileWriter w = new FileWriter(outputFile); Pattern extRegex = Pattern.compile("import-mbeans[>](.+)[<]"); Pattern aliasRegex = Pattern.compile("alias=\"([^\"]+\\.[^\"]+)\""); List<File> externalFiles = new ArrayList<File>(); LineIterator it = FileUtils.lineIterator(jmxConfigFile); while (it.hasNext()) { String line = it.next(); Matcher m = extRegex.matcher(line); if (m.find()) { externalFiles.add(new File(jmxConfigFile.getParentFile(), m.group(1))); } m = aliasRegex.matcher(line); if (m.find()) { String badDs = m.group(1); String fixedDs = getFixedDsName(badDs); log(" Replacing bad alias %s with %s on %s\n", badDs, fixedDs, line.trim()); line = line.replaceAll(badDs, fixedDs); if (badMetrics.contains(badDs) == false) { badMetrics.add(badDs); } } w.write(line + "\n"); } LineIterator.closeQuietly(it); w.close(); FileUtils.deleteQuietly(jmxConfigFile); FileUtils.moveFile(outputFile, jmxConfigFile); if (!externalFiles.isEmpty()) { for (File configFile : externalFiles) { fixJmxConfigurationFile(configFile); } } } catch (Exception e) { throw new OnmsUpgradeException("Can't fix " + jmxConfigFile + " because " + e.getMessage(), e); } }
From source file:org.opennms.upgrade.implementations.JmxRrdMigratorOffline.java
/** * Fixes a JMX graph template file./* w w w .j av a 2 s . c o m*/ * * @param jmxTemplateFile the JMX template file * @throws OnmsUpgradeException the OpenNMS upgrade exception */ private void fixJmxGraphTemplateFile(File jmxTemplateFile) throws OnmsUpgradeException { try { log("Updating JMX graph templates on %s\n", jmxTemplateFile); zipFile(jmxTemplateFile); backupFiles.add(new File(jmxTemplateFile.getAbsolutePath() + ZIP_EXT)); File outputFile = new File(jmxTemplateFile.getCanonicalFile() + ".temp"); FileWriter w = new FileWriter(outputFile); Pattern defRegex = Pattern.compile("DEF:.+:(.+\\..+):"); Pattern colRegex = Pattern.compile("\\.columns=(.+)$"); Pattern incRegex = Pattern.compile("^include.directory=(.+)$"); List<File> externalFiles = new ArrayList<File>(); boolean override = false; LineIterator it = FileUtils.lineIterator(jmxTemplateFile); while (it.hasNext()) { String line = it.next(); Matcher m = incRegex.matcher(line); if (m.find()) { File includeDirectory = new File(jmxTemplateFile.getParentFile(), m.group(1)); if (includeDirectory.isDirectory()) { FilenameFilter propertyFilesFilter = new FilenameFilter() { @Override public boolean accept(File dir, String name) { return (name.endsWith(".properties")); } }; for (File file : includeDirectory.listFiles(propertyFilesFilter)) { externalFiles.add(file); } } } m = colRegex.matcher(line); if (m.find()) { String[] badColumns = m.group(1).split(",(\\s)?"); for (String badDs : badColumns) { String fixedDs = getFixedDsName(badDs); if (fixedDs.equals(badDs)) { continue; } if (badMetrics.contains(badDs)) { override = true; log(" Replacing bad data source %s with %s on %s\n", badDs, fixedDs, line); line = line.replaceAll(badDs, fixedDs); } else { log(" Warning: a bad data source not related with JMX has been found: %s (this won't be updated)\n", badDs); } } } m = defRegex.matcher(line); if (m.find()) { String badDs = m.group(1); if (badMetrics.contains(badDs)) { override = true; String fixedDs = getFixedDsName(badDs); log(" Replacing bad data source %s with %s on %s\n", badDs, fixedDs, line); line = line.replaceAll(badDs, fixedDs); } else { log(" Warning: a bad data source not related with JMX has been found: %s (this won't be updated)\n", badDs); } } w.write(line + "\n"); } LineIterator.closeQuietly(it); w.close(); if (override) { FileUtils.deleteQuietly(jmxTemplateFile); FileUtils.moveFile(outputFile, jmxTemplateFile); } else { FileUtils.deleteQuietly(outputFile); } if (!externalFiles.isEmpty()) { for (File configFile : externalFiles) { fixJmxGraphTemplateFile(configFile); } } } catch (Exception e) { throw new OnmsUpgradeException("Can't fix " + jmxTemplateFile + " because " + e.getMessage(), e); } }