List of usage examples for org.apache.commons.io LineIterator next
public Object next()
Reader. From source file:org.apache.nifi.processors.csv.ParseCSVRecord.java
@Override public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { final FlowFile original = session.get(); if (original == null) { return;/*from ww w. ja v a 2s.c o m*/ } final AtomicBoolean lineFound = new AtomicBoolean(false); final Map<String, String> outputAttrs = new HashMap<>(); session.read(original, new InputStreamCallback() { @Override public void process(InputStream inputStream) throws IOException { final String fromAttribute = context.getProperty(PROP_RECORD_FROM_ATTRIBUTE).getValue(); String unparsedRecord; // data source is the attribute if (StringUtils.isNotBlank(fromAttribute)) { unparsedRecord = original.getAttribute(fromAttribute); if (StringUtils.isBlank(unparsedRecord)) { // will be routed to failure at the end of the method implementation return; } } else { // data source is the content // TODO expose the charset property? LineIterator iterator = IOUtils.lineIterator(inputStream, UTF_8); if (!iterator.hasNext()) { return; } unparsedRecord = iterator.next(); } lineFound.set(true); final String format = context.getProperty(PROP_FORMAT).getValue(); final String delimiter = context.getProperty(PROP_DELIMITER).evaluateAttributeExpressions(original) .getValue(); final String schemaPrefix = context.getProperty(PROP_SCHEMA_ATTR_PREFIX) .evaluateAttributeExpressions(original).getValue(); final String valuePrefix = context.getProperty(PROP_VALUE_ATTR_PREFIX) .evaluateAttributeExpressions(original).getValue(); final boolean trimValues = context.getProperty(PROP_TRIM_VALUES).asBoolean(); final CSVFormat csvFormat = buildFormat(format, delimiter, false, // this is a payload, not header anymore null); // no custom header final CSVParser parser = csvFormat.parse(new StringReader(unparsedRecord)); List<CSVRecord> records = parser.getRecords(); if (records.size() > 1) { // TODO revisit for NiFi's native micro-batching throw new ProcessException("Multi-line entries not supported"); } CSVRecord record = records.get(0); Map<String, String> originalAttrs = original.getAttributes(); // filter delimited schema attributes only Map<String, String> schemaAttrs = new HashMap<>(); for (String key : originalAttrs.keySet()) { if (key.startsWith(schemaPrefix)) { schemaAttrs.put(key, originalAttrs.get(key)); } } // put key/value pairs into attributes for (int i = 0; i < record.size(); i++) { String columnName = schemaAttrs.get(schemaPrefix + (i + 1)); // 1-based column numbering if (columnName == null) { // 1-based column index columnName = String.valueOf(i + 1); } // TODO indexed schemaless parsing vs auto-schema vs user-provided schema String columnValue = record.get(i); if (trimValues) { columnValue = columnValue.trim(); } String attrName = (StringUtils.isBlank(valuePrefix) ? "delimited.column." : valuePrefix) + columnName; outputAttrs.put(attrName, columnValue); } } }); if (lineFound.get()) { FlowFile ff = session.putAllAttributes(original, outputAttrs); session.transfer(ff, REL_SUCCESS); } else { session.transfer(original, REL_FAILURE); } }
From source file:org.asqatasun.referential.creator.CodeGeneratorMojo.java
/** * * @return//w w w . jav a2 s .c o m */ private Iterable<CSVRecord> getCsv() { // we parse the csv file to extract the first line and get the headers LineIterator lineIterator; try { lineIterator = FileUtils.lineIterator(dataFile, Charset.defaultCharset().name()); } catch (IOException ex) { Logger.getLogger(CodeGeneratorMojo.class.getName()).log(Level.SEVERE, null, ex); lineIterator = null; } String[] csvHeaders = lineIterator != null ? lineIterator.next().split(String.valueOf(delimiter)) : new String[0]; isCriterionPresent = extractCriterionFromCsvHeader(csvHeaders); try { extractAvailableLangsFromCsvHeader(csvHeaders); } catch (I18NLanguageNotFoundException ex) { Logger.getLogger(CodeGeneratorMojo.class.getName()).log(Level.SEVERE, null, ex); return null; } // from here we just add each line to a build to re-create the csv content // without the first line. StringBuilder strb = new StringBuilder(); while (lineIterator.hasNext()) { strb.append(lineIterator.next()); strb.append("\n"); } Reader in; try { in = new StringReader(strb.toString()); CSVFormat csvf = CSVFormat.newFormat(delimiter).withHeader(csvHeaders); return csvf.parse(in); } catch (FileNotFoundException ex) { Logger.getLogger(CodeGeneratorMojo.class.getName()).log(Level.SEVERE, null, ex); return null; } catch (IOException ex) { Logger.getLogger(CodeGeneratorMojo.class.getName()).log(Level.SEVERE, null, ex); return null; } }
From source file:org.asqatasun.rules.doc.utils.exportaw22torgaa3ruledesign.ExtractCsvAndCopy.java
public Iterable<CSVRecord> getCsv() throws IOException { // we parse the csv file to extract the first line and get the headers LineIterator lineIterator; lineIterator = FileUtils.lineIterator(DATA_FILE); csvHeaders = lineIterator.next().split(String.valueOf(DELIMITER)); StringBuilder strb = new StringBuilder(); while (lineIterator.hasNext()) { strb.append(lineIterator.next()); strb.append("\n"); }/*from w ww.j a v a 2 s. com*/ Reader in; try { in = new StringReader(strb.toString()); CSVFormat csvf = CSVFormat.newFormat(DELIMITER).withHeader(csvHeaders); return csvf.parse(in); } catch (FileNotFoundException ex) { return null; } catch (IOException ex) { return null; } }
From source file:org.bdval.WriteModel.java
@Override public void interpretArguments(final JSAP jsap, final JSAPResult result, final DAVOptions options) { super.interpretArguments(jsap, result, options); modelPrefix = result.contains("model-prefix") ? result.getString("model-prefix") : null; final Properties extraParameters = new Properties(); if (result.contains("use-parameters")) { final String filename = result.getString("use-parameters"); try {//w w w . ja v a2s. co m extraParameters.load(filename); final ObjectList<String> allParameters = new ObjectArrayList<String>(options.classifierParameters); final Iterator<String> parameterKeyIt = extraParameters.getKeys(); while (parameterKeyIt.hasNext()) { final String parameterName = parameterKeyIt.next(); final double value = extraParameters.getDouble(parameterName); final String added = parameterName + "=" + Double.toString(value); allParameters.add(added); if (LOG.isDebugEnabled()) { LOG.debug("Adding parameter from property file: " + added); } } options.classifierParameters = allParameters.toArray(new String[allParameters.size()]); } catch (ConfigurationException e) { LOG.error("Cannot load model parameters from file " + filename); } } if (result.contains("consensus-of-models")) { final String fileName = result.getString("consensus-of-models"); Reader reader = null; try { final ArrayList<String> list = new ArrayList<String>(); reader = new FileReader(fileName); final LineIterator it = new LineIterator(reader); while (it.hasNext()) { final String line = (String) it.next(); list.add(line); } modelComponentPrefixes = list.toArray(new String[list.size()]); } catch (FileNotFoundException e) { LOG.error("Cannot find list of model components for consensus model: " + fileName); } finally { IOUtils.closeQuietly(reader); } } }
From source file:org.carewebframework.maven.plugin.core.ConfigTemplate.java
/** * Creates a config file template from a classpath resource. * /* www. jav a2 s. c om*/ * @param filename Path to config file template. * @throws MojoExecutionException Error loading the template. */ ConfigTemplate(String filename) throws MojoExecutionException { this.filename = filename; InputStream in = getClass().getResourceAsStream("/" + filename); if (in == null) { throw new MojoExecutionException("Cannot find config file template: " + filename); } try { LineIterator lines = IOUtils.lineIterator(in, "UTF-8"); while (lines.hasNext()) { String line = lines.next(); if (line.startsWith("*")) { String[] pcs = line.split("\\*", 3); entries.put(pcs[1], new ConfigEntry(pcs[2], buffer.size())); buffer.add(""); } else { buffer.add(line); } } } catch (Exception e) { throw new MojoExecutionException("Unexpected error while creating configuration file.", e); } finally { IOUtils.closeQuietly(in); } if (entries.isEmpty()) { throw new MojoExecutionException("Failed to locate insertion point in configuration file."); } }
From source file:org.cleverbus.admin.services.log.LogEventParsingIterator.java
/** * Advance to the position where next line is available, * regardless of whether it passes filter requirements or not. * On the way to this position any lines that are not log events * will be appended to the previous log event. * <p/>//from w ww. jav a2s. co m * If this call is successful, either {@link #parsedEvent} will be a new event, * or {@link #preParsedEvent} will not be null; or both. * * @return true, if there might be more events; false otherwise * @throws IOException if there's a problem opening a new file */ private boolean seekToNextEvent() throws IOException { LogEvent nextEvent = config.createLogEvent(); LineIterator iterator; while ((iterator = getLineIterator()) != null) { while (iterator.hasNext()) { // process the line: LogEvent event = parser.parseLine(iterator.next(), nextEvent, preParsedEvent, config); // check what to do next: if (preParsedEvent != null && event != preParsedEvent) { // there is pre-parsed event, but the line was NOT appended to it // => pre-parsed event will not get any more lines, it can be considered fully parsed parsedEvent = preParsedEvent; // graduate pre-parsed event - it'll be the next event preParsedEvent = event; // next event that was found (if any) is now pre-parsed return true; // found full pre-parsed event => success } else if (event == nextEvent) { // nextEvent is now pre-parsed, but previous pre-parsed event is null, so no event graduated preParsedEvent = nextEvent; return true; } // otherwise line was ignored or added to the pre-parsed event, nothing really changed } } return false; // failure }
From source file:org.cloudbyexample.dc.agent.command.ImageClientImpl.java
private String convertInputStreamToString(InputStream response) { StringWriter logwriter = new StringWriter(); try {/*from w w w. j a v a 2 s . c o m*/ LineIterator itr = IOUtils.lineIterator(response, "UTF-8"); while (itr.hasNext()) { String line = itr.next(); logwriter.write(line + (itr.hasNext() ? "\n" : "")); logger.info(line); } return logwriter.toString(); } catch (IOException e) { throw new RuntimeException(e); } finally { IOUtils.closeQuietly(response); } }
From source file:org.cloudml.connectors.DockerConnector.java
public void BuildImageFromDockerFile(String path) { File baseDir = new File(path); InputStream response = dockerClient.buildImageCmd(baseDir).exec(); StringWriter logwriter = new StringWriter(); try {//w w w.j a va 2 s . c om LineIterator itr = IOUtils.lineIterator(response, "UTF-8"); while (itr.hasNext()) { String line = itr.next(); logwriter.write(line); journal.log(Level.INFO, line); } } catch (IOException e) { e.printStackTrace(); } finally { IOUtils.closeQuietly(response); } }
From source file:org.commonjava.freeki.data.FreekiStore.java
private Map<String, String> getMetadata(final String content, final boolean reformatContent) { final LineIterator li = new LineIterator(new StringReader(content)); final Map<String, String> metadata = new HashMap<>(); final StringBuilder sb = new StringBuilder(); boolean startMetadata = false; boolean stopMetadata = false; boolean startContent = false; while (li.hasNext()) { final String line = li.next(); if (line.trim().startsWith(COMMENT_BEGIN)) { startMetadata = true;//from ww w. j av a2s . c o m } else if (line.trim().endsWith(COMMENT_END)) { stopMetadata = true; } else if (startMetadata && !stopMetadata) { final String[] parts = line.trim().split("\\s*:\\s*"); if (parts.length == 2) { metadata.put(parts[0], parts[1]); } } else if (stopMetadata && !startContent && line.trim().length() > 0) { startContent = true; } if (startContent) { String title = metadata.get(MD_TITLE); if (title == null && line.startsWith(H1)) { title = line.substring(H1.length()); metadata.put(MD_TITLE, title); } else if (title == null && line.startsWith(H1_ALT)) { title = line.substring(H1_ALT.length()); if (!title.startsWith("#")) { metadata.put(MD_TITLE, title); } } if (!reformatContent && title != null) { break; } if (sb.length() > 0) { sb.append('\n'); } sb.append(line); } } if (sb.length() > 0) { metadata.put(MD_CONTENT, sb.toString()); } else { metadata.put(MD_CONTENT, content); } return metadata; }
From source file:org.commonjava.freeki.model.Page.java
public void syncContentWithTitle() { String title = getTitle();/*from w w w . ja v a 2s. co m*/ if (content != null) { boolean prependTitle = false; final LineIterator li = new LineIterator(new StringReader(content)); while (li.hasNext()) { final String line = li.next(); if (line.trim().length() > 0) { if (title == null) { title = line; } else if (title != null) { if (!line.trim().endsWith(title.trim())) { prependTitle = true; } } break; } } if (prependTitle) { content = "#" + title + "\n\n" + content; } } else if (title != null) { content = "#" + title + "\n\n"; } setTitle(title); }