Example usage for org.apache.commons.io IOUtils lineIterator

List of usage examples for org.apache.commons.io IOUtils lineIterator

  1. HOME
  2. Java
  3. org.apache.commons
  4. org.apache.commons.io.*
  5. IOUtils
  6. lineIterator

Introduction

In this page you can find the example usage for org.apache.commons.io IOUtils lineIterator.

Prototype

public static LineIterator lineIterator(InputStream input, String encoding) throws IOException

Source Link

Document

Return an Iterator for the lines in an InputStream, using the character encoding specified (or default encoding if null).

Usage

From source file:org.apache.http.nio.client.methods.TestZeroCopy.java

@Test
public void testTwoWayZeroCopy() throws Exception {
    this.serverBootstrap.registerHandler("*", new BasicAsyncRequestHandler(new TestHandler(false)));
    final HttpHost target = start();

    final File tmpdir = FileUtils.getTempDirectory();
    this.tmpfile = new File(tmpdir, "dst.test");
    final TestZeroCopyPost httppost = new TestZeroCopyPost(target.toURI() + "/bounce", false);
    final TestZeroCopyConsumer consumer = new TestZeroCopyConsumer(this.tmpfile);
    final Future<Integer> future = this.httpclient.execute(httppost, consumer, null);
    final Integer status = future.get();
    Assert.assertNotNull(status);/* w w w .  ja v  a  2  s  .  c o m*/
    Assert.assertEquals(HttpStatus.SC_OK, status.intValue());
    final InputStream instream = new FileInputStream(this.tmpfile);
    try {
        final LineIterator it = IOUtils.lineIterator(instream, ASCII.name());
        int count = 0;
        while (it.hasNext()) {
            final String line = it.next();
            final int i = count % TEXT.length;
            final String expected = TEXT[i];
            Assert.assertEquals(expected, line);
            count++;
        }
    } finally {
        instream.close();
    }
}

From source file:org.apache.http.nio.client.methods.TestZeroCopy.java

@Test
public void testZeroCopyFallback() throws Exception {
    this.serverBootstrap.registerHandler("*", new BasicAsyncRequestHandler(new TestHandler(true)));
    final HttpHost target = start();
    final File tmpdir = FileUtils.getTempDirectory();
    this.tmpfile = new File(tmpdir, "dst.test");
    final TestZeroCopyPost httppost = new TestZeroCopyPost(target.toURI() + "/bounce", true);
    final TestZeroCopyConsumer consumer = new TestZeroCopyConsumer(this.tmpfile);
    final Future<Integer> future = this.httpclient.execute(httppost, consumer, null);
    final Integer status = future.get();
    Assert.assertNotNull(status);// w ww.  j  a  va2  s.c  o m
    Assert.assertEquals(HttpStatus.SC_OK, status.intValue());
    final InputStream instream = new FileInputStream(this.tmpfile);
    try {
        final LineIterator it = IOUtils.lineIterator(instream, ASCII.name());
        int count = 0;
        while (it.hasNext()) {
            final String line = it.next();
            final int i = count % TEXT.length;
            final String expected = TEXT[i];
            Assert.assertEquals(expected, line);
            count++;
        }
    } finally {
        instream.close();
    }
}

From source file:org.apache.ignite.internal.client.impl.ClientPropertiesConfigurationSelfTest.java

/**
 * Uncomment properties./*w  w  w  .ja v  a  2  s . c o m*/
 *
 * @param url Source to uncomment client properties for.
 * @return Temporary file with uncommented client properties.
 * @throws IOException In case of IO exception.
 */
private File uncommentProperties(URL url) throws IOException {
    InputStream in = url.openStream();

    assertNotNull(in);

    LineIterator it = IOUtils.lineIterator(in, "UTF-8");
    Collection<String> lines = new ArrayList<>();

    while (it.hasNext())
        lines.add(it.nextLine().replace("#ignite.client.", "ignite.client."));

    IgniteUtils.closeQuiet(in);

    File tmp = File.createTempFile(UUID.randomUUID().toString(), "properties");

    tmp.deleteOnExit();

    FileUtils.writeLines(tmp, lines);

    return tmp;
}

From source file:org.apache.marmotta.loader.rio.GeonamesParser.java

/**
 * Parses the data from the supplied InputStream, using the supplied baseURI
 * to resolve any relative URI references.
 *
 * @param in      The InputStream from which to read the data.
 * @param baseURI The URI associated with the data in the InputStream.
 * @throws java.io.IOException                 If an I/O error occurred while data was read from the InputStream.
 * @throws org.openrdf.rio.RDFParseException   If the parser has found an unrecoverable parse error.
 * @throws org.openrdf.rio.RDFHandlerException If the configured statement handler has encountered an
 *                                             unrecoverable error.
 *//*from   www. j  a v  a2  s .c o  m*/
@Override
public void parse(InputStream in, String baseURI) throws IOException, RDFParseException, RDFHandlerException {
    LineIterator it = IOUtils.lineIterator(in, RDFFormat.RDFXML.getCharset());
    try {
        while (it.hasNext()) {
            lineNumber++;

            String line = it.nextLine();
            if (lineNumber % 2 == 0) {
                // only odd line numbers contain triples
                StringReader buffer = new StringReader(line);
                lineParser.parse(buffer, baseURI);
            }
        }
    } finally {
        it.close();
    }
}

From source file:org.apache.marmotta.platform.core.services.prefix.PrefixCC.java

@Override
public String getNamespace(final String prefix) {
    HttpGet get = new HttpGet(URI + prefix + ".file.txt");
    HttpRequestUtil.setUserAgentString(get, USER_AGENT);
    get.setHeader(ACCEPT, "text/plain");
    try {/*ww w . j a  v  a 2 s. c  o m*/
        return httpClientService.execute(get, new ResponseHandler<String>() {

            @Override
            public String handleResponse(HttpResponse response) throws ClientProtocolException, IOException {
                if (200 == response.getStatusLine().getStatusCode()) {
                    HttpEntity entity = response.getEntity();

                    final LineIterator it = IOUtils.lineIterator(entity.getContent(), Charset.defaultCharset());
                    try {
                        while (it.hasNext()) {
                            final String l = it.next();
                            if (l.startsWith(prefix + "\t")) {
                                return l.substring(prefix.length() + 1);
                            }
                        }
                    } finally {
                        it.close();
                    }
                }
                log.error("Error: prefix '" + prefix + "' not found at prefix.cc");
                return null;
            }
        });
    } catch (Exception e) {
        log.error("Error retrieving prefix '" + prefix + "' from prefix.cc: " + e.getMessage());
        return null;
    }
}

From source file:org.apache.marmotta.platform.core.services.prefix.PrefixCC.java

@Override
public String getPrefix(final String namespace) {
    try {//from w  ww  .j a va2 s. com
        HttpGet get = new HttpGet(URI + "reverse?format=txt&uri=" + URLEncoder.encode(namespace, "utf-8"));
        HttpRequestUtil.setUserAgentString(get, USER_AGENT);
        get.setHeader(ACCEPT, "text/plain");

        return httpClientService.execute(get, new ResponseHandler<String>() {

            @Override
            public String handleResponse(HttpResponse response) throws ClientProtocolException, IOException {
                if (200 == response.getStatusLine().getStatusCode()) {
                    HttpEntity entity = response.getEntity();

                    final LineIterator it = IOUtils.lineIterator(entity.getContent(), Charset.defaultCharset());
                    try {
                        while (it.hasNext()) {
                            final String l = it.next();
                            if (l.endsWith("\t" + namespace)) {
                                return l.substring(0, l.indexOf("\t"));
                            }
                        }
                    } finally {
                        it.close();
                    }
                }
                log.error("Error: reverse namespace lookup for '" + namespace + "' not found at prefix.cc");
                return null;
            }
        });
    } catch (Exception e) {
        log.error("Error trying to retrieve prefic.cc reverse lookup for namespace '" + namespace + "': "
                + e.getMessage());
        return null;
    }
}

From source file:org.apache.nifi.processors.csv.ExtractCSVHeader.java

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    final FlowFile original = session.get();
    if (original == null) {
        return;//  ww  w. ja v  a2s  .  com
    }

    final AtomicBoolean lineFound = new AtomicBoolean(false);
    final Map<String, String> attrs = new HashMap<>();

    final AtomicInteger headerLength = new AtomicInteger(0);

    session.read(original, new InputStreamCallback() {
        @Override
        public void process(InputStream inputStream) throws IOException {
            // TODO expose the charset property?
            LineIterator iterator = IOUtils.lineIterator(inputStream, UTF_8);
            if (iterator.hasNext()) {
                lineFound.set(true);
                final String header = iterator.nextLine();

                final String format = context.getProperty(PROP_FORMAT).getValue();
                final String delimiter = context.getProperty(PROP_DELIMITER)
                        .evaluateAttributeExpressions(original).getValue();
                final String prefix = context.getProperty(PROP_SCHEMA_ATTR_PREFIX)
                        .evaluateAttributeExpressions(original).getValue();

                attrs.put(prefix + ATTR_HEADER_ORIGINAL, header);
                // TODO validate delimiter in the callback first
                final CSVFormat csvFormat = buildFormat(format, delimiter, true, // we assume first line is the header
                        null); // no custom header
                final CSVParser parser = csvFormat.parse(new StringReader(header));
                final Map<String, Integer> headers = parser.getHeaderMap();
                final int columnCount = headers.size();
                attrs.put(prefix + ATTR_HEADER_COLUMN_COUNT, String.valueOf(columnCount));
                for (Map.Entry<String, Integer> h : headers.entrySet()) {
                    // CSV columns are 1-based in Excel
                    attrs.put(prefix + (h.getValue() + 1), h.getKey());
                }

                // strip the header and send to the 'content' relationship
                if (StringUtils.isNotBlank(header)) {
                    int hLength = header.length();
                    // move past the new line if there are more lines
                    if (original.getSize() > hLength + 1) {
                        hLength++;
                    }
                    headerLength.set(hLength);
                }
            }
        }
    });

    if (lineFound.get()) {
        FlowFile ff = session.putAllAttributes(original, attrs);

        int offset = headerLength.get();
        if (offset > 0) {
            FlowFile contentOnly = session.clone(ff, offset, original.getSize() - offset);
            session.transfer(contentOnly, REL_CONTENT);
        }

        session.transfer(ff, REL_ORIGINAL);
    } else {
        session.transfer(original, REL_FAILURE);
    }
}

From source file:org.apache.nifi.processors.csv.ParseCSVRecord.java

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    final FlowFile original = session.get();
    if (original == null) {
        return;//from  w w w . j a  v  a 2 s  .  com
    }

    final AtomicBoolean lineFound = new AtomicBoolean(false);
    final Map<String, String> outputAttrs = new HashMap<>();

    session.read(original, new InputStreamCallback() {
        @Override
        public void process(InputStream inputStream) throws IOException {
            final String fromAttribute = context.getProperty(PROP_RECORD_FROM_ATTRIBUTE).getValue();

            String unparsedRecord;
            // data source is the attribute
            if (StringUtils.isNotBlank(fromAttribute)) {
                unparsedRecord = original.getAttribute(fromAttribute);
                if (StringUtils.isBlank(unparsedRecord)) {
                    // will be routed to failure at the end of the method implementation
                    return;
                }
            } else {
                // data source is the content
                // TODO expose the charset property?
                LineIterator iterator = IOUtils.lineIterator(inputStream, UTF_8);
                if (!iterator.hasNext()) {
                    return;
                }
                unparsedRecord = iterator.next();
            }

            lineFound.set(true);
            final String format = context.getProperty(PROP_FORMAT).getValue();
            final String delimiter = context.getProperty(PROP_DELIMITER).evaluateAttributeExpressions(original)
                    .getValue();
            final String schemaPrefix = context.getProperty(PROP_SCHEMA_ATTR_PREFIX)
                    .evaluateAttributeExpressions(original).getValue();
            final String valuePrefix = context.getProperty(PROP_VALUE_ATTR_PREFIX)
                    .evaluateAttributeExpressions(original).getValue();
            final boolean trimValues = context.getProperty(PROP_TRIM_VALUES).asBoolean();

            final CSVFormat csvFormat = buildFormat(format, delimiter, false, // this is a payload, not header anymore
                    null); // no custom header

            final CSVParser parser = csvFormat.parse(new StringReader(unparsedRecord));
            List<CSVRecord> records = parser.getRecords();
            if (records.size() > 1) {
                // TODO revisit for NiFi's native micro-batching
                throw new ProcessException("Multi-line entries not supported");
            }

            CSVRecord record = records.get(0);

            Map<String, String> originalAttrs = original.getAttributes();
            // filter delimited schema attributes only
            Map<String, String> schemaAttrs = new HashMap<>();
            for (String key : originalAttrs.keySet()) {
                if (key.startsWith(schemaPrefix)) {
                    schemaAttrs.put(key, originalAttrs.get(key));
                }
            }

            // put key/value pairs into attributes
            for (int i = 0; i < record.size(); i++) {
                String columnName = schemaAttrs.get(schemaPrefix + (i + 1)); // 1-based column numbering
                if (columnName == null) {
                    // 1-based column index
                    columnName = String.valueOf(i + 1);
                }
                // TODO indexed schemaless parsing vs auto-schema vs user-provided schema
                String columnValue = record.get(i);
                if (trimValues) {
                    columnValue = columnValue.trim();
                }
                String attrName = (StringUtils.isBlank(valuePrefix) ? "delimited.column." : valuePrefix)
                        + columnName;
                outputAttrs.put(attrName, columnValue);
            }
        }
    });

    if (lineFound.get()) {
        FlowFile ff = session.putAllAttributes(original, outputAttrs);
        session.transfer(ff, REL_SUCCESS);
    } else {
        session.transfer(original, REL_FAILURE);
    }
}

From source file:org.apache.olingo.client.core.communication.request.batch.ODataBatchUtilities.java

/**
 * Reads headers from the batch starting from the given position.
 * <p>//from  w  w w  .ja va2 s.co m
 * Retrieved headers will be added to the map given by target parameter.
 *
 * @param iterator batch iterator.
 * @param target destination of the retrieved headers.
 */
public static void readHeaders(final ODataBatchLineIterator iterator,
        final Map<String, Collection<String>> target) {

    try {
        final ByteArrayOutputStream baos = new ByteArrayOutputStream();
        readBatchPart(new ODataBatchController(iterator, null), baos, true);

        final LineIterator headers = IOUtils.lineIterator(new ByteArrayInputStream(baos.toByteArray()),
                Constants.UTF8);
        while (headers.hasNext()) {
            final String line = headers.nextLine().trim();
            if (StringUtils.isNotBlank(line)) {
                addHeaderLine(line, target);
            }
        }
    } catch (Exception e) {
        LOG.error("Error retrieving headers", e);
        throw new IllegalStateException(e);
    }
}

From source file:org.apache.olingo.client.core.communication.response.AbstractODataResponse.java

@Override
public ODataResponse initFromEnclosedPart(final InputStream part) {
    try {/*from   w  ww  .j a  v  a 2s .  c  o  m*/
        if (hasBeenInitialized) {
            throw new IllegalStateException("Request already initialized");
        }

        final ODataBatchLineIteratorImpl batchLineIterator = new ODataBatchLineIteratorImpl(
                IOUtils.lineIterator(part, Constants.UTF8));

        final Map.Entry<Integer, String> partResponseLine = ODataBatchUtilities
                .readResponseLine(batchLineIterator);
        LOG.debug("Retrieved async item response {}", partResponseLine);

        this.statusCode = partResponseLine.getKey();
        this.statusMessage = partResponseLine.getValue();

        final Map<String, Collection<String>> partHeaders = ODataBatchUtilities.readHeaders(batchLineIterator);
        LOG.debug("Retrieved async item headers {}", partHeaders);

        this.headers.putAll(partHeaders);

        final ByteArrayOutputStream bos = new ByteArrayOutputStream();

        while (batchLineIterator.hasNext()) {
            bos.write(batchLineIterator.nextLine().getBytes(Constants.UTF8));
            bos.write(ODataStreamer.CRLF);
        }

        try {
            this.payload = new ByteArrayInputStream(bos.toByteArray());
        } catch (Exception e) {
            LOG.error("Error retrieving payload", e);
            throw new IllegalStateException(e);
        }

        this.hasBeenInitialized = true;
        return this;
    } catch (IOException e) {
        LOG.error("Error streaming payload response", e);
        throw new IllegalStateException(e);
    }
}