Example usage for org.apache.commons.io LineIterator hasNext

Introduction

In this page you can find the example usage for org.apache.commons.io LineIterator hasNext.

Prototype

public boolean hasNext() 

Source Link

Document

Indicates whether the Reader has more lines.

Usage

From source file:org.trend.hgraph.mapreduce.pagerank.DriverTest.java

private static void generateGraphDataTest(String dataPath, String tableName, String cq, boolean isDouble)
        throws IOException {
    InputStream data = DriverTest.class.getClassLoader().getResourceAsStream(dataPath);
    LineIterator it = IOUtils.lineIterator(new InputStreamReader(data));
    String record = null;/* w w  w.  j  a v  a 2s .  co m*/
    Assert.assertEquals(true, it.hasNext());

    HTable table = null;
    try {
        table = new HTable(TEST_UTIL.getConfiguration(), tableName);
        table.setAutoFlush(false);
        Put put = null;
        String[] values = null;
        while (it.hasNext()) {
            record = it.next();
            values = record.split("\\|");
            Assert.assertNotNull(values);
            Assert.assertEquals(2, values.length);
            put = new Put(Bytes.toBytes(values[0]));
            if (isDouble) {
                put.add(Bytes.toBytes(HBaseGraphConstants.HBASE_GRAPH_TABLE_COLFAM_PROPERTY_NAME),
                        Bytes.toBytes(cq + HBaseGraphConstants.HBASE_GRAPH_TABLE_COLFAM_PROPERTY_NAME_DELIMITER
                                + "Double"),
                        Bytes.toBytes(Double.parseDouble(values[1])));
            } else {
                put.add(Bytes.toBytes(HBaseGraphConstants.HBASE_GRAPH_TABLE_COLFAM_PROPERTY_NAME),
                        Bytes.toBytes(cq + HBaseGraphConstants.HBASE_GRAPH_TABLE_COLFAM_PROPERTY_NAME_DELIMITER
                                + "String"),
                        Bytes.toBytes(values[1]));
            }
            table.put(put);
        }
        table.flushCommits();
    } catch (IOException e) {
        System.err.println("generate data for table:" + tableName + " failed");
        e.printStackTrace(System.err);
        throw e;
    } finally {
        table.close();
        LineIterator.closeQuietly(it);
        IOUtils.closeQuietly(data);
    }
}

---

From source file:org.trend.hgraph.mapreduce.pagerank.LocalPrTest.java

private static List<V> generateVs(String vertexData, String edgeData) {
    InputStream data = DriverTest.class.getClassLoader().getResourceAsStream(vertexData);
    LineIterator it = IOUtils.lineIterator(new InputStreamReader(data));

    // load all Vs
    List<V> vs = new ArrayList<V>();
    String record = null;/*w w w.  j a v  a  2 s  . co m*/
    String[] values = null;
    V v = null;
    while (it.hasNext()) {
        record = it.next();
        values = record.split("\\|");
        Assert.assertNotNull(values);
        Assert.assertEquals(2, values.length);
        v = new V();
        v.key = values[0];
        v.pr = Double.parseDouble(values[1]);
        vs.add(v);
    }
    LineIterator.closeQuietly(it);
    IOUtils.closeQuietly(data);

    // build adjacency list
    data = DriverTest.class.getClassLoader().getResourceAsStream(edgeData);
    it = IOUtils.lineIterator(new InputStreamReader(data));
    while (it.hasNext()) {
        record = it.next();
        values = record.split("\\|");
        Assert.assertNotNull(values);
        Assert.assertEquals(2, values.length);

        values = values[0].split(HBaseGraphConstants.HBASE_GRAPH_TABLE_EDGE_DELIMITER_1);
        Assert.assertNotNull(values);
        Assert.assertEquals(3, values.length);
        for (V tv1 : vs) {
            if (tv1.key.equals(values[0])) {
                for (V tv2 : vs) {
                    if (tv2.key.equals(values[2])) {
                        tv1.al.add(tv2);
                        break;
                    }
                }
                break;
            }
        }
    }
    LineIterator.closeQuietly(it);
    IOUtils.closeQuietly(data);

    System.out.println("Vs=\n" + vs);
    return vs;
}

---

From source file:org.trend.hgraph.util.MoveEntities.java

private static void moveEntities(String f, Configuration conf, String st, String dt) throws IOException {
    Reader fr = null;/*from   ww w  . j a  v a 2 s .c  om*/
    LineIterator it = null;

    String key = null;
    HTable sTable = null;
    HTable dTable = null;

    try {
        fr = new FileReader(f);
    } catch (FileNotFoundException e) {
        System.err.println("file:" + f + " does not exist");
        e.printStackTrace(System.err);
        throw e;
    }
    try {
        it = IOUtils.lineIterator(fr);
        if (it.hasNext()) {
            sTable = new HTable(conf, st);
            dTable = new HTable(conf, dt);
            Get get = null;
            Put put = null;
            Result r = null;
            byte[] bKey = null;
            byte[] cf = null;
            byte[] cq = null;
            byte[] v = null;
            NavigableMap<byte[], NavigableMap<byte[], byte[]>> cfMap = null;
            NavigableMap<byte[], byte[]> cqMap = null;
            Entry<byte[], NavigableMap<byte[], byte[]>> cfEntry = null;
            Entry<byte[], byte[]> cqEntry = null;
            long cnt = 0L;
            while (it.hasNext()) {
                key = it.next();
                bKey = Bytes.toBytes(key);
                get = new Get(bKey);
                r = sTable.get(get);
                if (r.isEmpty()) {
                    String msg = "result is empty for key:" + key + " from table:" + sTable;
                    System.err.println(msg);
                    throw new IllegalStateException(msg);
                }
                // dump the values from r to p
                put = new Put(bKey);
                cfMap = r.getNoVersionMap();
                for (Iterator<Entry<byte[], NavigableMap<byte[], byte[]>>> cfIt = cfMap.entrySet()
                        .iterator(); cfIt.hasNext();) {
                    cfEntry = cfIt.next();
                    cf = cfEntry.getKey();
                    cqMap = cfEntry.getValue();
                    for (Iterator<Entry<byte[], byte[]>> cqIt = cqMap.entrySet().iterator(); cqIt.hasNext();) {
                        cqEntry = cqIt.next();
                        cq = cqEntry.getKey();
                        v = cqEntry.getValue();
                        put.add(cf, cq, v);
                    }
                }
                dTable.put(put);
                cnt++;
            }
            System.out.println("put " + cnt + " row(s) into table:" + dt);
        }
    } catch (IOException e) {
        System.err.println("error occurs while doing HBase manipultations !!");
        e.printStackTrace(System.err);
        throw e;
    } finally {
        LineIterator.closeQuietly(it);
        IOUtils.closeQuietly(fr);
        if (null != sTable) {
            sTable.close();
        }
        if (null != dTable) {
            dTable.close();
        }
    }
}

---

From source file:org.vafer.jdeb.signing.PGPSigner.java

/**
 * Creates a clear sign signature over the input data. (Not detached)
 *
 * @param input      the content to be signed
 * @param output     the output destination of the signature
 *//*from w ww  .  j  a  va2s .c  o m*/
public void clearSign(InputStream input, OutputStream output)
        throws IOException, PGPException, GeneralSecurityException {
    int digest = PGPUtil.SHA1;

    PGPSignatureGenerator signatureGenerator = new PGPSignatureGenerator(
            new BcPGPContentSignerBuilder(privateKey.getPublicKeyPacket().getAlgorithm(), digest));
    signatureGenerator.init(PGPSignature.CANONICAL_TEXT_DOCUMENT, privateKey);

    ArmoredOutputStream armoredOutput = new ArmoredOutputStream(output);
    armoredOutput.beginClearText(digest);

    LineIterator iterator = new LineIterator(new InputStreamReader(input));

    while (iterator.hasNext()) {
        String line = iterator.nextLine();

        // trailing spaces must be removed for signature calculation (see http://tools.ietf.org/html/rfc4880#section-7.1)
        byte[] data = trim(line).getBytes("UTF-8");

        armoredOutput.write(data);
        armoredOutput.write(EOL);

        signatureGenerator.update(data);
        if (iterator.hasNext()) {
            signatureGenerator.update(EOL);
        }
    }

    armoredOutput.endClearText();

    PGPSignature signature = signatureGenerator.generate();
    signature.encode(new BCPGOutputStream(armoredOutput));

    armoredOutput.close();
}

---

From source file:org.verwandlung.voj.judger.core.Comparator.java

/**
 * ??.//from   w  ww  .  jav  a  2s. c  o m
 * @param stdFileItr - 
 * @param fileItr - 
 * @return ??
 */
private boolean isFileOutputTheSame(LineIterator stdFileItr, LineIterator fileItr) {
    try {
        while (stdFileItr.hasNext() && fileItr.hasNext()) {
            String stdLine = stdFileItr.nextLine();
            String line = fileItr.nextLine();

            if (!isLineOutputTheSame(stdLine, line)) {
                return false;
            }
        }
        while (stdFileItr.hasNext()) {
            String line = stdFileItr.nextLine();
            if (!isLineEmpty(line, 0)) {
                return false;
            }
        }
        while (fileItr.hasNext()) {
            String line = fileItr.nextLine();
            if (!isLineEmpty(line, 0)) {
                return false;
            }
        }
    } catch (OutOfMemoryError ex) {
        LOGGER.catching(ex);
        return false;
    }
    return true;
}

---

From source file:org.wikimedia.analytics.varnishkafka.Cli.java

private Integer writeEscapedOutput() {
    int n = 0;/*from  w w  w . j  a  v a 2  s. com*/
    OutputStream out = null;
    BufferedOutputStream bos = null;
    try {
        LineIterator it = FileUtils.lineIterator(inputFile, "UTF-8");
        File outputFile = new File(cwd.getPath(), "test." + getFormat());
        outputFile.delete();
        log.info("Output file path: " + outputFile.toString());
        out = new FileOutputStream(outputFile);
        bos = new BufferedOutputStream(out);
        setStart(System.nanoTime());
        while (it.hasNext()) {
            n++;
            String line = it.nextLine();
            String[] fields = line.split("\\t");
            String ua = fields[14].replace("\\t", " ").replace("\\n", " ");
            fields[14] = ua;

            for (String field : fields) {
                bos.write(field.getBytes());
                bos.write("\t".getBytes());
            }
            bos.write("\n".getBytes());
        }
        setEnd(System.nanoTime());
    } catch (IOException e) {
        e.printStackTrace();
    } finally {
        try {
            out.close();
            bos.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
    return n;
}

---

From source file:org.wikimedia.analytics.varnishkafka.Cli.java

private Integer writeJsonOutput() {
    int n = 0;//  www  .  j  ava  2  s  .  co m
    JsonFactory jfactory = new JsonFactory();

    /*** write to file ***/
    try {
        JsonGenerator jGenerator;

        SnappyOutputStream snappyOutputStream = null;
        File outputFile = new File(cwd.getPath(), "test." + getFormat());
        OutputStream out = new FileOutputStream(outputFile);
        BufferedOutputStream bos = new BufferedOutputStream(out);

        if (compress) {
            snappyOutputStream = new SnappyOutputStream(bos);
            jGenerator = jfactory.createJsonGenerator(snappyOutputStream, JsonEncoding.UTF8);
        } else {
            jGenerator = jfactory.createJsonGenerator(bos, JsonEncoding.UTF8);
        }

        log.info("Output file path: " + outputFile.toString());

        LineIterator it = FileUtils.lineIterator(inputFile, "UTF-8");

        try {
            setStart(System.nanoTime());
            while (it.hasNext()) {
                n++;
                String line = it.nextLine();
                String[] fields = line.split("\\t");

                jGenerator.writeStartObject();

                jGenerator.writeNumberField("kafka_offset", Long.parseLong(fields[0]));
                jGenerator.writeStringField("host", fields[1]);
                jGenerator.writeNumberField("seq_num", Long.parseLong(fields[2]));
                jGenerator.writeStringField("timestamp", fields[3]);
                jGenerator.writeNumberField("response", Float.parseFloat(fields[4]));
                jGenerator.writeStringField("ip", fields[5]);
                jGenerator.writeStringField("http_status", fields[6]);
                jGenerator.writeNumberField("bytes_sent", parseBytesSent(fields[7]));
                jGenerator.writeStringField("request_method", fields[8]);
                jGenerator.writeStringField("uri", fields[9]);
                jGenerator.writeStringField("proxy_host", fields[10]);
                jGenerator.writeStringField("mime_type", fields[11]);
                jGenerator.writeStringField("referer", fields[12]);
                jGenerator.writeStringField("x_forwarded_for", fields[13]);
                jGenerator.writeStringField("user_agent", fields[14]);
                jGenerator.writeStringField("accept_language", fields[15]);
                jGenerator.writeStringField("x_analytics", fields[16]);

                jGenerator.writeEndObject();
            }
            setEnd(System.nanoTime());
        } finally {
            it.close();
            jGenerator.flush();
            jGenerator.close();
            if (compress) {
                snappyOutputStream.close();
            } else {
                out.close();
                bos.close();
            }
        }
    } catch (JsonGenerationException e) {
        e.printStackTrace();
    } catch (JsonMappingException e) {
        e.printStackTrace();
    } catch (IOException e) {
        e.printStackTrace();
    }
    return n;
}

---

From source file:org.wikimedia.analytics.varnishkafka.Cli.java

private Integer writeProtobufOutput() {
    int n = 0;/*from w w w. j  av a  2  s . c o m*/
    try {
        LineIterator it = FileUtils.lineIterator(inputFile, "UTF-8");
        File outputFile = new File(cwd.getPath(), "test." + getFormat());
        outputFile.delete();
        OutputStream out = new FileOutputStream(outputFile);
        BufferedOutputStream bos = new BufferedOutputStream(out);
        SnappyOutputStream snappyOutputStream = null;

        if (compress) {
            snappyOutputStream = new SnappyOutputStream(bos);
        }

        log.info("Output file path: " + outputFile.toString());
        try {
            setStart(System.nanoTime());
            while (it.hasNext()) {
                n++;
                String line = it.nextLine();
                String[] fields = line.split("\\t");
                Logline.LogLine logline = Logline.LogLine.newBuilder().setKafkaOffset(Long.parseLong(fields[0]))
                        .setHost(fields[1]).setSeqNum(Long.parseLong(fields[2])).setTimestamp(fields[3])
                        .setResponse(Float.parseFloat(fields[4])).setIp(fields[5]).setHttpStatus(fields[6])
                        .setBytesSent(parseBytesSent(fields[7])).setRequestMethod(fields[8]).setUri(fields[9])
                        .setProxyHost(fields[10]).setMimeType(fields[11]).setReferer(fields[12])
                        .setXForwardedFor(fields[13]).setUserAgent(fields[14]).setAcceptLanguage(fields[15])
                        .setXAnalytics(fields[16]).build();

                if (compress) {
                    snappyOutputStream.write(logline.toByteArray());
                } else {
                    bos.write(logline.toByteArray());
                }
            }
            setEnd(System.nanoTime());
        } finally {
            try {
                bos.flush();
                out.flush();
                out.close();
                bos.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
    } catch (Exception e) {
        e.printStackTrace();
    }
    return n;
}

---

From source file:org.wikimedia.analytics.varnishkafka.Cli.java

private Integer writeAvroOutput() {
    Schema schema = null;/*  www .ja v a 2  s.  c om*/
    int n = 0;

    try {
        InputStream inputStream = ClassLoader.getSystemClassLoader()
                .getResourceAsStream("WebRequest.avro.json");
        schema = new Schema.Parser().parse(inputStream);
        inputStream.close();

        File file = new File(cwd.getPath(), "test." + getFormat());
        log.info("Output file path: " + file.toString());
        file.delete();
        DatumWriter<GenericRecord> writer = new GenericDatumWriter<GenericRecord>(schema);
        DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<GenericRecord>(writer);

        if (compress) {
            dataFileWriter.setCodec(CodecFactory.snappyCodec());
        }

        dataFileWriter.create(schema, file);

        try {
            LineIterator it = FileUtils.lineIterator(inputFile, "UTF-8");

            try {
                setStart(System.nanoTime());
                while (it.hasNext()) {
                    n++;
                    String line = it.nextLine();
                    String[] fields = line.split("\\t");

                    // Populate data
                    GenericRecord r = new GenericData.Record(schema);
                    r.put("kafka_offset", Long.parseLong(fields[0]));
                    r.put("host", fields[1]);
                    r.put("seq_num", Long.parseLong(fields[2]));
                    r.put("timestamp", fields[3]);
                    r.put("response", Float.parseFloat(fields[4]));
                    r.put("ip", fields[5]);
                    r.put("http_status", fields[6]);
                    r.put("bytes_sent", parseBytesSent(fields[7]));
                    r.put("request_method", fields[8]);
                    r.put("uri", fields[9]);
                    r.put("proxy_host", fields[10]);
                    r.put("mime_type", fields[11]);
                    r.put("referer", fields[12]);
                    r.put("x_forwarded_for", fields[13]);
                    r.put("user_agent", fields[14]);
                    r.put("accept_language", fields[15]);
                    r.put("x_analytics", fields[16]);
                    dataFileWriter.append(r);
                }

                setEnd(System.nanoTime());
            } finally {
                dataFileWriter.flush();
                dataFileWriter.close();
            }
        } catch (IOException ex) {
            ex.printStackTrace();
        }
    } catch (IOException e) {
        e.printStackTrace();
    }
    return n;
}

---

From source file:org.wso2.identity.integration.test.user.mgt.UserImportLoggingTestCase.java

/**
 * Read the audit log file and extract the log entries as lines.
 *
 * @return : An Array List which contains audit log lines.
 * @throws IOException : If any error occurred while reading the file.
 *///from  w  w w .j  a  v a 2 s  . c o  m
private List<String> readAuditLogFile() throws IOException {

    List<String> bulkUserImportAuditLogs = new ArrayList<>();
    String auditLogFile = LOG_FILE_LOCATION + File.separatorChar + AUDIT_LOG_FILE_NAME;
    File auditFile = new File(auditLogFile);

    // Iterate through the file and read lines.
    LineIterator iterator = FileUtils.lineIterator(auditFile, ENCODING);

    while (iterator.hasNext()) {
        String auditLine = iterator.nextLine();

        if (StringUtils.contains(auditLine, BULK_USER_IMPORT_OP)) {
            bulkUserImportAuditLogs.add(auditLine);
        }
    }
    return bulkUserImportAuditLogs;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• »