Example usage for org.apache.commons.io LineIterator next

Introduction

In this page you can find the example usage for org.apache.commons.io LineIterator next.

Prototype

public Object next() 

Source Link

Document

Returns the next line in the wrapped Reader.

Usage

From source file:com.senseidb.search.node.inmemory.InMemoryIndexPerfEval.java

public static void main(String[] args) throws Exception {
    final InMemorySenseiService memorySenseiService = InMemorySenseiService.valueOf(
            new File(InMemoryIndexPerfEval.class.getClassLoader().getResource("test-conf/node1/").toURI()));

    final List<JSONObject> docs = new ArrayList<JSONObject>(15000);
    LineIterator lineIterator = FileUtils.lineIterator(
            new File(InMemoryIndexPerfEval.class.getClassLoader().getResource("data/test_data.json").toURI()));
    int i = 0;// w w  w . j  a  v  a2s.  c  o m
    while (lineIterator.hasNext() && i < 100) {
        String car = lineIterator.next();
        if (car != null && car.contains("{"))
            docs.add(new JSONObject(car));
        i++;
    }
    Thread[] threads = new Thread[10];
    for (int k = 0; k < threads.length; k++) {
        threads[k] = new Thread(new Runnable() {
            public void run() {
                long time = System.currentTimeMillis();
                //System.out.println("Start thread");
                for (int j = 0; j < 1000; j++) {
                    //System.out.println("Send request");
                    memorySenseiService.doQuery(getRequest(), docs);
                }
                System.out.println("time = " + (System.currentTimeMillis() - time));
            }
        });
        threads[k].start();
    }
    Thread.sleep(500000);
}

---

From source file:de.tum.i13.ConvertCsvToProtobuf.java

public static void main(String args[]) {
    try {//from   w w w  .  ja  va  2s . co  m
        LineIterator it = FileUtils.lineIterator(new File("/Users/manit/Projects/sdcbenchmark/Dataset/debscsv"),
                "UTF-8");
        FileOutputStream out = new FileOutputStream("/Users/manit/Projects/sdcbenchmark/Dataset/debsprotobuf",
                true);

        while (it.hasNext()) {

            String csvLine = (String) it.next();
            byte[] csvLineBytes = csvLine.getBytes();
            String line = new String(csvLineBytes, StandardCharsets.UTF_8);
            Debs2015Protos.Taxitrip.Builder builder = Debs2015Protos.Taxitrip.newBuilder();
            String[] splitted = line.split(",");

            builder.setMedallion(splitted[0]);
            builder.setHackLicense(splitted[1]);
            builder.setPickupDatetime(splitted[2]);
            builder.setDropoffDatetime(splitted[3]);
            builder.setTripTimeInSecs(Integer.parseInt(splitted[4]));
            builder.setTripDistance(Float.parseFloat(splitted[5]));
            builder.setPickupLongitude(Float.parseFloat(splitted[6]));
            builder.setPickupLatitude(Float.parseFloat(splitted[7]));
            builder.setDropoffLongitude(Float.parseFloat(splitted[8]));
            builder.setDropoffLatitude(Float.parseFloat(splitted[9]));
            builder.setPaymentType(splitted[10]);
            builder.setFareAmount(Float.parseFloat(splitted[11]));
            builder.setSurcharge(Float.parseFloat(splitted[12]));
            builder.setMtaTax(Float.parseFloat(splitted[13]));
            builder.setTipAmount(Float.parseFloat(splitted[14]));
            builder.setTollsAmount(Float.parseFloat(splitted[15]));
            builder.setTotalAmount(Float.parseFloat(splitted[16]));

            builder.build().writeDelimitedTo(out);
        }
        out.close();
    } catch (Exception e) {
        e.printStackTrace();
    }
}

---

From source file:de.tudarmstadt.ukp.dkpro.c4corpus.hadoop.statistics.StatisticsTableCreator.java

public static Table<String, String, Long> loadTable(InputStream stream) throws IOException {
    Table<String, String, Long> result = TreeBasedTable.create();

    LineIterator lineIterator = IOUtils.lineIterator(stream, "utf-8");
    while (lineIterator.hasNext()) {
        String line = lineIterator.next();

        System.out.println(line);

        String[] split = line.split("\t");
        String language = split[0];
        String license = split[1];
        Long documents = Long.valueOf(split[2]);
        Long tokens = Long.valueOf(split[3]);

        result.put(language, "docs " + license, documents);
        result.put(language, "tokens " + license, tokens);
    }/*from ww  w .ja  va2  s  .  co  m*/

    return result;
}

---

From source file:de.tudarmstadt.ukp.dkpro.c4corpus.hadoop.statistics.vocabulary.TopNWordsCorrelation.java

public static LinkedHashMap<String, Integer> loadCorpusToRankedVocabulary(InputStream corpus)
        throws IOException {
    LinkedHashMap<String, Integer> result = new LinkedHashMap<>();

    LineIterator lineIterator = IOUtils.lineIterator(corpus, "utf-8");
    int counter = 0;
    while (lineIterator.hasNext()) {
        String line = lineIterator.next();

        String word = line.split("\\s+")[0];

        result.put(word, counter);/*  www.j  av  a2  s.  com*/
        counter++;
    }

    return result;
}

---

From source file:com.adobe.acs.tools.csv.impl.CsvUtil.java

/**
 * Adds a populated terminating field to the ends of CSV entries.
 * If the last entry in a CSV row is empty, the CSV library has difficulty understanding that is the end of the row.
 *
 * @param is        the CSV file as an inputstream
 * @param separator The field separator/*from ww w.  j a  v  a2  s .co m*/
 * @param charset   The charset
 * @return An inputstream that is the same as is, but each line has a populated line termination entry
 * @throws IOException
 */
public static InputStream terminateLines(final InputStream is, final char separator, final String charset)
        throws IOException {

    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final PrintStream printStream = new PrintStream(baos);

    final LineIterator lineIterator = IOUtils.lineIterator(is, charset);

    while (lineIterator.hasNext()) {
        String line = StringUtils.stripToNull(lineIterator.next());

        if (line != null) {
            line += separator + TERMINATED;
            printStream.println(line);
        }
    }

    return new ByteArrayInputStream(baos.toByteArray());
}

---

From source file:com.el.wordament.NodeLoader.java

public static Node init() throws IOException {
    LineIterator iterator = FileUtils.lineIterator(new File("src/main/resources/dict.txt"));

    Node root = new Node();
    root.setWord(false);/*from  w  w  w . ja v a 2 s  .c  om*/

    while (iterator.hasNext()) {
        String line = iterator.next().toLowerCase().trim();
        createNodes(line, root, 0).setWord(true);
    }

    return root;
}

---

From source file:de.rnd7.kata.reversi.logic.ai.AIMatrix.java

public static AIMatrix fromResource(final String name) throws IOException {
    final AIMatrix matrix = new AIMatrix();

    try (InputStream input = AIMatrix.class.getResourceAsStream(name)) {
        final LineIterator iterator = IOUtils.lineIterator(input, CharEncoding.UTF_8);

        int lineNumber = 0;
        while (iterator.hasNext()) {
            processLine(matrix, lineNumber++, iterator.next());
        }//from  www.j av a 2  s.  co  m
    }

    return matrix;
}

---

From source file:com.meltmedia.cadmium.core.meta.MimeTypeConfigProcessor.java

static void addDefaultMimeTypes(Map<String, String> mimeTypeMap)
        throws IllegalArgumentException, UnsupportedEncodingException {
    InputStream in = null;//  www  .  ja va  2 s.  c  o  m
    try {
        in = MimeTypeConfigProcessor.class.getResourceAsStream("mime.types");
        if (in == null) {
            log.warn("The default mime type file is missing.");
            return;
        }
        LineIterator lineIterator = new LineIterator(new InputStreamReader(in, "UTF-8"));
        while (lineIterator.hasNext()) {
            String line = lineIterator.next();
            line = line.replaceAll("(\\A[^#]*)(#.*)?\\Z", "$1").trim(); // kill comments.
            if (line.length() == 0)
                continue; // skip blank lines.
            String[] parts = line.split("\\s+");
            if (parts.length < 2)
                continue; // skip lines with no extensions.
            String type = parts[0];
            for (int i = 1; i < parts.length; i++) {
                mimeTypeMap.put(parts[i], type);
            }
        }
    } finally {
        IOUtils.closeQuietly(in);
    }
}

---

From source file:com.mewmew.fairy.v1.book.Xargs.java

public static void exec(Map<String, String> env, String cmd[], boolean redirectError, Output<String> output)
        throws IOException, InterruptedException {
    ProcessBuilder pb = new ProcessBuilder(cmd);
    if (env != null) {
        pb.environment().putAll(env);/* w  w w  .j  av a 2s .  c o m*/
    }
    if (redirectError) {
        pb.redirectErrorStream(true);
    }
    final Process p = pb.start();
    if (!redirectError) {
        new Thread(new Runnable() {
            public void run() {
                try {
                    LineIterator err = new LineIterator(new InputStreamReader(p.getErrorStream()));
                    while (err.hasNext()) {
                        err.next();
                    }
                } finally {
                }
            }
        }).start();
    }
    LineIterator out = new LineIterator(new InputStreamReader(p.getInputStream()));
    while (out.hasNext()) {
        output.output(out.nextLine());
    }
    int code = p.waitFor();
    if (code != 0) {
        throw new RuntimeException(String.format("return != 0, code = %d", code));
    }
}

---

From source file:de.tudarmstadt.ukp.experiments.argumentation.clustering.ClusterCentroidsMain.java

public static TreeMap<Integer, Vector> computeClusterCentroids(String inputVectorsPath,
        String clusterOutputPath) throws IOException {
    TreeMap<Integer, Vector> result = new TreeMap<>();
    Map<Integer, Integer> counts = new TreeMap<>();

    // input for cluto
    File inputVectors = new File(inputVectorsPath);

    // resulting clusters
    File clutoClustersOutput = new File(clusterOutputPath);

    LineIterator clustersIterator = IOUtils.lineIterator(new FileInputStream(clutoClustersOutput), "utf-8");

    LineIterator vectorsIterator = IOUtils.lineIterator(new FileInputStream(inputVectors), "utf-8");

    // skip first line (number of clusters and vector size
    vectorsIterator.next();

    while (clustersIterator.hasNext()) {
        String clusterString = clustersIterator.next();
        String vectorString = vectorsIterator.next();

        int clusterNumber = Integer.valueOf(clusterString);

        // now parse the vector
        DenseVector vector = ClusteringUtils.parseVector(vectorString);

        // if there is no resulting vector for the particular cluster, add this one
        if (!result.containsKey(clusterNumber)) {
            result.put(clusterNumber, vector);
        } else {//from  w ww  .  j a v  a 2  s .  com
            // otherwise add this one to the previous one
            result.put(clusterNumber, result.get(clusterNumber).add(vector));
        }

        // and update counts
        if (!counts.containsKey(clusterNumber)) {
            counts.put(clusterNumber, 0);
        }

        counts.put(clusterNumber, counts.get(clusterNumber) + 1);
    }

    // now compute average for each vector
    for (Map.Entry<Integer, Vector> entry : result.entrySet()) {
        // cluster number
        int clusterNumber = entry.getKey();
        // get counts
        int count = counts.get(clusterNumber);

        // divide by count of vectors for each cluster (averaging)
        for (VectorEntry vectorEntry : entry.getValue()) {
            vectorEntry.set(vectorEntry.get() / (double) count);
        }
    }

    return result;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• »