Example usage for org.apache.commons.io LineIterator nextLine

Introduction

In this page you can find the example usage for org.apache.commons.io LineIterator nextLine.

Prototype

public String nextLine() 

Source Link

Document

Returns the next line in the wrapped Reader.

Usage

From source file:org.deeplearning4j.models.glove.Glove.java

/**
 * Load a glove model from an input stream.
 * The format is://from w w  w. ja  v  a2s . c  o m
 * word num1 num2....
 * @param is the input stream to read from for the weights
 * @param biases the bias input stream
 * @return the loaded model
 * @throws IOException if one occurs
 */
public static Glove load(InputStream is, InputStream biases) throws IOException {
    LineIterator iter = IOUtils.lineIterator(is, "UTF-8");
    Glove glove = new Glove();
    Map<String, float[]> wordVectors = new HashMap<>();
    int count = 0;
    while (iter.hasNext()) {
        String line = iter.nextLine().trim();
        if (line.isEmpty())
            continue;
        String[] split = line.split(" ");
        String word = split[0];
        if (glove.vocab() == null)
            glove.setVocab(new InMemoryLookupCache());

        if (glove.lookupTable() == null) {
            glove.lookupTable = new GloveWeightLookupTable.Builder().cache(glove.vocab())
                    .vectorLength(split.length - 1).build();

        }

        if (word.isEmpty())
            continue;
        float[] read = read(split, glove.lookupTable().getVectorLength());
        if (read.length < 1)
            continue;

        VocabWord w1 = new VocabWord(1, word);
        w1.setIndex(count);
        glove.vocab().addToken(w1);
        glove.vocab().addWordToIndex(count, word);
        glove.vocab().putVocabWord(word);
        wordVectors.put(word, read);
        count++;

    }

    glove.lookupTable().setSyn0(weights(glove, wordVectors));

    iter.close();

    glove.lookupTable().setBias(Nd4j.read(biases));

    return glove;

}

---

From source file:org.deeplearning4j.models.glove.GloveWeightLookupTable.java

/**
 * Load a glove model from an input stream.
 * The format is:/*from   ww  w  .ja  v  a  2s .  c  o m*/
 * word num1 num2....
 * @param is the input stream to read from for the weights
 * @param vocab the vocab for the lookuptable
 * @return the loaded model
 * @throws java.io.IOException if one occurs
 */
public static GloveWeightLookupTable load(InputStream is, VocabCache<? extends SequenceElement> vocab)
        throws IOException {
    LineIterator iter = IOUtils.lineIterator(is, "UTF-8");
    GloveWeightLookupTable glove = null;
    Map<String, float[]> wordVectors = new HashMap<>();
    while (iter.hasNext()) {
        String line = iter.nextLine().trim();
        if (line.isEmpty())
            continue;
        String[] split = line.split(" ");
        String word = split[0];
        if (glove == null)
            glove = new GloveWeightLookupTable.Builder().cache(vocab).vectorLength(split.length - 1).build();

        if (word.isEmpty())
            continue;
        float[] read = read(split, glove.layerSize());
        if (read.length < 1)
            continue;

        wordVectors.put(word, read);

    }

    glove.setSyn0(weights(glove, wordVectors, vocab));
    glove.resetWeights(false);

    iter.close();

    return glove;

}

---

From source file:org.deeplearning4j.models.glove.LegacyGlove.java

/**
 * Load a glove model from an input stream.
 * The format is://from   w  w  w.  ja v a  2s .c om
 * word num1 num2....
 * @param is the input stream to read from for the weights
 * @param biases the bias input stream
 * @return the loaded model
 * @throws IOException if one occurs
 */
public static LegacyGlove load(InputStream is, InputStream biases) throws IOException {
    LineIterator iter = IOUtils.lineIterator(is, "UTF-8");
    LegacyGlove glove = new LegacyGlove();
    Map<String, float[]> wordVectors = new HashMap<>();
    int count = 0;
    while (iter.hasNext()) {
        String line = iter.nextLine().trim();
        if (line.isEmpty())
            continue;
        String[] split = line.split(" ");
        String word = split[0];
        if (glove.vocab() == null)
            glove.setVocab(new InMemoryLookupCache());

        if (glove.lookupTable() == null) {
            glove.lookupTable = new GloveWeightLookupTable.Builder().cache(glove.vocab())
                    .vectorLength(split.length - 1).build();

        }

        if (word.isEmpty())
            continue;
        float[] read = read(split, glove.lookupTable().layerSize());
        if (read.length < 1)
            continue;

        VocabWord w1 = new VocabWord(1, word);
        w1.setIndex(count);
        glove.vocab().addToken(w1);
        glove.vocab().addWordToIndex(count, word);
        glove.vocab().putVocabWord(word);
        wordVectors.put(word, read);
        count++;

    }

    glove.lookupTable().setSyn0(weights(glove, wordVectors));

    iter.close();

    glove.lookupTable().setBias(Nd4j.read(biases));

    return glove;

}

---

From source file:org.deeplearning4j.models.word2vec.iterator.Word2VecDataFetcher.java

@Override
public DataSet next() {
    //pop from cache when possible, or when there's nothing left
    if (cache.size() >= batch || !files.hasNext())
        return fromCache();

    File f = files.next();/*from w w  w. j  a  va 2s.  co  m*/
    try {
        LineIterator lines = FileUtils.lineIterator(f);
        INDArray outcomes = null;
        INDArray input = null;

        while (lines.hasNext()) {
            List<Window> windows = Windows.windows(lines.nextLine());
            if (windows.isEmpty() && lines.hasNext())
                continue;

            if (windows.size() < batch) {
                input = Nd4j.create(windows.size(), vec.lookupTable().layerSize() * vec.getWindow());
                outcomes = Nd4j.create(batch, labels.size());
                for (int i = 0; i < windows.size(); i++) {
                    input.putRow(i, WindowConverter.asExampleMatrix(cache.get(i), vec));
                    int idx = labels.indexOf(windows.get(i).getLabel());
                    if (idx < 0)
                        idx = 0;
                    INDArray outcomeRow = FeatureUtil.toOutcomeVector(idx, labels.size());
                    outcomes.putRow(i, outcomeRow);
                }
                return new DataSet(input, outcomes);

            } else {
                input = Nd4j.create(batch, vec.lookupTable().layerSize() * vec.getWindow());
                outcomes = Nd4j.create(batch, labels.size());
                for (int i = 0; i < batch; i++) {
                    input.putRow(i, WindowConverter.asExampleMatrix(cache.get(i), vec));
                    int idx = labels.indexOf(windows.get(i).getLabel());
                    if (idx < 0)
                        idx = 0;
                    INDArray outcomeRow = FeatureUtil.toOutcomeVector(idx, labels.size());
                    outcomes.putRow(i, outcomeRow);
                }
                //add left over to cache; need to ensure that only batch rows are returned
                /*
                 * Note that I'm aware of possible concerns for sentence sequencing.
                 * This is a hack right now in place of something
                 * that will be way more elegant in the future.
                 */
                if (windows.size() > batch) {
                    List<Window> leftOvers = windows.subList(batch, windows.size());
                    cache.addAll(leftOvers);
                }
                return new DataSet(input, outcomes);
            }

        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    }

    return null;
}

---

From source file:org.deeplearning4j.models.word2vec.wordstore.inmemory.InMemoryLookupCache.java

/**
 * Load a look up cache from an input stream
 * delimited by \n// w  w  w.  ja  v  a2  s .  co m
 * @param from the input stream to read from
 * @return the in memory lookup cache
 */
public static InMemoryLookupCache load(InputStream from) {
    Reader inputStream = new InputStreamReader(from);
    LineIterator iter = IOUtils.lineIterator(inputStream);
    String line;
    InMemoryLookupCache ret = new InMemoryLookupCache();
    int count = 0;
    while ((iter.hasNext())) {
        line = iter.nextLine();
        if (line.isEmpty())
            continue;
        ret.incrementWordCount(line);
        VocabWord word = new VocabWord(1.0, line);
        word.setIndex(count);
        ret.addToken(word);
        ret.addWordToIndex(count, line);
        ret.putVocabWord(line);
        count++;

    }

    return ret;
}

---

From source file:org.duracloud.syncui.controller.LogController.java

@RequestMapping(value = { "/log" }, params = "download")
public String download(HttpServletResponse response) throws IOException {
    log.debug("accessing log download page");
    StringBuffer contentDisposition = new StringBuffer();
    contentDisposition.append("attachment;filename=\"history.log\"");
    response.setHeader("Content-Disposition", contentDisposition.toString());
    File file = new File(syncConfigurationManager.getWorkDirectory().getAbsoluteFile() + "/logs/history.log");

    PrintWriter writer = response.getWriter();
    if (file.exists()) {
        LineIterator it = new LineIterator(new FileReader(file));
        while (it.hasNext()) {
            writer.write(it.nextLine() + "\n");
        }/*from  ww  w .  j  av a  2s . c om*/
    } else {
        writer.write("The history log is empty.");
    }

    return null;

}

---

From source file:org.ebayopensource.turmeric.eclipse.test.util.FunctionalTestHelper.java

@SuppressWarnings("unchecked")
public static String getConsumerFQN(IProject prj) {

    // String className = null;
    String className = "Consumer";
    NameFileFilter fileFilter = new NameFileFilter("Consumer.java");

    Collection<File> files = FileUtils.listFiles(prj.getLocation().toFile(), fileFilter,
            TrueFileFilter.INSTANCE);/* www.j  av a 2s  .  c o  m*/

    Assert.assertNotNull(files);
    Assert.assertTrue(files.size() > 0);

    File consFile = files.iterator().next();

    InputStream input = null;
    try {
        input = new FileInputStream(consFile);

        LineIterator iter = IOUtils.lineIterator(input, null);
        while (iter.hasNext()) {
            String line = iter.nextLine();
            if (line.startsWith("package")) {
                className = StringUtils.substringBetween(line, "package", ";").trim();
                className = className + ".Consumer";
                break;
            }
        }
        iter.close();
    } catch (Exception e) {
        e.printStackTrace();
        IOUtils.closeQuietly(input);
    }

    return className;

}

---

From source file:org.ebayopensource.turmeric.eclipse.test.utils.ServicesUtil.java

/**
 * Gets the consumer fqn./*w w  w . j  a v  a2s.c  om*/
 *
 * @param prj the prj
 * @return the consumer fqn
 */
@SuppressWarnings("unchecked")
public static String getConsumerFQN(IProject prj) {

    String className = null;
    NameFileFilter fileFilter = new NameFileFilter("TestConsumer.java");

    Collection<File> files = FileUtils.listFiles(prj.getLocation().toFile(), fileFilter,
            TrueFileFilter.INSTANCE);

    Assert.assertNotNull(files);
    Assert.assertTrue(files.size() > 0);

    File consFile = files.iterator().next();

    InputStream input = null;
    try {
        input = new FileInputStream(consFile);

        LineIterator iter = IOUtils.lineIterator(input, null);
        while (iter.hasNext()) {
            String line = iter.nextLine();
            if (line.startsWith("package")) {
                className = StringUtils.substringBetween(line, "package", ";").trim();
                className = className + ".TestConsumer";
                break;
            }
        }
        iter.close();
    } catch (Exception e) {
        e.printStackTrace();
        IOUtils.closeQuietly(input);
    }

    return className;

}

---

From source file:org.eclipse.orion.internal.server.search.FileGrepper.java

/**
 * Searches the contents of a file//from w  ww .j av  a2 s. c o  m
 * @param file The file to search
 * @return returns whether the search was successful
 * @throws IOException thrown if there is an error reading the file
 */
private boolean searchFile(File file) {
    LineIterator lineIterator = null;
    try {
        lineIterator = FileUtils.lineIterator(file);
    } catch (IOException e) {
        logger.error("FileGrepper.searchFile: " + e.getLocalizedMessage());
        return false;
    }
    try {
        while (lineIterator.hasNext()) {
            String line = lineIterator.nextLine();
            if (line.contains("\0")) {
                // file contains binary content
                return false;
            }
            matcher.reset(line);
            if (matcher.find()) {
                return true;
            }
        }
    } finally {
        if (lineIterator != null)
            lineIterator.close();
    }
    return false;
}

---

From source file:org.eclipse.orion.internal.server.search.grep.FileGrepper.java

/**
 * Searches the contents of a file/*from   w  ww  .j a  v a 2s . c om*/
 * @param file The file to search
 * @return returns whether the search was successful
 * @throws IOException thrown if there is an error reading the file
 */
private boolean searchFile(File file) {
    LineIterator lineIterator = null;
    try {
        lineIterator = FileUtils.lineIterator(file);
    } catch (IOException e) {
        logger.error("FileGrepper.searchFile: " + e.getLocalizedMessage());
        return false;
    }
    try {
        while (lineIterator.hasNext()) {
            String line = lineIterator.nextLine();
            matcher.reset(line);
            if (matcher.find()) {
                return true;
            }
        }
    } finally {
        if (lineIterator != null)
            lineIterator.close();
    }
    return false;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• »