Example usage for org.apache.commons.io IOUtils lineIterator

Introduction

In this page you can find the example usage for org.apache.commons.io IOUtils lineIterator.

Prototype

public static LineIterator lineIterator(InputStream input, String encoding) throws IOException 

Source Link

Document

Return an Iterator for the lines in an InputStream, using the character encoding specified (or default encoding if null).

Usage

From source file:org.collectionspace.chain.csp.webui.main.StreamUIRequest.java

public StreamUIRequest(InputStream in, OutputStream out, OutputStream err, Operation request, String[] path,
        Map<String, String> args) throws IOException {
    this.ibody = in;
    this.in = IOUtils.lineIterator(in, "UTF-8");
    this.out = out;
    this.err = err;
    this.path = path;
    this.request = request;
    if (args != null) {
        qargs = new HashMap<String, String>(args);
    }//from w w  w . j a  va  2  s .  c o m
}

---

From source file:org.dataconservancy.access.connector.HttpRegistryConnector.java

private Set<URL> getReferences(String endpoint) throws DcsClientFault {
    Set<URL> references = new HashSet<URL>();
    LineIterator lineItr = null;/* w w w.  jav a2s.  co m*/
    InputStream contentStream = null;
    try {
        HttpResponse response = execute(new HttpGet(endpoint), 200);
        contentStream = response.getEntity().getContent();
        lineItr = IOUtils.lineIterator(contentStream, "UTF-8");
    } catch (IOException e) {
        try {
            if (contentStream != null) {
                contentStream.close();
            }
        } catch (IOException ioe) {
            // ignore
        }
        String msg = "Unable to read inputstream for URL " + endpoint;
        log.debug(msg);
        throw new DcsServerException(msg, e);
    }

    try {
        while (lineItr.hasNext()) {
            String regUrl = lineItr.nextLine();
            try {
                references.add(new URL(regUrl));
            } catch (MalformedURLException e) {
                log.debug("Received invalid registry URL {}", regUrl);
            }
        }
    } finally {
        try {
            contentStream.close();
        } catch (IOException e) {
            // ignore
        }
        lineItr.close();
    }

    return references;
}

---

From source file:org.deeplearning4j.models.glove.Glove.java

/**
 * Load a glove model from an input stream.
 * The format is:/*w w  w  . jav a2  s.  c o m*/
 * word num1 num2....
 * @param is the input stream to read from for the weights
 * @param biases the bias input stream
 * @return the loaded model
 * @throws IOException if one occurs
 */
public static Glove load(InputStream is, InputStream biases) throws IOException {
    LineIterator iter = IOUtils.lineIterator(is, "UTF-8");
    Glove glove = new Glove();
    Map<String, float[]> wordVectors = new HashMap<>();
    int count = 0;
    while (iter.hasNext()) {
        String line = iter.nextLine().trim();
        if (line.isEmpty())
            continue;
        String[] split = line.split(" ");
        String word = split[0];
        if (glove.vocab() == null)
            glove.setVocab(new InMemoryLookupCache());

        if (glove.lookupTable() == null) {
            glove.lookupTable = new GloveWeightLookupTable.Builder().cache(glove.vocab())
                    .vectorLength(split.length - 1).build();

        }

        if (word.isEmpty())
            continue;
        float[] read = read(split, glove.lookupTable().getVectorLength());
        if (read.length < 1)
            continue;

        VocabWord w1 = new VocabWord(1, word);
        w1.setIndex(count);
        glove.vocab().addToken(w1);
        glove.vocab().addWordToIndex(count, word);
        glove.vocab().putVocabWord(word);
        wordVectors.put(word, read);
        count++;

    }

    glove.lookupTable().setSyn0(weights(glove, wordVectors));

    iter.close();

    glove.lookupTable().setBias(Nd4j.read(biases));

    return glove;

}

---

From source file:org.deeplearning4j.models.glove.GloveWeightLookupTable.java

/**
 * Load a glove model from an input stream.
 * The format is://w  w  w . j  av a2 s. co  m
 * word num1 num2....
 * @param is the input stream to read from for the weights
 * @param vocab the vocab for the lookuptable
 * @return the loaded model
 * @throws java.io.IOException if one occurs
 */
public static GloveWeightLookupTable load(InputStream is, VocabCache<? extends SequenceElement> vocab)
        throws IOException {
    LineIterator iter = IOUtils.lineIterator(is, "UTF-8");
    GloveWeightLookupTable glove = null;
    Map<String, float[]> wordVectors = new HashMap<>();
    while (iter.hasNext()) {
        String line = iter.nextLine().trim();
        if (line.isEmpty())
            continue;
        String[] split = line.split(" ");
        String word = split[0];
        if (glove == null)
            glove = new GloveWeightLookupTable.Builder().cache(vocab).vectorLength(split.length - 1).build();

        if (word.isEmpty())
            continue;
        float[] read = read(split, glove.layerSize());
        if (read.length < 1)
            continue;

        wordVectors.put(word, read);

    }

    glove.setSyn0(weights(glove, wordVectors, vocab));
    glove.resetWeights(false);

    iter.close();

    return glove;

}

---

From source file:org.deeplearning4j.models.glove.LegacyGlove.java

/**
 * Load a glove model from an input stream.
 * The format is://  w  ww. j  a v a2s .  c o  m
 * word num1 num2....
 * @param is the input stream to read from for the weights
 * @param biases the bias input stream
 * @return the loaded model
 * @throws IOException if one occurs
 */
public static LegacyGlove load(InputStream is, InputStream biases) throws IOException {
    LineIterator iter = IOUtils.lineIterator(is, "UTF-8");
    LegacyGlove glove = new LegacyGlove();
    Map<String, float[]> wordVectors = new HashMap<>();
    int count = 0;
    while (iter.hasNext()) {
        String line = iter.nextLine().trim();
        if (line.isEmpty())
            continue;
        String[] split = line.split(" ");
        String word = split[0];
        if (glove.vocab() == null)
            glove.setVocab(new InMemoryLookupCache());

        if (glove.lookupTable() == null) {
            glove.lookupTable = new GloveWeightLookupTable.Builder().cache(glove.vocab())
                    .vectorLength(split.length - 1).build();

        }

        if (word.isEmpty())
            continue;
        float[] read = read(split, glove.lookupTable().layerSize());
        if (read.length < 1)
            continue;

        VocabWord w1 = new VocabWord(1, word);
        w1.setIndex(count);
        glove.vocab().addToken(w1);
        glove.vocab().addWordToIndex(count, word);
        glove.vocab().putVocabWord(word);
        wordVectors.put(word, read);
        count++;

    }

    glove.lookupTable().setSyn0(weights(glove, wordVectors));

    iter.close();

    glove.lookupTable().setBias(Nd4j.read(biases));

    return glove;

}

---

From source file:org.deeplearning4j.text.sentenceiterator.FileSentenceIterator.java

private void nextLineIter() {
    if (fileIterator.hasNext()) {
        try {//  w  ww .  j ava2s  .co m
            File next = fileIterator.next();
            currentFile = next;
            if (next.getAbsolutePath().endsWith(".gz")) {
                if (currLineIterator != null)
                    currLineIterator.close();
                currLineIterator = IOUtils.lineIterator(
                        new BufferedInputStream(new GZIPInputStream(new FileInputStream(next))), "UTF-8");

            } else {
                if (currLineIterator != null) {
                    currLineIterator.close();
                }
                currLineIterator = FileUtils.lineIterator(next);

            }
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }
}

---

From source file:org.deeplearning4j.text.sentenceiterator.LineSentenceIterator.java

public LineSentenceIterator(File f) {
    if (!f.exists() || !f.isFile())
        throw new IllegalArgumentException("Please specify an existing file");
    try {//  ww w . j av a 2  s  . c o m
        this.f = f;
        this.file = new BufferedInputStream(new FileInputStream(f));
        iter = IOUtils.lineIterator(this.file, "UTF-8");
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

---

From source file:org.deeplearning4j.text.sentenceiterator.LineSentenceIterator.java

@Override
public void reset() {
    try {/*from  w  w w .j  a va 2  s .  c o m*/
        if (file != null)
            file.close();
        if (iter != null)
            iter.close();
        this.file = new BufferedInputStream(new FileInputStream(f));
        iter = IOUtils.lineIterator(this.file, "UTF-8");
    } catch (IOException e) {
        throw new RuntimeException(e);
    }

}

---

From source file:org.ebayopensource.turmeric.eclipse.test.util.FunctionalTestHelper.java

@SuppressWarnings("unchecked")
public static String getConsumerFQN(IProject prj) {

    // String className = null;
    String className = "Consumer";
    NameFileFilter fileFilter = new NameFileFilter("Consumer.java");

    Collection<File> files = FileUtils.listFiles(prj.getLocation().toFile(), fileFilter,
            TrueFileFilter.INSTANCE);/*from w ww  . j a  va2s.  c o m*/

    Assert.assertNotNull(files);
    Assert.assertTrue(files.size() > 0);

    File consFile = files.iterator().next();

    InputStream input = null;
    try {
        input = new FileInputStream(consFile);

        LineIterator iter = IOUtils.lineIterator(input, null);
        while (iter.hasNext()) {
            String line = iter.nextLine();
            if (line.startsWith("package")) {
                className = StringUtils.substringBetween(line, "package", ";").trim();
                className = className + ".Consumer";
                break;
            }
        }
        iter.close();
    } catch (Exception e) {
        e.printStackTrace();
        IOUtils.closeQuietly(input);
    }

    return className;

}

---

From source file:org.ebayopensource.turmeric.eclipse.test.utils.ServicesUtil.java

/**
 * Gets the consumer fqn./*from ww w.  ja v a 2  s. co m*/
 *
 * @param prj the prj
 * @return the consumer fqn
 */
@SuppressWarnings("unchecked")
public static String getConsumerFQN(IProject prj) {

    String className = null;
    NameFileFilter fileFilter = new NameFileFilter("TestConsumer.java");

    Collection<File> files = FileUtils.listFiles(prj.getLocation().toFile(), fileFilter,
            TrueFileFilter.INSTANCE);

    Assert.assertNotNull(files);
    Assert.assertTrue(files.size() > 0);

    File consFile = files.iterator().next();

    InputStream input = null;
    try {
        input = new FileInputStream(consFile);

        LineIterator iter = IOUtils.lineIterator(input, null);
        while (iter.hasNext()) {
            String line = iter.nextLine();
            if (line.startsWith("package")) {
                className = StringUtils.substringBetween(line, "package", ";").trim();
                className = className + ".TestConsumer";
                break;
            }
        }
        iter.close();
    } catch (Exception e) {
        e.printStackTrace();
        IOUtils.closeQuietly(input);
    }

    return className;

}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• »