Example usage for opennlp.tools.util StringList StringList

Introduction

In this page you can find the example usage for opennlp.tools.util StringList StringList.

Prototype

public StringList(String... tokens) 

Source Link

Document

Initializes the current instance.

Usage

From source file:opennlp.tools.languagemodel.NgramLanguageModelTest.java

@Test
public void testEmptyVocabularyProbability() throws Exception {
    NGramLanguageModel model = new NGramLanguageModel();
    Assert.assertEquals("probability with an empty vocabulary is always 0", 0d,
            model.calculateProbability(new StringList("")), 0d);
    Assert.assertEquals("probability with an empty vocabulary is always 0", 0d,
            model.calculateProbability(new StringList("1", "2", "3")), 0d);
}

---

From source file:opennlp.tools.languagemodel.NgramLanguageModelTest.java

@Test
public void testTrigramLanguageModelCreationFromText() throws Exception {
    int ngramSize = 3;
    NGramLanguageModel languageModel = new NGramLanguageModel(ngramSize);
    InputStream stream = getClass().getResourceAsStream("/opennlp/tools/languagemodel/sentences.txt");
    for (String line : IOUtils.readLines(stream)) {
        String[] array = line.split(" ");
        List<String> split = Arrays.asList(array);
        List<String> generatedStrings = NGramGenerator.generate(split, ngramSize, " ");
        for (String generatedString : generatedStrings) {
            String[] tokens = generatedString.split(" ");
            if (tokens.length > 0) {
                languageModel.add(new StringList(tokens), 1, ngramSize);
            }/*from   www .  jav a 2  s.c  o m*/
        }
    }
    StringList tokens = languageModel.predictNextTokens(new StringList("neural", "network", "language"));
    Assert.assertNotNull(tokens);
    Assert.assertEquals(new StringList("models"), tokens);
    double p1 = languageModel.calculateProbability(new StringList("neural", "network", "language", "models"));
    double p2 = languageModel.calculateProbability(new StringList("neural", "network", "language", "model"));
    Assert.assertTrue(p1 > p2);
}

---

From source file:opennlp.tools.ngram.NGramModelTest.java

@Test
public void testZeroGetCount() throws Exception {
    NGramModel ngramModel = new NGramModel();
    int count = ngramModel.getCount(new StringList(""));
    Assert.assertEquals(0, count);/*from   w  w  w .  ja  va2s  . c  o  m*/
    Assert.assertEquals(0, ngramModel.size());
}

---