Example usage for opennlp.tools.languagemodel NGramLanguageModel NGramLanguageModel

Introduction

In this page you can find the example usage for opennlp.tools.languagemodel NGramLanguageModel NGramLanguageModel.

Prototype

public NGramLanguageModel(InputStream in) throws IOException 

Source Link

Usage

From source file:opennlp.tools.languagemodel.NgramLanguageModelTest.java

@Test
public void testNgramModel() throws Exception {
    NGramLanguageModel model = new NGramLanguageModel(4);
    model.add(new StringList("I", "saw", "the", "fox"), 1, 4);
    model.add(new StringList("the", "red", "house"), 1, 4);
    model.add(new StringList("I", "saw", "something", "nice"), 1, 2);
    double probability = model.calculateProbability(new StringList("I", "saw", "the", "red", "house"));
    Assert.assertTrue("a probability measure should be between 0 and 1 [was " + probability + "]",
            probability >= 0 && probability <= 1);

    StringList tokens = model.predictNextTokens(new StringList("I", "saw"));
    Assert.assertNotNull(tokens);//from w  w w  .  j a  v  a  2  s . co m
    Assert.assertEquals(new StringList("the", "fox"), tokens);
}

---

From source file:opennlp.tools.languagemodel.NgramLanguageModelTest.java

@Test
public void testTrigram() throws Exception {
    NGramLanguageModel model = new NGramLanguageModel(3);
    model.add(new StringList("I", "see", "the", "fox"), 2, 3);
    model.add(new StringList("the", "red", "house"), 2, 3);
    model.add(new StringList("I", "saw", "something", "nice"), 2, 3);
    double probability = model.calculateProbability(new StringList("I", "saw", "the", "red", "house"));
    Assert.assertTrue("a probability measure should be between 0 and 1 [was " + probability + "]",
            probability >= 0 && probability <= 1);

    StringList tokens = model.predictNextTokens(new StringList("I", "saw"));
    Assert.assertNotNull(tokens);//from  w  w  w .ja v  a2 s.  co  m
    Assert.assertEquals(new StringList("something", "nice"), tokens);
}

---

From source file:opennlp.tools.languagemodel.NgramLanguageModelTest.java

@Test
public void testBigram() throws Exception {
    NGramLanguageModel model = new NGramLanguageModel(2);
    model.add(new StringList("I", "see", "the", "fox"), 1, 2);
    model.add(new StringList("the", "red", "house"), 1, 2);
    model.add(new StringList("I", "saw", "something", "nice"), 1, 2);
    double probability = model.calculateProbability(new StringList("I", "saw", "the", "red", "house"));
    Assert.assertTrue("a probability measure should be between 0 and 1 [was " + probability + "]",
            probability >= 0 && probability <= 1);

    StringList tokens = model.predictNextTokens(new StringList("I", "saw"));
    Assert.assertNotNull(tokens);/*from   ww  w . java  2 s . c  o  m*/
    Assert.assertEquals(new StringList("something"), tokens);
}

---

From source file:opennlp.tools.languagemodel.NgramLanguageModelTest.java

@Test
public void testTrigramLanguageModelCreationFromText() throws Exception {
    int ngramSize = 3;
    NGramLanguageModel languageModel = new NGramLanguageModel(ngramSize);
    InputStream stream = getClass().getResourceAsStream("/opennlp/tools/languagemodel/sentences.txt");
    for (String line : IOUtils.readLines(stream)) {
        String[] array = line.split(" ");
        List<String> split = Arrays.asList(array);
        List<String> generatedStrings = NGramGenerator.generate(split, ngramSize, " ");
        for (String generatedString : generatedStrings) {
            String[] tokens = generatedString.split(" ");
            if (tokens.length > 0) {
                languageModel.add(new StringList(tokens), 1, ngramSize);
            }/*from w  ww .ja v  a2s.c o  m*/
        }
    }
    StringList tokens = languageModel.predictNextTokens(new StringList("neural", "network", "language"));
    Assert.assertNotNull(tokens);
    Assert.assertEquals(new StringList("models"), tokens);
    double p1 = languageModel.calculateProbability(new StringList("neural", "network", "language", "models"));
    double p2 = languageModel.calculateProbability(new StringList("neural", "network", "language", "model"));
    Assert.assertTrue(p1 > p2);
}

---