Example usage for org.apache.lucene.analysis TokenFilter close

Introduction

In this page you can find the example usage for org.apache.lucene.analysis TokenFilter close.

Prototype

@Override
public void close() throws IOException 

Source Link

Document

NOTE: The default implementation chains the call to the input TokenStream, so be sure to call super.close() when overriding this method.

Usage

From source file:com.sindicetech.siren.analysis.filter.TestMailtoFilter.java

License:Open Source License

private void assertURLDecodedTo(final Tokenizer t, final String uri, final String[] expectedStems,
        final String[] expectedTypes, final int[] expectedPosIncr) throws IOException {
    assertTrue("has CharTermAttribute", t.hasAttribute(CharTermAttribute.class));
    final CharTermAttribute termAtt = t.getAttribute(CharTermAttribute.class);

    assertTrue("has TypeAttribute", t.hasAttribute(TypeAttribute.class));
    final TypeAttribute typeAtt = t.getAttribute(TypeAttribute.class);

    assertTrue("has PositionIncrementAttribute", t.hasAttribute(PositionIncrementAttribute.class));
    final PositionIncrementAttribute posIncrAtt = t.getAttribute(PositionIncrementAttribute.class);

    t.setReader(new StringReader(uri));
    t.reset();/*from ww w.  j  a  va  2  s  .  c  om*/

    final TokenFilter filter = new MailtoFilter(t);
    for (int i = 0; i < expectedStems.length; i++) {
        assertTrue("token " + i + " exists", filter.incrementToken());
        assertEquals(expectedStems[i], termAtt.toString());
        if (expectedTypes == null)
            assertEquals(uritype, typeAtt.type());
        else
            assertEquals(expectedTypes[i], typeAtt.type());
        if (expectedPosIncr != null)
            assertEquals(expectedPosIncr[i], posIncrAtt.getPositionIncrement());
    }
    filter.end();
    filter.close();
}

---

From source file:org.edits.LuceneTokenizer.java

License:Open Source License

@Override
public List<Annotation> annotate(String text) throws Exception {
    text = SimpleTokenizer.format(text);
    Analyzer analyser = new EnglishAnalyzer(Version.LUCENE_47, CharArraySet.EMPTY_SET);
    TokenFilter filter = new EnglishMinimalStemFilter(analyser.tokenStream("text", new StringReader(text)));
    List<Annotation> out = Lists.newArrayList();
    while (filter.incrementToken()) {
        CharTermAttribute az = filter.getAttribute(CharTermAttribute.class);
        OffsetAttribute o = filter.getAttribute(OffsetAttribute.class);
        String token = text.substring(o.startOffset(), o.endOffset());
        String lemma = az.toString();
        Annotation t = new Annotation();
        t.setForm(token);/*from   w ww.  j av a  2 s.  c  om*/
        t.setLemma(lemma);
        out.add(t);
    }
    if (out.size() == 0) {
        log.debug("Input string is empty");
    }
    filter.close();
    analyser.close();
    return out;
}

---

From source file:org.sindice.siren.qparser.analysis.QNamesFilterTest.java

License:Apache License

@Test
public void testQName() throws Exception {
    final String query = "foaf:name";
    final WhitespaceAnalyzer analyzer = new WhitespaceAnalyzer(Version.LUCENE_31);
    final TokenStream stream = analyzer.tokenStream(null, new StringReader(query));
    final TokenFilter filter = new QNamesFilter(stream, "./src/test/resources/conf/qnames");

    final CharTermAttribute cTermAtt = filter.getAttribute(CharTermAttribute.class);
    Assert.assertTrue(filter.incrementToken());
    Assert.assertEquals("http://xmlns.com/foaf/0.1/name", cTermAtt.toString());
    filter.close();
}

---

From source file:org.sindice.siren.qparser.analysis.QNamesFilterTest.java

License:Apache License

@Test
public void testNotQName() throws Exception {
    final String query = "mailto:aidan.hogan@deri.org";
    final WhitespaceAnalyzer analyzer = new WhitespaceAnalyzer(Version.LUCENE_31);
    final TokenStream stream = analyzer.tokenStream(null, new StringReader(query));
    final TokenFilter filter = new QNamesFilter(stream, "./src/test/resources/conf/qnames");

    final CharTermAttribute cTermAtt = filter.getAttribute(CharTermAttribute.class);
    Assert.assertTrue(filter.incrementToken());
    Assert.assertEquals("mailto:aidan.hogan@deri.org", cTermAtt.toString());
    filter.close();
}

---