Example usage for org.apache.lucene.analysis TokenStream TokenStream

Introduction

In this page you can find the example usage for org.apache.lucene.analysis TokenStream TokenStream.

Prototype

protected TokenStream() 

Source Link

Document

A TokenStream using the default attribute factory.

Usage

From source file:org.apache.solr.analysis.TestWordDelimiterFilter.java

License:Apache License

public void testOffsetChange4() throws Exception {
    WordDelimiterFilter wdf = new WordDelimiterFilter(new TokenStream() {
        private Token t;

        public Token next() {
            if (t != null)
                return null;
            t = new Token("(foo,bar)", 7, 16);
            return t;
        }/*w w  w.  j ava  2 s.c o m*/
    }, 1, 1, 0, 0, 1, 1, 0);

    Token t = wdf.next();

    assertNotNull(t);
    assertEquals("foo", t.term());
    assertEquals(8, t.startOffset());
    assertEquals(11, t.endOffset());

    t = wdf.next();

    assertNotNull(t);
    assertEquals("bar", t.term());
    assertEquals(12, t.startOffset());
    assertEquals(15, t.endOffset());
}

---

From source file:org.apache.solr.analysis.TestWordDelimiterFilter.java

License:Apache License

public void doSplit(final String input, String... output) throws Exception {
    WordDelimiterFilter wdf = new WordDelimiterFilter(new TokenStream() {
        boolean done = false;

        @Override/*from w  w w  .j  ava 2 s.  c  om*/
        public Token next() throws IOException {
            if (done)
                return null;
            done = true;
            return new Token(input, 0, input.length());
        }
    }, 1, 1, 0, 0, 0);

    for (String expected : output) {
        Token t = wdf.next();
        assertEquals(expected, t.term());
    }

    assertEquals(null, wdf.next());
}

---

From source file:org.apache.solr.analysis.TestWordDelimiterFilter.java

License:Apache License

public void doSplitPossessive(int stemPossessive, final String input, final String... output) throws Exception {
    WordDelimiterFilter wdf = new WordDelimiterFilter(new TokenStream() {
        boolean done = false;

        @Override/*  w  w w.ja  va 2  s . co m*/
        public Token next() throws IOException {
            if (done)
                return null;
            done = true;
            return new Token(input, 0, input.length());
        }
    }, 1, 1, 0, 0, 0, 1, 0, 1, stemPossessive, null);

    for (String expected : output) {
        Token t = wdf.next();
        assertEquals(expected, t.term());
    }

    assertEquals(null, wdf.next());
}

---

• «
• 1
• 2
• »