Example usage for org.apache.lucene.analysis TokenStream reset

List of usage examples for org.apache.lucene.analysis TokenStream reset

  1. HOME
  2. Java
  3. org.apache.lucene
  4. org.apache.lucene.analysis.*
  5. TokenStream
  6. reset

Introduction

In this page you can find the example usage for org.apache.lucene.analysis TokenStream reset.

Prototype

public void reset() throws IOException

Source Link

Document

This method is called by a consumer before it begins consumption using #incrementToken() .

Usage

From source file:jp.sf.fess.solr.plugin.analysis.synonym.NGramSynonymTokenizerTest.java

License:Apache License

@Test
public void testSandwichStrExpand4() throws Exception {
    Analyzer a = new NGramSynonymTokenizerTestAnalyzer(4, true, "a,aa");
    TokenStream stream = a.tokenStream("f", new StringReader("aba"));
    stream.reset();
    assertTokenStream(stream, "a,0,1,1/aa,0,1,0/b,1,2,1/a,2,3,1/aa,2,3,0");

    a = new NGramSynonymTokenizerTestAnalyzer(4, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("abba"));
    stream.reset();//from   w  ww .j  a va2 s  .c  o m
    assertTokenStream(stream, "a,0,1,1/aa,0,1,0/b,1,2,1/bb,1,3,0/b,2,3,0/a,3,4,1/aa,3,4,0");

    a = new NGramSynonymTokenizerTestAnalyzer(4, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("abcda"));
    stream.reset();
    assertTokenStream(stream, "a,0,1,1/aa,0,1,0/b,1,2,1/bc,1,3,0/bcd,1,4,0/cd,2,4,0/d,3,4,0/a,4,5,1/aa,4,5,0");

    a = new NGramSynonymTokenizerTestAnalyzer(4, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("abcdea"));
    stream.reset();
    assertTokenStream(stream,
            "a,0,1,1/aa,0,1,0/b,1,2,1/bc,1,3,0/bcd,1,4,0/bcde,1,5,0/cde,2,5,0/de,3,5,0/e,4,5,0/a,5,6,1/aa,5,6,0");

    a = new NGramSynonymTokenizerTestAnalyzer(4, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("abcdefa"));
    stream.reset();
    assertTokenStream(stream,
            "a,0,1,1/aa,0,1,0/b,1,2,1/bc,1,3,0/bcd,1,4,0/bcde,1,5,0/cdef,2,6,1/def,3,6,0/ef,4,6,0/f,5,6,0/a,6,7,1/aa,6,7,0");
}

From source file:jp.sf.fess.solr.plugin.analysis.synonym.NGramSynonymTokenizerTest.java

License:Apache License

@Test
public void testSandwichSynonym1() throws Exception {
    Analyzer a = new NGramSynonymTokenizerTestAnalyzer(1, false, "a,aa");
    TokenStream stream = a.tokenStream("f", new StringReader("bab"));
    stream.reset();
    assertTokenStream(stream, "b,0,1,1/a,1,2,1/b,2,3,1");

    a = new NGramSynonymTokenizerTestAnalyzer(1, false, "a,aa");
    stream = a.tokenStream("f", new StringReader("bbabb"));
    stream.reset();/*from  www  .  j a  va  2 s .co m*/
    assertTokenStream(stream, "b,0,1,1/b,1,2,1/a,2,3,1/b,3,4,1/b,4,5,1");

    a = new NGramSynonymTokenizerTestAnalyzer(1, false, "a,aa");
    stream = a.tokenStream("f", new StringReader("dcbabcd"));
    stream.reset();
    assertTokenStream(stream, "d,0,1,1/c,1,2,1/b,2,3,1/a,3,4,1/b,4,5,1/c,5,6,1/d,6,7,1");

    a = new NGramSynonymTokenizerTestAnalyzer(1, false, "a,aa");
    stream = a.tokenStream("f", new StringReader("edcbabcde"));
    stream.reset();
    assertTokenStream(stream, "e,0,1,1/d,1,2,1/c,2,3,1/b,3,4,1/a,4,5,1/b,5,6,1/c,6,7,1/d,7,8,1/e,8,9,1");
}

From source file:jp.sf.fess.solr.plugin.analysis.synonym.NGramSynonymTokenizerTest.java

License:Apache License

@Test
public void testSandwichSynonym2() throws Exception {
    Analyzer a = new NGramSynonymTokenizerTestAnalyzer(2, false, "a,aa");
    TokenStream stream = a.tokenStream("f", new StringReader("bab"));
    stream.reset();
    assertTokenStream(stream, "b,0,1,1/a,1,2,1/b,2,3,1");

    a = new NGramSynonymTokenizerTestAnalyzer(2, false, "a,aa");
    stream = a.tokenStream("f", new StringReader("bbabb"));
    stream.reset();/*  ww  w .  j av  a2 s.c o m*/
    assertTokenStream(stream, "bb,0,2,1/a,2,3,1/bb,3,5,1");

    a = new NGramSynonymTokenizerTestAnalyzer(2, false, "a,aa");
    stream = a.tokenStream("f", new StringReader("dcbabcd"));
    stream.reset();
    assertTokenStream(stream, "dc,0,2,1/cb,1,3,1/a,3,4,1/bc,4,6,1/cd,5,7,1");

    a = new NGramSynonymTokenizerTestAnalyzer(2, false, "a,aa");
    stream = a.tokenStream("f", new StringReader("edcbabcde"));
    stream.reset();
    assertTokenStream(stream, "ed,0,2,1/dc,1,3,1/cb,2,4,1/a,4,5,1/bc,5,7,1/cd,6,8,1/de,7,9,1");
}

From source file:jp.sf.fess.solr.plugin.analysis.synonym.NGramSynonymTokenizerTest.java

License:Apache License

@Test
public void testSandwichSynonym3() throws Exception {
    Analyzer a = new NGramSynonymTokenizerTestAnalyzer(3, false, "a,aa");
    TokenStream stream = a.tokenStream("f", new StringReader("bab"));
    stream.reset();
    assertTokenStream(stream, "b,0,1,1/a,1,2,1/b,2,3,1");

    a = new NGramSynonymTokenizerTestAnalyzer(3, false, "a,aa");
    stream = a.tokenStream("f", new StringReader("bbabb"));
    stream.reset();/*from   w w  w  .ja v  a  2  s.co  m*/
    assertTokenStream(stream, "bb,0,2,1/a,2,3,1/bb,3,5,1");

    a = new NGramSynonymTokenizerTestAnalyzer(3, false, "a,aa");
    stream = a.tokenStream("f", new StringReader("dcbabcd"));
    stream.reset();
    assertTokenStream(stream, "dcb,0,3,1/a,3,4,1/bcd,4,7,1");

    a = new NGramSynonymTokenizerTestAnalyzer(3, false, "a,aa");
    stream = a.tokenStream("f", new StringReader("edcbabcde"));
    stream.reset();
    assertTokenStream(stream, "edc,0,3,1/dcb,1,4,1/a,4,5,1/bcd,5,8,1/cde,6,9,1");

    a = new NGramSynonymTokenizerTestAnalyzer(3, false, "a,aa");
    stream = a.tokenStream("f", new StringReader("fedcbabcdef"));
    stream.reset();
    assertTokenStream(stream, "fed,0,3,1/edc,1,4,1/dcb,2,5,1/a,5,6,1/bcd,6,9,1/cde,7,10,1/def,8,11,1");
}

From source file:jp.sf.fess.solr.plugin.analysis.synonym.NGramSynonymTokenizerTest.java

License:Apache License

@Test
public void testSandwichSynonym4() throws Exception {
    Analyzer a = new NGramSynonymTokenizerTestAnalyzer(4, false, "a,aa");
    TokenStream stream = a.tokenStream("f", new StringReader("bab"));
    stream.reset();
    assertTokenStream(stream, "b,0,1,1/a,1,2,1/b,2,3,1");

    a = new NGramSynonymTokenizerTestAnalyzer(4, false, "a,aa");
    stream = a.tokenStream("f", new StringReader("bbabb"));
    stream.reset();/*w w  w . ja  v  a 2  s. c  om*/
    assertTokenStream(stream, "bb,0,2,1/a,2,3,1/bb,3,5,1");

    a = new NGramSynonymTokenizerTestAnalyzer(4, false, "a,aa");
    stream = a.tokenStream("f", new StringReader("dcbabcd"));
    stream.reset();
    assertTokenStream(stream, "dcb,0,3,1/a,3,4,1/bcd,4,7,1");

    a = new NGramSynonymTokenizerTestAnalyzer(4, false, "a,aa");
    stream = a.tokenStream("f", new StringReader("edcbabcde"));
    stream.reset();
    assertTokenStream(stream, "edcb,0,4,1/a,4,5,1/bcde,5,9,1");

    a = new NGramSynonymTokenizerTestAnalyzer(4, false, "a,aa");
    stream = a.tokenStream("f", new StringReader("fedcbabcdef"));
    stream.reset();
    assertTokenStream(stream, "fedc,0,4,1/edcb,1,5,1/a,5,6,1/bcde,6,10,1/cdef,7,11,1");
}

From source file:jp.sf.fess.solr.plugin.analysis.synonym.NGramSynonymTokenizerTest.java

License:Apache License

@Test
public void testSandwichSynonymExpand1() throws Exception {
    Analyzer a = new NGramSynonymTokenizerTestAnalyzer(1, true, "a,aa");
    TokenStream stream = a.tokenStream("f", new StringReader("bab"));
    stream.reset();
    assertTokenStream(stream, "b,0,1,1/a,1,2,1/aa,1,2,0/b,2,3,1");

    a = new NGramSynonymTokenizerTestAnalyzer(1, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("bbabb"));
    stream.reset();//from w  w  w.  j  ava 2  s . com
    assertTokenStream(stream, "b,0,1,1/b,1,2,1/a,2,3,1/aa,2,3,0/b,3,4,1/b,4,5,1");

    a = new NGramSynonymTokenizerTestAnalyzer(1, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("dcbabcd"));
    stream.reset();
    assertTokenStream(stream, "d,0,1,1/c,1,2,1/b,2,3,1/a,3,4,1/aa,3,4,0/b,4,5,1/c,5,6,1/d,6,7,1");

    a = new NGramSynonymTokenizerTestAnalyzer(1, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("edcbabcde"));
    stream.reset();
    assertTokenStream(stream,
            "e,0,1,1/d,1,2,1/c,2,3,1/b,3,4,1/a,4,5,1/aa,4,5,0/b,5,6,1/c,6,7,1/d,7,8,1/e,8,9,1");
}

From source file:jp.sf.fess.solr.plugin.analysis.synonym.NGramSynonymTokenizerTest.java

License:Apache License

@Test
public void testSandwichSynonymExpand2() throws Exception {
    Analyzer a = new NGramSynonymTokenizerTestAnalyzer(2, true, "a,aa");
    TokenStream stream = a.tokenStream("f", new StringReader("bab"));
    stream.reset();
    assertTokenStream(stream, "b,0,1,1/a,1,2,1/aa,1,2,0/b,2,3,1");

    a = new NGramSynonymTokenizerTestAnalyzer(2, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("bbabb"));
    stream.reset();//from  w  w  w .  j av a2  s . co  m
    assertTokenStream(stream, "bb,0,2,1/b,1,2,0/a,2,3,1/aa,2,3,0/b,3,4,1/bb,3,5,0");

    a = new NGramSynonymTokenizerTestAnalyzer(2, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("dcbabcd"));
    stream.reset();
    assertTokenStream(stream, "dc,0,2,1/cb,1,3,1/b,2,3,0/a,3,4,1/aa,3,4,0/b,4,5,1/bc,4,6,0/cd,5,7,1");

    a = new NGramSynonymTokenizerTestAnalyzer(2, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("edcbabcde"));
    stream.reset();
    assertTokenStream(stream,
            "ed,0,2,1/dc,1,3,1/cb,2,4,1/b,3,4,0/a,4,5,1/aa,4,5,0/b,5,6,1/bc,5,7,0/cd,6,8,1/de,7,9,1");
}

From source file:jp.sf.fess.solr.plugin.analysis.synonym.NGramSynonymTokenizerTest.java

License:Apache License

@Test
public void testSandwichSynonymExpand3() throws Exception {
    Analyzer a = new NGramSynonymTokenizerTestAnalyzer(3, true, "a,aa");
    TokenStream stream = a.tokenStream("f", new StringReader("bab"));
    stream.reset();
    assertTokenStream(stream, "b,0,1,1/a,1,2,1/aa,1,2,0/b,2,3,1");

    a = new NGramSynonymTokenizerTestAnalyzer(3, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("bbabb"));
    stream.reset();//from  ww  w  .  j av a 2 s.  c om
    assertTokenStream(stream, "bb,0,2,1/b,1,2,0/a,2,3,1/aa,2,3,0/b,3,4,1/bb,3,5,0");

    a = new NGramSynonymTokenizerTestAnalyzer(3, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("dcbabcd"));
    stream.reset();
    assertTokenStream(stream, "dcb,0,3,1/cb,1,3,0/b,2,3,0/a,3,4,1/aa,3,4,0/b,4,5,1/bc,4,6,0/bcd,4,7,0");

    a = new NGramSynonymTokenizerTestAnalyzer(3, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("edcbabcde"));
    stream.reset();
    assertTokenStream(stream,
            "edc,0,3,1/dcb,1,4,1/cb,2,4,0/b,3,4,0/a,4,5,1/aa,4,5,0/b,5,6,1/bc,5,7,0/bcd,5,8,0/cde,6,9,1");

    a = new NGramSynonymTokenizerTestAnalyzer(3, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("fedcbabcdef"));
    stream.reset();
    assertTokenStream(stream,
            "fed,0,3,1/edc,1,4,1/dcb,2,5,1/cb,3,5,0/b,4,5,0/a,5,6,1/aa,5,6,0/b,6,7,1/bc,6,8,0/bcd,6,9,0/cde,7,10,1/def,8,11,1");
}

From source file:jp.sf.fess.solr.plugin.analysis.synonym.NGramSynonymTokenizerTest.java

License:Apache License

@Test
public void testSandwichSynonymExpand4() throws Exception {
    Analyzer a = new NGramSynonymTokenizerTestAnalyzer(4, true, "a,aa");
    TokenStream stream = a.tokenStream("f", new StringReader("bab"));
    stream.reset();
    assertTokenStream(stream, "b,0,1,1/a,1,2,1/aa,1,2,0/b,2,3,1");

    a = new NGramSynonymTokenizerTestAnalyzer(4, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("bbabb"));
    stream.reset();/*  ww w . ja v a 2 s .  c  o  m*/
    assertTokenStream(stream, "bb,0,2,1/b,1,2,0/a,2,3,1/aa,2,3,0/b,3,4,1/bb,3,5,0");

    a = new NGramSynonymTokenizerTestAnalyzer(4, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("dcbabcd"));
    stream.reset();
    assertTokenStream(stream, "dcb,0,3,1/cb,1,3,0/b,2,3,0/a,3,4,1/aa,3,4,0/b,4,5,1/bc,4,6,0/bcd,4,7,0");

    a = new NGramSynonymTokenizerTestAnalyzer(4, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("edcbabcde"));
    stream.reset();
    assertTokenStream(stream,
            "edcb,0,4,1/dcb,1,4,0/cb,2,4,0/b,3,4,0/a,4,5,1/aa,4,5,0/b,5,6,1/bc,5,7,0/bcd,5,8,0/bcde,5,9,0");

    a = new NGramSynonymTokenizerTestAnalyzer(4, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("fedcbabcdef"));
    stream.reset();
    assertTokenStream(stream,
            "fedc,0,4,1/edcb,1,5,1/dcb,2,5,0/cb,3,5,0/b,4,5,0/a,5,6,1/aa,5,6,0/b,6,7,1/bc,6,8,0/bcd,6,9,0/bcde,6,10,0/cdef,7,11,1");
}

From source file:jp.sf.fess.solr.plugin.analysis.synonym.NGramSynonymTokenizerTest.java

License:Apache License

@Test
public void testComplex1() throws Exception {
    Analyzer a = new NGramSynonymTokenizerTestAnalyzer(1, false, "a,aa/b,bb");
    TokenStream stream = a.tokenStream("f", new StringReader("cabca"));
    stream.reset();
    assertTokenStream(stream, "c,0,1,1/a,1,2,1/b,2,3,1/c,3,4,1/a,4,5,1");

    a = new NGramSynonymTokenizerTestAnalyzer(1, false, "a,aa/b,bb");
    stream = a.tokenStream("f", new StringReader("ccabcca"));
    stream.reset();//  w  ww . j  a v a 2s  . c om
    assertTokenStream(stream, "c,0,1,1/c,1,2,1/a,2,3,1/b,3,4,1/c,4,5,1/c,5,6,1/a,6,7,1");

    a = new NGramSynonymTokenizerTestAnalyzer(1, false, "a,aa/b,bb");
    stream = a.tokenStream("f", new StringReader("edcabcdea"));
    stream.reset();
    assertTokenStream(stream, "e,0,1,1/d,1,2,1/c,2,3,1/a,3,4,1/b,4,5,1/c,5,6,1/d,6,7,1/e,7,8,1/a,8,9,1");

    a = new NGramSynonymTokenizerTestAnalyzer(1, false, "a,aa/b,bb");
    stream = a.tokenStream("f", new StringReader("fedcabcdefa"));
    stream.reset();
    assertTokenStream(stream,
            "f,0,1,1/e,1,2,1/d,2,3,1/c,3,4,1/a,4,5,1/b,5,6,1/c,6,7,1/d,7,8,1/e,8,9,1/f,9,10,1/a,10,11,1");
}