Example usage for org.apache.lucene.analysis TokenStream reset

List of usage examples for org.apache.lucene.analysis TokenStream reset

  1. HOME
  2. Java
  3. org.apache.lucene
  4. org.apache.lucene.analysis.*
  5. TokenStream
  6. reset

Introduction

In this page you can find the example usage for org.apache.lucene.analysis TokenStream reset.

Prototype

public void reset() throws IOException

Source Link

Document

This method is called by a consumer before it begins consumption using #incrementToken() .

Usage

From source file:jp.sf.fess.solr.plugin.analysis.synonym.NGramSynonymTokenizerTest.java

License:Apache License

@Test
public void testPrevStrSingleSynonym4() throws Exception {
    Analyzer a = new NGramSynonymTokenizerTestAnalyzer(4, false, "a,aa");
    TokenStream stream = a.tokenStream("f", new StringReader("ba"));
    stream.reset();
    assertTokenStream(stream, "b,0,1,1/a,1,2,1");

    a = new NGramSynonymTokenizerTestAnalyzer(4, false, "a,aa");
    stream = a.tokenStream("f", new StringReader("bba"));
    stream.reset();/*from   ww  w  .  j a  va2  s .  c  om*/
    assertTokenStream(stream, "bb,0,2,1/a,2,3,1");

    a = new NGramSynonymTokenizerTestAnalyzer(4, false, "a,aa");
    stream = a.tokenStream("f", new StringReader("dcba"));
    stream.reset();
    assertTokenStream(stream, "dcb,0,3,1/a,3,4,1");

    a = new NGramSynonymTokenizerTestAnalyzer(4, false, "a,aa");
    stream = a.tokenStream("f", new StringReader("edcba"));
    stream.reset();
    assertTokenStream(stream, "edcb,0,4,1/a,4,5,1");

    a = new NGramSynonymTokenizerTestAnalyzer(4, false, "a,aa");
    stream = a.tokenStream("f", new StringReader("fedcba"));
    stream.reset();
    assertTokenStream(stream, "fedc,0,4,1/edcb,1,5,1/a,5,6,1");
}

From source file:jp.sf.fess.solr.plugin.analysis.synonym.NGramSynonymTokenizerTest.java

License:Apache License

@Test
public void testPrevStrSingleSynonymExpand1() throws Exception {
    Analyzer a = new NGramSynonymTokenizerTestAnalyzer(1, true, "a,aa");
    TokenStream stream = a.tokenStream("f", new StringReader("ba"));
    stream.reset();
    assertTokenStream(stream, "b,0,1,1/a,1,2,1/aa,1,2,0");

    a = new NGramSynonymTokenizerTestAnalyzer(1, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("bba"));
    stream.reset();// w w  w  .  java 2 s . co m
    assertTokenStream(stream, "b,0,1,1/b,1,2,1/a,2,3,1/aa,2,3,0");

    a = new NGramSynonymTokenizerTestAnalyzer(1, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("dcba"));
    stream.reset();
    assertTokenStream(stream, "d,0,1,1/c,1,2,1/b,2,3,1/a,3,4,1/aa,3,4,0");

    a = new NGramSynonymTokenizerTestAnalyzer(1, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("edcba"));
    stream.reset();
    assertTokenStream(stream, "e,0,1,1/d,1,2,1/c,2,3,1/b,3,4,1/a,4,5,1/aa,4,5,0");
}

From source file:jp.sf.fess.solr.plugin.analysis.synonym.NGramSynonymTokenizerTest.java

License:Apache License

@Test
public void testPrevStrSingleSynonymExpand2() throws Exception {
    Analyzer a = new NGramSynonymTokenizerTestAnalyzer(2, true, "a,aa");
    TokenStream stream = a.tokenStream("f", new StringReader("ba"));
    stream.reset();
    assertTokenStream(stream, "b,0,1,1/a,1,2,1/aa,1,2,0");

    a = new NGramSynonymTokenizerTestAnalyzer(2, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("bba"));
    stream.reset();/*from   w w  w .  j a  v  a 2 s . c  o m*/
    assertTokenStream(stream, "bb,0,2,1/b,1,2,0/a,2,3,1/aa,2,3,0");

    a = new NGramSynonymTokenizerTestAnalyzer(2, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("dcba"));
    stream.reset();
    assertTokenStream(stream, "dc,0,2,1/cb,1,3,1/b,2,3,0/a,3,4,1/aa,3,4,0");

    a = new NGramSynonymTokenizerTestAnalyzer(2, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("edcba"));
    stream.reset();
    assertTokenStream(stream, "ed,0,2,1/dc,1,3,1/cb,2,4,1/b,3,4,0/a,4,5,1/aa,4,5,0");
}

From source file:jp.sf.fess.solr.plugin.analysis.synonym.NGramSynonymTokenizerTest.java

License:Apache License

@Test
public void testPrevStrSingleSynonymExpand3() throws Exception {
    Analyzer a = new NGramSynonymTokenizerTestAnalyzer(3, true, "a,aa");
    TokenStream stream = a.tokenStream("f", new StringReader("ba"));
    stream.reset();
    assertTokenStream(stream, "b,0,1,1/a,1,2,1/aa,1,2,0");

    a = new NGramSynonymTokenizerTestAnalyzer(3, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("bba"));
    stream.reset();/*  ww w  .j av a2s.  c o m*/
    assertTokenStream(stream, "bb,0,2,1/b,1,2,0/a,2,3,1/aa,2,3,0");

    a = new NGramSynonymTokenizerTestAnalyzer(3, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("dcba"));
    stream.reset();
    assertTokenStream(stream, "dcb,0,3,1/cb,1,3,0/b,2,3,0/a,3,4,1/aa,3,4,0");

    a = new NGramSynonymTokenizerTestAnalyzer(3, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("edcba"));
    stream.reset();
    assertTokenStream(stream, "edc,0,3,1/dcb,1,4,1/cb,2,4,0/b,3,4,0/a,4,5,1/aa,4,5,0");
}

From source file:jp.sf.fess.solr.plugin.analysis.synonym.NGramSynonymTokenizerTest.java

License:Apache License

@Test
public void testPrevStrSingleSynonymExpand4() throws Exception {
    Analyzer a = new NGramSynonymTokenizerTestAnalyzer(4, true, "a,aa");
    TokenStream stream = a.tokenStream("f", new StringReader("ba"));
    stream.reset();
    assertTokenStream(stream, "b,0,1,1/a,1,2,1/aa,1,2,0");

    a = new NGramSynonymTokenizerTestAnalyzer(4, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("bba"));
    stream.reset();/*from ww w .ja va  2 s.c o  m*/
    assertTokenStream(stream, "bb,0,2,1/b,1,2,0/a,2,3,1/aa,2,3,0");

    a = new NGramSynonymTokenizerTestAnalyzer(4, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("dcba"));
    stream.reset();
    assertTokenStream(stream, "dcb,0,3,1/cb,1,3,0/b,2,3,0/a,3,4,1/aa,3,4,0");

    a = new NGramSynonymTokenizerTestAnalyzer(4, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("edcba"));
    stream.reset();
    assertTokenStream(stream, "edcb,0,4,1/dcb,1,4,0/cb,2,4,0/b,3,4,0/a,4,5,1/aa,4,5,0");

    a = new NGramSynonymTokenizerTestAnalyzer(4, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("fedcba"));
    stream.reset();
    assertTokenStream(stream, "fedc,0,4,1/edcb,1,5,1/dcb,2,5,0/cb,3,5,0/b,4,5,0/a,5,6,1/aa,5,6,0");
}

From source file:jp.sf.fess.solr.plugin.analysis.synonym.NGramSynonymTokenizerTest.java

License:Apache License

@Test
public void testAfterStrSingleSynonym1() throws Exception {
    Analyzer a = new NGramSynonymTokenizerTestAnalyzer(1, false, "a,aa");
    TokenStream stream = a.tokenStream("f", new StringReader("ab"));
    stream.reset();
    assertTokenStream(stream, "a,0,1,1/b,1,2,1");

    a = new NGramSynonymTokenizerTestAnalyzer(1, false, "a,aa");
    stream = a.tokenStream("f", new StringReader("abb"));
    stream.reset();//from   w w  w .jav a2s.  co  m
    assertTokenStream(stream, "a,0,1,1/b,1,2,1/b,2,3,1");

    a = new NGramSynonymTokenizerTestAnalyzer(1, false, "a,aa");
    stream = a.tokenStream("f", new StringReader("abcd"));
    stream.reset();
    assertTokenStream(stream, "a,0,1,1/b,1,2,1/c,2,3,1/d,3,4,1");

    a = new NGramSynonymTokenizerTestAnalyzer(1, false, "a,aa");
    stream = a.tokenStream("f", new StringReader("abcde"));
    stream.reset();
    assertTokenStream(stream, "a,0,1,1/b,1,2,1/c,2,3,1/d,3,4,1/e,4,5,1");
}

From source file:jp.sf.fess.solr.plugin.analysis.synonym.NGramSynonymTokenizerTest.java

License:Apache License

@Test
public void testAfterStrSingleSynonym2() throws Exception {
    Analyzer a = new NGramSynonymTokenizerTestAnalyzer(2, false, "a,aa");
    TokenStream stream = a.tokenStream("f", new StringReader("ab"));
    stream.reset();
    assertTokenStream(stream, "a,0,1,1/b,1,2,1");

    a = new NGramSynonymTokenizerTestAnalyzer(2, false, "a,aa");
    stream = a.tokenStream("f", new StringReader("abb"));
    stream.reset();/*w  ww  .ja v a2 s .c o m*/
    assertTokenStream(stream, "a,0,1,1/bb,1,3,1");

    a = new NGramSynonymTokenizerTestAnalyzer(2, false, "a,aa");
    stream = a.tokenStream("f", new StringReader("abcd"));
    stream.reset();
    assertTokenStream(stream, "a,0,1,1/bc,1,3,1/cd,2,4,1");

    a = new NGramSynonymTokenizerTestAnalyzer(2, false, "a,aa");
    stream = a.tokenStream("f", new StringReader("abcde"));
    stream.reset();
    assertTokenStream(stream, "a,0,1,1/bc,1,3,1/cd,2,4,1/de,3,5,1");
}

From source file:jp.sf.fess.solr.plugin.analysis.synonym.NGramSynonymTokenizerTest.java

License:Apache License

@Test
public void testAfterStrSingleSynonym3() throws Exception {
    Analyzer a = new NGramSynonymTokenizerTestAnalyzer(3, false, "a,aa");
    TokenStream stream = a.tokenStream("f", new StringReader("ab"));
    stream.reset();
    assertTokenStream(stream, "a,0,1,1/b,1,2,1");

    a = new NGramSynonymTokenizerTestAnalyzer(3, false, "a,aa");
    stream = a.tokenStream("f", new StringReader("abb"));
    stream.reset();//from  w  w w.  j  a  va2 s  .  c  o  m
    assertTokenStream(stream, "a,0,1,1/bb,1,3,1");

    a = new NGramSynonymTokenizerTestAnalyzer(3, false, "a,aa");
    stream = a.tokenStream("f", new StringReader("abcd"));
    stream.reset();
    assertTokenStream(stream, "a,0,1,1/bcd,1,4,1");

    a = new NGramSynonymTokenizerTestAnalyzer(3, false, "a,aa");
    stream = a.tokenStream("f", new StringReader("abcde"));
    stream.reset();
    assertTokenStream(stream, "a,0,1,1/bcd,1,4,1/cde,2,5,1");
}

From source file:jp.sf.fess.solr.plugin.analysis.synonym.NGramSynonymTokenizerTest.java

License:Apache License

@Test
public void testAfterStrSingleSynonym4() throws Exception {
    Analyzer a = new NGramSynonymTokenizerTestAnalyzer(4, false, "a,aa");
    TokenStream stream = a.tokenStream("f", new StringReader("ab"));
    stream.reset();
    assertTokenStream(stream, "a,0,1,1/b,1,2,1");

    a = new NGramSynonymTokenizerTestAnalyzer(4, false, "a,aa");
    stream = a.tokenStream("f", new StringReader("abb"));
    stream.reset();/*from   w  w w .  j ava 2 s .c o m*/
    assertTokenStream(stream, "a,0,1,1/bb,1,3,1");

    a = new NGramSynonymTokenizerTestAnalyzer(4, false, "a,aa");
    stream = a.tokenStream("f", new StringReader("abcd"));
    stream.reset();
    assertTokenStream(stream, "a,0,1,1/bcd,1,4,1");

    a = new NGramSynonymTokenizerTestAnalyzer(4, false, "a,aa");
    stream = a.tokenStream("f", new StringReader("abcde"));
    stream.reset();
    assertTokenStream(stream, "a,0,1,1/bcde,1,5,1");

    a = new NGramSynonymTokenizerTestAnalyzer(4, false, "a,aa");
    stream = a.tokenStream("f", new StringReader("abcdef"));
    stream.reset();
    assertTokenStream(stream, "a,0,1,1/bcde,1,5,1/cdef,2,6,1");
}

From source file:jp.sf.fess.solr.plugin.analysis.synonym.NGramSynonymTokenizerTest.java

License:Apache License

@Test
public void testAfterStrSingleSynonymExpand1() throws Exception {
    Analyzer a = new NGramSynonymTokenizerTestAnalyzer(1, true, "a,aa");
    TokenStream stream = a.tokenStream("f", new StringReader("ab"));
    stream.reset();
    assertTokenStream(stream, "a,0,1,1/aa,0,1,0/b,1,2,1");

    a = new NGramSynonymTokenizerTestAnalyzer(1, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("abb"));
    stream.reset();/*from w w w .  j a  v  a2  s . co m*/
    assertTokenStream(stream, "a,0,1,1/aa,0,1,0/b,1,2,1/b,2,3,1");

    a = new NGramSynonymTokenizerTestAnalyzer(1, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("abcd"));
    stream.reset();
    assertTokenStream(stream, "a,0,1,1/aa,0,1,0/b,1,2,1/c,2,3,1/d,3,4,1");

    a = new NGramSynonymTokenizerTestAnalyzer(1, true, "a,aa");
    stream = a.tokenStream("f", new StringReader("abcde"));
    stream.reset();
    assertTokenStream(stream, "a,0,1,1/aa,0,1,0/b,1,2,1/c,2,3,1/d,3,4,1/e,4,5,1");
}