Example usage for org.apache.lucene.analysis CannedTokenStream CannedTokenStream

Introduction

In this page you can find the example usage for org.apache.lucene.analysis CannedTokenStream CannedTokenStream.

Prototype

public CannedTokenStream(Token... tokens) 

Source Link

Usage

From source file:org.apache.solr.legacy.TestLegacyField.java

License:Apache License

private void trySetTokenStreamValue(Field f) {
    expectThrows(IllegalArgumentException.class, () -> {
        f.setTokenStream(new CannedTokenStream(new Token("foo", 0, 3)));
    });/*w w w .  jav  a  2  s.  co m*/
}

---

From source file:org.apache.solr.legacy.TestLegacyFieldReuse.java

License:Apache License

public void testNumericReuse() throws IOException {
    LegacyIntField legacyIntField = new LegacyIntField("foo", 5, Field.Store.NO);

    // passing null
    TokenStream ts = legacyIntField.tokenStream(null, null);
    assertTrue(ts instanceof LegacyNumericTokenStream);
    assertEquals(LegacyNumericUtils.PRECISION_STEP_DEFAULT_32,
            ((LegacyNumericTokenStream) ts).getPrecisionStep());
    assertNumericContents(5, ts);//from   w  ww. j  a  v a2s  .  c o  m

    // now reuse previous stream
    legacyIntField = new LegacyIntField("foo", 20, Field.Store.NO);
    TokenStream ts2 = legacyIntField.tokenStream(null, ts);
    assertSame(ts, ts2);
    assertNumericContents(20, ts);

    // pass a bogus stream and ensure it's still ok
    legacyIntField = new LegacyIntField("foo", 2343, Field.Store.NO);
    TokenStream bogus = new CannedTokenStream(new Token("bogus", 0, 5));
    ts = legacyIntField.tokenStream(null, bogus);
    assertNotSame(bogus, ts);
    assertNumericContents(2343, ts);

    // pass another bogus stream (numeric, but different precision step!)
    legacyIntField = new LegacyIntField("foo", 42, Field.Store.NO);
    assert 3 != LegacyNumericUtils.PRECISION_STEP_DEFAULT;
    bogus = new LegacyNumericTokenStream(3);
    ts = legacyIntField.tokenStream(null, bogus);
    assertNotSame(bogus, ts);
    assertNumericContents(42, ts);
}

---

From source file:org.apache.solr.spelling.TestSuggestSpellingConverter.java

License:Apache License

public void assertConvertsTo(String text, String expected[]) throws IOException {
    Collection<Token> tokens = converter.convert(text);
    TokenStream ts = new CannedTokenStream(tokens.toArray(new Token[0]));
    assertTokenStreamContents(ts, expected);
}

---

From source file:org.elasticsearch.index.merge.policy.VersionFieldUpgraderTest.java

License:Apache License

/** Simple test: one doc in the old format, check that it looks correct */
public void testUpgradeOneDocument() throws Exception {
    Directory dir = newDirectory();//w w w  . j av  a2 s. co m
    IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(null));

    // add a document with a _uid having a payload of 3
    Document doc = new Document();
    Token token = new Token("1", 0, 1);
    token.setPayload(new BytesRef(Numbers.longToBytes(3)));
    doc.add(new TextField(UidFieldMapper.NAME, new CannedTokenStream(token)));
    iw.addDocument(doc);
    iw.commit();

    CodecReader reader = getOnlySegmentReader(DirectoryReader.open(iw, true));
    CodecReader upgraded = VersionFieldUpgrader.wrap(reader);
    // we need to be upgraded, should be a different instance
    assertNotSame(reader, upgraded);

    // make sure we can see our numericdocvalues in fieldinfos
    FieldInfo versionField = upgraded.getFieldInfos().fieldInfo(VersionFieldMapper.NAME);
    assertNotNull(versionField);
    assertEquals(DocValuesType.NUMERIC, versionField.getDocValuesType());
    // should have a value of 3, and be visible in docsWithField
    assertEquals(3, upgraded.getNumericDocValues(VersionFieldMapper.NAME).get(0));
    assertTrue(upgraded.getDocsWithField(VersionFieldMapper.NAME).get(0));

    // verify filterreader with checkindex
    TestUtil.checkReader(upgraded);

    reader.close();
    iw.close();
    dir.close();
}

---

From source file:org.elasticsearch.index.merge.policy.VersionFieldUpgraderTest.java

License:Apache License

/** test that we are a non-op if the segment already has the version field */
public void testAlreadyUpgraded() throws Exception {
    Directory dir = newDirectory();/*w w w.j a v  a2 s  .c  om*/
    IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(null));

    // add a document with a _uid having a payload of 3
    Document doc = new Document();
    Token token = new Token("1", 0, 1);
    token.setPayload(new BytesRef(Numbers.longToBytes(3)));
    doc.add(new TextField(UidFieldMapper.NAME, new CannedTokenStream(token)));
    doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 3));
    iw.addDocument(doc);
    iw.commit();

    CodecReader reader = getOnlySegmentReader(DirectoryReader.open(iw, true));
    CodecReader upgraded = VersionFieldUpgrader.wrap(reader);
    // we already upgraded: should be same instance
    assertSame(reader, upgraded);

    reader.close();
    iw.close();
    dir.close();
}

---

From source file:org.elasticsearch.index.merge.policy.VersionFieldUpgraderTest.java

License:Apache License

/** Test upgrading two documents */
public void testUpgradeTwoDocuments() throws Exception {
    Directory dir = newDirectory();// w w  w .  j a  v  a 2 s  .  c o  m
    IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(null));

    // add a document with a _uid having a payload of 3
    Document doc = new Document();
    Token token = new Token("1", 0, 1);
    token.setPayload(new BytesRef(Numbers.longToBytes(3)));
    doc.add(new TextField(UidFieldMapper.NAME, new CannedTokenStream(token)));
    iw.addDocument(doc);

    doc = new Document();
    token = new Token("2", 0, 1);
    token.setPayload(new BytesRef(Numbers.longToBytes(4)));
    doc.add(new TextField(UidFieldMapper.NAME, new CannedTokenStream(token)));
    iw.addDocument(doc);

    iw.commit();

    CodecReader reader = getOnlySegmentReader(DirectoryReader.open(iw, true));
    CodecReader upgraded = VersionFieldUpgrader.wrap(reader);
    // we need to be upgraded, should be a different instance
    assertNotSame(reader, upgraded);

    // make sure we can see our numericdocvalues in fieldinfos
    FieldInfo versionField = upgraded.getFieldInfos().fieldInfo(VersionFieldMapper.NAME);
    assertNotNull(versionField);
    assertEquals(DocValuesType.NUMERIC, versionField.getDocValuesType());
    // should have a values of 3 and 4, and be visible in docsWithField
    assertEquals(3, upgraded.getNumericDocValues(VersionFieldMapper.NAME).get(0));
    assertEquals(4, upgraded.getNumericDocValues(VersionFieldMapper.NAME).get(1));
    assertTrue(upgraded.getDocsWithField(VersionFieldMapper.NAME).get(0));
    assertTrue(upgraded.getDocsWithField(VersionFieldMapper.NAME).get(1));

    // verify filterreader with checkindex
    TestUtil.checkReader(upgraded);

    reader.close();
    iw.close();
    dir.close();
}

---