Example usage for org.apache.lucene.index IndexableFieldType tokenized

Introduction

In this page you can find the example usage for org.apache.lucene.index IndexableFieldType tokenized.

Prototype

public boolean tokenized();

Source Link

Document

True if this field's value should be analyzed by the Analyzer .

Usage

From source file:com.github.hotware.lucene.extension.bean.test.BeanConverterTest.java

License:BEER-WARE LICENSE

public void testBeanDocumentConversionViceVersa() throws IllegalArgumentException, IllegalAccessException {
    BeanConverter converter = new BeanConverterImpl(new BeanInformationCacheImpl());
    Field[] fields = TestBean.class.getFields();
    TestBean testBean = new TestBean();
    for (Field field : fields) {
        String fieldName = field.getName();
        Class<?> type = field.getType();
        if (type.equals(int.class)) {
            field.setInt(testBean, Integer.MAX_VALUE);
        } else if (type.equals(long.class)) {
            field.setLong(testBean, Long.MAX_VALUE);
        } else if (type.equals(double.class)) {
            field.setDouble(testBean, Double.MAX_VALUE);
        } else if (type.equals(float.class)) {
            field.setFloat(testBean, Float.MAX_VALUE);
        } else if (type.equals(boolean.class)) {
            field.setBoolean(testBean, true);
        } else if (type.equals(Integer.class)) {
            field.set(testBean, Integer.MAX_VALUE);
        } else if (type.equals(Long.class)) {
            field.set(testBean, Long.MAX_VALUE);
        } else if (type.equals(Double.class)) {
            field.set(testBean, Double.MAX_VALUE);
        } else if (type.equals(Float.class)) {
            field.set(testBean, Float.MAX_VALUE);
        } else if (type.equals(Boolean.class)) {
            field.set(testBean, true);//from  w ww. j a  v a2  s . com
        } else if (type.equals(String.class)) {
            field.set(testBean, "Test");
        } else if (fieldName.equals("emptySetTest")) {
            field.set(testBean, new HashSet<String>());
        } else if (type.equals(Set.class)) {
            Set<String> set = new HashSet<String>();
            set.add("1");
            set.add("2");
            set.add("3");
            field.set(testBean, set);
        } else if (type.equals(List.class)) {
            List<String> list = new ArrayList<String>();
            list.add("1");
            list.add("2");
            list.add("3");
            field.set(testBean, list);
        } else if (type.equals(Object.class)) {
            field.set(testBean, new Date());
        } else {
            fail("type is not handled in the Unit-Test, please add " + type);
        }
        Document document = converter.beanToDocument(testBean);
        // check if all values are stored the same way they were entered
        if (fieldName.equals("serializeTest")) {
            System.out.println("doing serialize equality test.");
            assertTrue(Arrays.equals(toSerializedLuceneValue(field.get(testBean)),
                    document.getBinaryValue(fieldName).bytes));
        } else if (fieldName.equals("customNameTest")) {
            System.out.println("doing custom name equality test.");
            String originalValue = (String) field.get(testBean);
            String documentValue = document.get("customName");
            assertEquals(originalValue, documentValue);
        } else if (fieldName.equals("notAnnotatedTest")) {
            System.out.println("doing not annotated test.");
            assertEquals(null, document.get(fieldName));
        } else if (fieldName.equals("listTest")) {
            System.out.println("doing listTest");
            @SuppressWarnings("unchecked")
            List<String> originalList = (List<String>) field.get(testBean);
            IndexableField[] documentFields = document.getFields(fieldName);
            for (int i = 0; i < originalList.size(); ++i) {
                assertEquals(originalList.get(i), documentFields[i].stringValue());
            }
        } else if (fieldName.equals("setTest")) {
            System.out.println("doing listTest");
            @SuppressWarnings("unchecked")
            Set<String> originalSet = (Set<String>) field.get(testBean);
            Set<String> docSet = new HashSet<String>();
            for (IndexableField documentField : document.getFields(fieldName)) {
                docSet.add(documentField.stringValue());
            }
            assertEquals(originalSet, docSet);
        } else if (fieldName.equals("emptySetTest")) {
            System.out.println("doing emptySetTest");
            assertEquals(null, document.get(fieldName));
        } else if (fieldName.equals("multiTest")) {
            System.out.println("doing multiTest");
            assertEquals("multiTest", document.get(fieldName));
        } else if (fieldName.equals("multiMultiTest")) {
            System.out.println("doint multiMultiTest");
            assertEquals("multiMultiTest", document.get(fieldName));
        } else {
            // normally a check is needed, but in the test-case we
            // can do this without checking for a present annotation
            BeanField bf = field.getAnnotation(BeanField.class);
            System.out.println("doing " + bf.type() + " tests on \"" + fieldName + "\".");
            assertEquals(field.get(testBean).toString(), document.get(fieldName));
            IndexableField indexField = document.getField(fieldName);
            IndexableFieldType indexFieldType = indexField.fieldType();
            assertEquals(bf.store(), indexFieldType.stored());
            assertEquals(bf.index(), indexFieldType.indexed());
            assertEquals(bf.tokenized(), indexFieldType.tokenized());
            // TODO: test if fieldType is correct?
        }
    }

    // now that all the conversion works we can safely generate
    // a document with that and work backwards :)
    System.out.println("doing reverse conversion (document to bean) test.");
    Document document = converter.beanToDocument(testBean);
    TestBean reverseBean = converter.documentToBean(TestBean.class, document);

    // setting the stuff that can not be in the document and therefore not
    // in the reverseBean
    reverseBean.notAnnotatedTest = testBean.notAnnotatedTest;
    reverseBean.notStoredButIndexedTest = testBean.notStoredButIndexedTest;
    assertTrue(testBean.equals(reverseBean));

    System.out.println("Result: conversion test successfull.");
}

---

From source file:org.elasticsearch.index.mapper.core.KeywordFieldMapperTests.java

License:Apache License

public void testDefaults() throws Exception {
    String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
            .startObject("field").field("type", "keyword").endObject().endObject().endObject().endObject()
            .string();/* w w w  . j a v  a 2 s  .  co  m*/

    DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));

    assertEquals(mapping, mapper.mappingSource().toString());

    ParsedDocument doc = mapper.parse("test", "type", "1",
            XContentFactory.jsonBuilder().startObject().field("field", "1234").endObject().bytes());

    IndexableField[] fields = doc.rootDoc().getFields("field");
    assertEquals(2, fields.length);

    assertEquals("1234", fields[0].stringValue());
    IndexableFieldType fieldType = fields[0].fieldType();
    assertThat(fieldType.omitNorms(), equalTo(true));
    assertFalse(fieldType.tokenized());
    assertFalse(fieldType.stored());
    assertThat(fieldType.indexOptions(), equalTo(IndexOptions.DOCS));
    assertThat(fieldType.storeTermVectors(), equalTo(false));
    assertThat(fieldType.storeTermVectorOffsets(), equalTo(false));
    assertThat(fieldType.storeTermVectorPositions(), equalTo(false));
    assertThat(fieldType.storeTermVectorPayloads(), equalTo(false));
    assertEquals(DocValuesType.NONE, fieldType.docValuesType());

    assertEquals(new BytesRef("1234"), fields[1].binaryValue());
    fieldType = fields[1].fieldType();
    assertThat(fieldType.indexOptions(), equalTo(IndexOptions.NONE));
    assertEquals(DocValuesType.SORTED_SET, fieldType.docValuesType());
}

---

From source file:org.elasticsearch.index.mapper.core.TextFieldMapperTests.java

License:Apache License

public void testDefaults() throws Exception {
    String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
            .startObject("field").field("type", "text").endObject().endObject().endObject().endObject()
            .string();//from www.jav  a2  s  .c o  m

    DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));

    assertEquals(mapping, mapper.mappingSource().toString());

    ParsedDocument doc = mapper.parse("test", "type", "1",
            XContentFactory.jsonBuilder().startObject().field("field", "1234").endObject().bytes());

    IndexableField[] fields = doc.rootDoc().getFields("field");
    assertEquals(1, fields.length);

    assertEquals("1234", fields[0].stringValue());
    IndexableFieldType fieldType = fields[0].fieldType();
    assertThat(fieldType.omitNorms(), equalTo(false));
    assertTrue(fieldType.tokenized());
    assertFalse(fieldType.stored());
    assertThat(fieldType.indexOptions(), equalTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS));
    assertThat(fieldType.storeTermVectors(), equalTo(false));
    assertThat(fieldType.storeTermVectorOffsets(), equalTo(false));
    assertThat(fieldType.storeTermVectorPositions(), equalTo(false));
    assertThat(fieldType.storeTermVectorPayloads(), equalTo(false));
    assertEquals(DocValuesType.NONE, fieldType.docValuesType());
}

---

From source file:org.elasticsearch.index.mapper.ICUCollationKeywordFieldMapperTests.java

License:Apache License

public void testDefaults() throws Exception {
    String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
            .startObject("field").field("type", FIELD_TYPE).endObject().endObject().endObject().endObject()
            .string();//  ww w .  java2 s . co  m

    DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));

    assertEquals(mapping, mapper.mappingSource().toString());

    ParsedDocument doc = mapper.parse(SourceToParse.source("test", "type", "1",
            XContentFactory.jsonBuilder().startObject().field("field", "1234").endObject().bytes(),
            XContentType.JSON));

    IndexableField[] fields = doc.rootDoc().getFields("field");
    assertEquals(2, fields.length);

    Collator collator = Collator.getInstance(ULocale.ROOT);
    RawCollationKey key = collator.getRawCollationKey("1234", null);
    BytesRef expected = new BytesRef(key.bytes, 0, key.size);

    assertEquals(expected, fields[0].binaryValue());
    IndexableFieldType fieldType = fields[0].fieldType();
    assertThat(fieldType.omitNorms(), equalTo(true));
    assertFalse(fieldType.tokenized());
    assertFalse(fieldType.stored());
    assertThat(fieldType.indexOptions(), equalTo(IndexOptions.DOCS));
    assertThat(fieldType.storeTermVectors(), equalTo(false));
    assertThat(fieldType.storeTermVectorOffsets(), equalTo(false));
    assertThat(fieldType.storeTermVectorPositions(), equalTo(false));
    assertThat(fieldType.storeTermVectorPayloads(), equalTo(false));
    assertEquals(DocValuesType.NONE, fieldType.docValuesType());

    assertEquals(expected, fields[1].binaryValue());
    fieldType = fields[1].fieldType();
    assertThat(fieldType.indexOptions(), equalTo(IndexOptions.NONE));
    assertEquals(DocValuesType.SORTED_SET, fieldType.docValuesType());
}

---

From source file:org.elasticsearch.index.mapper.ICUCollationKeywordFieldMapperTests.java

License:Apache License

public void testBackCompat() throws Exception {
    indexService = createIndex("oldindex",
            Settings.builder().put("index.version.created", Version.V_5_5_0).build());
    parser = indexService.mapperService().documentMapperParser();

    String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
            .startObject("field").field("type", FIELD_TYPE).endObject().endObject().endObject().endObject()
            .string();/* w  w  w  .  ja  va  2 s. co m*/

    DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));

    assertEquals(mapping, mapper.mappingSource().toString());

    ParsedDocument doc = mapper.parse(SourceToParse.source("oldindex", "type", "1",
            XContentFactory.jsonBuilder().startObject().field("field", "1234").endObject().bytes(),
            XContentType.JSON));

    IndexableField[] fields = doc.rootDoc().getFields("field");
    assertEquals(2, fields.length);

    Collator collator = Collator.getInstance(ULocale.ROOT);
    RawCollationKey key = collator.getRawCollationKey("1234", null);
    BytesRef expected = new BytesRef(key.bytes, 0, key.size);

    assertEquals(expected, fields[0].binaryValue());
    IndexableFieldType fieldType = fields[0].fieldType();
    assertThat(fieldType.omitNorms(), equalTo(true));
    assertFalse(fieldType.tokenized());
    assertFalse(fieldType.stored());
    assertThat(fieldType.indexOptions(), equalTo(IndexOptions.DOCS));
    assertThat(fieldType.storeTermVectors(), equalTo(false));
    assertThat(fieldType.storeTermVectorOffsets(), equalTo(false));
    assertThat(fieldType.storeTermVectorPositions(), equalTo(false));
    assertThat(fieldType.storeTermVectorPayloads(), equalTo(false));
    assertEquals(DocValuesType.NONE, fieldType.docValuesType());

    assertEquals(expected, fields[1].binaryValue());
    fieldType = fields[1].fieldType();
    assertThat(fieldType.indexOptions(), equalTo(IndexOptions.NONE));
    assertEquals(DocValuesType.SORTED, fieldType.docValuesType());
}

---

From source file:org.elasticsearch.index.mapper.ICUCollationKeywordFieldMapperTests.java

License:Apache License

public void testMultipleValues() throws IOException {
    String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
            .startObject("field").field("type", FIELD_TYPE).endObject().endObject().endObject().endObject()
            .string();/*from ww w.  java 2s .  c  o  m*/

    DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));

    assertEquals(mapping, mapper.mappingSource().toString());

    ParsedDocument doc = mapper
            .parse(SourceToParse.source(
                    "test", "type", "1", XContentFactory.jsonBuilder().startObject()
                            .field("field", Arrays.asList("1234", "5678")).endObject().bytes(),
                    XContentType.JSON));

    IndexableField[] fields = doc.rootDoc().getFields("field");
    assertEquals(4, fields.length);

    Collator collator = Collator.getInstance(ULocale.ROOT);
    RawCollationKey key = collator.getRawCollationKey("1234", null);
    BytesRef expected = new BytesRef(key.bytes, 0, key.size);

    assertEquals(expected, fields[0].binaryValue());
    IndexableFieldType fieldType = fields[0].fieldType();
    assertThat(fieldType.omitNorms(), equalTo(true));
    assertFalse(fieldType.tokenized());
    assertFalse(fieldType.stored());
    assertThat(fieldType.indexOptions(), equalTo(IndexOptions.DOCS));
    assertThat(fieldType.storeTermVectors(), equalTo(false));
    assertThat(fieldType.storeTermVectorOffsets(), equalTo(false));
    assertThat(fieldType.storeTermVectorPositions(), equalTo(false));
    assertThat(fieldType.storeTermVectorPayloads(), equalTo(false));
    assertEquals(DocValuesType.NONE, fieldType.docValuesType());

    assertEquals(expected, fields[1].binaryValue());
    fieldType = fields[1].fieldType();
    assertThat(fieldType.indexOptions(), equalTo(IndexOptions.NONE));
    assertEquals(DocValuesType.SORTED_SET, fieldType.docValuesType());

    collator = Collator.getInstance(ULocale.ROOT);
    key = collator.getRawCollationKey("5678", null);
    expected = new BytesRef(key.bytes, 0, key.size);

    assertEquals(expected, fields[2].binaryValue());
    fieldType = fields[2].fieldType();
    assertThat(fieldType.omitNorms(), equalTo(true));
    assertFalse(fieldType.tokenized());
    assertFalse(fieldType.stored());
    assertThat(fieldType.indexOptions(), equalTo(IndexOptions.DOCS));
    assertThat(fieldType.storeTermVectors(), equalTo(false));
    assertThat(fieldType.storeTermVectorOffsets(), equalTo(false));
    assertThat(fieldType.storeTermVectorPositions(), equalTo(false));
    assertThat(fieldType.storeTermVectorPayloads(), equalTo(false));
    assertEquals(DocValuesType.NONE, fieldType.docValuesType());

    assertEquals(expected, fields[3].binaryValue());
    fieldType = fields[3].fieldType();
    assertThat(fieldType.indexOptions(), equalTo(IndexOptions.NONE));
    assertEquals(DocValuesType.SORTED_SET, fieldType.docValuesType());
}

---

From source file:org.elasticsearch.index.mapper.ICUCollationKeywordFieldMapperTests.java

License:Apache License

public void testCollator() throws IOException {
    String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
            .startObject("field").field("type", FIELD_TYPE).field("language", "tr").field("strength", "primary")
            .endObject().endObject().endObject().endObject().string();

    DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));

    assertEquals(mapping, mapper.mappingSource().toString());

    ParsedDocument doc = mapper/*from w ww .  j av  a2s  .c  o  m*/
            .parse(SourceToParse.source(
                    "test", "type", "1", XContentFactory.jsonBuilder().startObject()
                            .field("field", "I WLL USE TURKSH CASING").endObject().bytes(),
                    XContentType.JSON));

    Collator collator = Collator.getInstance(new ULocale("tr"));
    collator.setStrength(Collator.PRIMARY);
    RawCollationKey key = collator.getRawCollationKey(" will use turkish casng", null); // should collate to same value
    BytesRef expected = new BytesRef(key.bytes, 0, key.size);

    IndexableField[] fields = doc.rootDoc().getFields("field");
    assertEquals(2, fields.length);

    assertEquals(expected, fields[0].binaryValue());
    IndexableFieldType fieldType = fields[0].fieldType();
    assertThat(fieldType.omitNorms(), equalTo(true));
    assertFalse(fieldType.tokenized());
    assertFalse(fieldType.stored());
    assertThat(fieldType.indexOptions(), equalTo(IndexOptions.DOCS));
    assertThat(fieldType.storeTermVectors(), equalTo(false));
    assertThat(fieldType.storeTermVectorOffsets(), equalTo(false));
    assertThat(fieldType.storeTermVectorPositions(), equalTo(false));
    assertThat(fieldType.storeTermVectorPayloads(), equalTo(false));
    assertEquals(DocValuesType.NONE, fieldType.docValuesType());

    assertEquals(expected, fields[1].binaryValue());
    fieldType = fields[1].fieldType();
    assertThat(fieldType.indexOptions(), equalTo(IndexOptions.NONE));
    assertEquals(DocValuesType.SORTED_SET, fieldType.docValuesType());
}

---

From source file:org.elasticsearch.index.mapper.KeywordFieldMapperTests.java

License:Apache License

public void testDefaults() throws Exception {
    String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
            .startObject("field").field("type", "keyword").endObject().endObject().endObject().endObject()
            .string();/* w ww. ja  v a2s .  c o m*/

    DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));

    assertEquals(mapping, mapper.mappingSource().toString());

    ParsedDocument doc = mapper.parse("test", "type", "1",
            XContentFactory.jsonBuilder().startObject().field("field", "1234").endObject().bytes());

    IndexableField[] fields = doc.rootDoc().getFields("field");
    assertEquals(2, fields.length);

    assertEquals(new BytesRef("1234"), fields[0].binaryValue());
    IndexableFieldType fieldType = fields[0].fieldType();
    assertThat(fieldType.omitNorms(), equalTo(true));
    assertFalse(fieldType.tokenized());
    assertFalse(fieldType.stored());
    assertThat(fieldType.indexOptions(), equalTo(IndexOptions.DOCS));
    assertThat(fieldType.storeTermVectors(), equalTo(false));
    assertThat(fieldType.storeTermVectorOffsets(), equalTo(false));
    assertThat(fieldType.storeTermVectorPositions(), equalTo(false));
    assertThat(fieldType.storeTermVectorPayloads(), equalTo(false));
    assertEquals(DocValuesType.NONE, fieldType.docValuesType());

    assertEquals(new BytesRef("1234"), fields[1].binaryValue());
    fieldType = fields[1].fieldType();
    assertThat(fieldType.indexOptions(), equalTo(IndexOptions.NONE));
    assertEquals(DocValuesType.SORTED_SET, fieldType.docValuesType());
}

---

From source file:org.elasticsearch.index.mapper.LegacyStringMappingTests.java

License:Apache License

private void assertEquals(IndexableFieldType ft1, IndexableFieldType ft2) {
    assertEquals(ft1.tokenized(), ft2.tokenized());
    assertEquals(ft1.omitNorms(), ft2.omitNorms());
    assertEquals(ft1.indexOptions(), ft2.indexOptions());
    assertEquals(ft1.storeTermVectors(), ft2.storeTermVectors());
    assertEquals(ft1.docValuesType(), ft2.docValuesType());
}

---

From source file:org.elasticsearch.index.mapper.string.SimpleStringMappingTests.java

License:Apache License

private void assertEquals(IndexableFieldType ft1, IndexableFieldType ft2) {
    assertEquals(ft1.indexed(), ft2.indexed());
    assertEquals(ft1.tokenized(), ft2.tokenized());
    assertEquals(ft1.omitNorms(), ft2.omitNorms());
    assertEquals(ft1.indexOptions(), ft2.indexOptions());
    assertEquals(ft1.storeTermVectors(), ft2.storeTermVectors());
    assertEquals(ft1.docValueType(), ft2.docValueType());
}

---

• «
• 1
• 2
• »