Example usage for org.apache.lucene.document Field setTokenStream

Introduction

In this page you can find the example usage for org.apache.lucene.document Field setTokenStream.

Prototype

public void setTokenStream(TokenStream tokenStream) 

Source Link

Document

Expert: sets the token stream to be used for indexing and causes isIndexed() and isTokenized() to return true.

Usage

From source file:cz.muni.fi.mir.tools.CanonicOutputBridge.java

License:Apache License

private Field newField(String name, String value, LuceneOptions luceneOptions, Analyzer analyzer) {
    Field f = new Field(name, value, luceneOptions.getStore(), luceneOptions.getIndex(),
            luceneOptions.getTermVector());

    f.setBoost(luceneOptions.getBoost());

    if (analyzer != null) {
        f.setTokenStream(analyzer.tokenStream(name, new StringReader(value)));
    }/*from   ww w  . j  a v a 2  s.  c om*/

    return f;
}

---

From source file:de.ids_mannheim.korap.index.FieldDocument.java

public void addTV(String key, String value, MultiTermTokenStream ts) {
    Field textField = new Field(key, value, tvField);
    textField.setTokenStream(ts);
    doc.add(textField);/* w w w  .j a v  a  2  s .  co  m*/
}

---

From source file:org.apache.solr.legacy.TestLegacyField.java

License:Apache License

private void trySetTokenStreamValue(Field f) {
    expectThrows(IllegalArgumentException.class, () -> {
        f.setTokenStream(new CannedTokenStream(new Token("foo", 0, 3)));
    });//w  w w . j  av  a  2 s  . c o  m
}

---

From source file:org.apache.solr.schema.PreAnalyzedField.java

License:Apache License

public IndexableField fromString(SchemaField field, String val, float boost) throws Exception {
    if (val == null || val.trim().length() == 0) {
        return null;
    }/*w  w  w.j  a va 2  s.  c o m*/
    PreAnalyzedTokenizer parse = new PreAnalyzedTokenizer(new StringReader(val), parser);
    parse.reset(); // consume
    org.apache.lucene.document.FieldType type = createFieldType(field);
    if (type == null) {
        parse.close();
        return null;
    }
    Field f = null;
    if (parse.getStringValue() != null) {
        if (field.stored()) {
            f = new Field(field.getName(), parse.getStringValue(), type);
        } else {
            type.setStored(false);
        }
    } else if (parse.getBinaryValue() != null) {
        if (field.isBinary()) {
            f = new Field(field.getName(), parse.getBinaryValue(), type);
        }
    } else {
        type.setStored(false);
    }

    if (parse.hasTokenStream()) {
        if (field.indexed()) {
            type.setIndexed(true);
            type.setTokenized(true);
            if (f != null) {
                f.setTokenStream(parse);
            } else {
                f = new Field(field.getName(), parse, type);
            }
        } else {
            if (f != null) {
                f.fieldType().setIndexed(false);
                f.fieldType().setTokenized(false);
            }
        }
    }
    if (f != null) {
        f.setBoost(boost);
    }
    return f;
}

---

From source file:org.apache.solr.schema.TrieFieldHelper.java

License:Apache License

private static Fieldable createField(String name, byte[] arr, TokenStream ts, FieldInfo info, float boost) {

    Field f;
    if (info.store) {
        f = new Field(name, arr);
        if (info.index)
            f.setTokenStream(ts);
    } else {//  www. ja v  a2  s.  c  o m
        f = new Field(name, ts);
    }

    // term vectors aren't supported
    f.setOmitNorms(info.omitNorms);
    f.setOmitTermFreqAndPositions(info.omitTF);
    f.setBoost(boost);
    return f;
}

---

From source file:org.elasticsearch.index.mapper.json.JsonAllFieldMapper.java

License:Apache License

@Override
protected Field parseCreateField(JsonParseContext jsonContext) throws IOException {
    if (!enabled) {
        return null;
    }/*from   w  w w .java2  s .co  m*/
    // reset the entries
    jsonContext.allEntries().reset();

    Analyzer analyzer = findAnalyzer(jsonContext.docMapper());
    TokenStream tokenStream = allTokenStream(names.indexName(), jsonContext.allEntries(), analyzer);
    if (stored()) {
        // TODO when its possible to pass char[] to field, we can optimize
        Field field = new Field(names.indexName(), jsonContext.allEntries().buildText(), store, index,
                termVector);
        field.setTokenStream(tokenStream);
        return field;
    } else {
        return new Field(names.indexName(), tokenStream, termVector);
    }
}

---

From source file:org.elasticsearch.index.mapper.json.JsonBoostFieldMapper.java

License:Apache License

@Override
protected Field parseCreateField(JsonParseContext jsonContext) throws IOException {
    float value = parseFloatValue(jsonContext);
    if (Float.isNaN(value)) {
        return null;
    }/* w  ww. j  a  v  a  2 s .c om*/
    jsonContext.doc().setBoost(value);
    Field field = null;
    if (stored()) {
        field = new Field(names.indexName(), Numbers.floatToBytes(value), store);
        if (indexed()) {
            field.setTokenStream(popCachedStream(precisionStep).setFloatValue(value));
        }
    } else if (indexed()) {
        field = new Field(names.indexName(), popCachedStream(precisionStep).setFloatValue(value));
    }
    return field;
}

---

From source file:org.elasticsearch.index.mapper.json.JsonDateFieldMapper.java

License:Apache License

@Override
protected Field parseCreateField(JsonParseContext jsonContext) throws IOException {
    String dateAsString;/*w  ww  . j  av a2 s  .c  o m*/
    if (jsonContext.externalValueSet()) {
        dateAsString = (String) jsonContext.externalValue();
        if (dateAsString == null) {
            dateAsString = nullValue;
        }
    } else {
        if (jsonContext.jp().getCurrentToken() == JsonToken.VALUE_NULL) {
            dateAsString = nullValue;
        } else {
            dateAsString = jsonContext.jp().getText();
        }
    }

    if (dateAsString == null) {
        return null;
    }
    if (includeInAll == null || includeInAll) {
        jsonContext.allEntries().addText(names.fullName(), dateAsString, boost);
    }

    long value = dateTimeFormatter.parser().parseMillis(dateAsString);
    Field field = null;
    if (stored()) {
        field = new Field(names.indexName(), Numbers.longToBytes(value), store);
        if (indexed()) {
            field.setTokenStream(popCachedStream(precisionStep).setLongValue(value));
        }
    } else if (indexed()) {
        field = new Field(names.indexName(), popCachedStream(precisionStep).setLongValue(value));
    }
    return field;
}

---

From source file:org.elasticsearch.index.mapper.json.JsonDoubleFieldMapper.java

License:Apache License

@Override
protected Field parseCreateField(JsonParseContext jsonContext) throws IOException {
    double value;
    if (jsonContext.externalValueSet()) {
        Object externalValue = jsonContext.externalValue();
        if (externalValue == null) {
            if (nullValue == null) {
                return null;
            }/*from ww w  .  ja v  a2s  .  com*/
            value = nullValue;
        } else {
            value = ((Number) externalValue).doubleValue();
        }
        if (includeInAll == null || includeInAll) {
            jsonContext.allEntries().addText(names.fullName(), Double.toString(value), boost);
        }
    } else {
        if (jsonContext.jp().getCurrentToken() == JsonToken.VALUE_NULL) {
            if (nullValue == null) {
                return null;
            }
            value = nullValue;
            if (nullValueAsString != null && (includeInAll == null || includeInAll)) {
                jsonContext.allEntries().addText(names.fullName(), nullValueAsString, boost);
            }
        } else {
            if (jsonContext.jp().getCurrentToken() == JsonToken.VALUE_STRING) {
                value = Double.parseDouble(jsonContext.jp().getText());
            } else {
                value = jsonContext.jp().getDoubleValue();
            }
            if (includeInAll == null || includeInAll) {
                jsonContext.allEntries().addText(names.fullName(), jsonContext.jp().getText(), boost);
            }
        }
    }

    Field field = null;
    if (stored()) {
        field = new Field(names.indexName(), Numbers.doubleToBytes(value), store);
        if (indexed()) {
            field.setTokenStream(popCachedStream(precisionStep).setDoubleValue(value));
        }
    } else if (indexed()) {
        field = new Field(names.indexName(), popCachedStream(precisionStep).setDoubleValue(value));
    }
    return field;
}

---

From source file:org.elasticsearch.index.mapper.json.JsonFloatFieldMapper.java

License:Apache License

@Override
protected Field parseCreateField(JsonParseContext jsonContext) throws IOException {
    float value;/*from  w  ww  . j  av a  2 s . co  m*/
    if (jsonContext.externalValueSet()) {
        Object externalValue = jsonContext.externalValue();
        if (externalValue == null) {
            if (nullValue == null) {
                return null;
            }
            value = nullValue;
        } else {
            value = ((Number) externalValue).floatValue();
        }
        if (includeInAll == null || includeInAll) {
            jsonContext.allEntries().addText(names.fullName(), Float.toString(value), boost);
        }
    } else {
        if (jsonContext.jp().getCurrentToken() == JsonToken.VALUE_NULL) {
            if (nullValue == null) {
                return null;
            }
            value = nullValue;
            if (nullValueAsString != null && (includeInAll == null || includeInAll)) {
                jsonContext.allEntries().addText(names.fullName(), nullValueAsString, boost);
            }
        } else {
            if (jsonContext.jp().getCurrentToken() == JsonToken.VALUE_STRING) {
                value = Float.parseFloat(jsonContext.jp().getText());
            } else {
                value = jsonContext.jp().getFloatValue();
            }
            if (includeInAll == null || includeInAll) {
                jsonContext.allEntries().addText(names.fullName(), jsonContext.jp().getText(), boost);
            }
        }
    }

    Field field = null;
    if (stored()) {
        field = new Field(names.indexName(), Numbers.floatToBytes(value), store);
        if (indexed()) {
            field.setTokenStream(popCachedStream(precisionStep).setFloatValue(value));
        }
    } else if (indexed()) {
        field = new Field(names.indexName(), popCachedStream(precisionStep).setFloatValue(value));
    }
    return field;
}

---

• «
• 1
• 2
• »