Java tutorial
/* * Licensed to Elasticsearch under one or more contributor * license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright * ownership. Elasticsearch licenses this file to you under * the Apache License, Version 2.0 (the "License"); you may * not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.elasticsearch.index.mapper.core; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.codec.docvaluesformat.DocValuesFormatProvider; import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider; import org.elasticsearch.index.mapper.*; import org.elasticsearch.index.mapper.core.StringFieldMapper.ValueAndBoost; import org.elasticsearch.index.similarity.SimilarityProvider; import java.io.IOException; import java.util.List; import java.util.Map; import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeIntegerValue; import static org.elasticsearch.index.mapper.MapperBuilders.tokenCountField; import static org.elasticsearch.index.mapper.core.TypeParsers.parseNumberField; /** * A {@link FieldMapper} that takes a string and writes a count of the tokens in that string * to the index. In most ways the mapper acts just like an {@link IntegerFieldMapper}. */ public class TokenCountFieldMapper extends IntegerFieldMapper { public static final String CONTENT_TYPE = "token_count"; public static class Defaults extends IntegerFieldMapper.Defaults { } public static class Builder extends NumberFieldMapper.Builder<Builder, TokenCountFieldMapper> { private Integer nullValue = Defaults.NULL_VALUE; private NamedAnalyzer analyzer; public Builder(String name) { super(name, new FieldType(Defaults.FIELD_TYPE)); builder = this; } public Builder nullValue(int nullValue) { this.nullValue = nullValue; return this; } public Builder analyzer(NamedAnalyzer analyzer) { this.analyzer = analyzer; return this; } public NamedAnalyzer analyzer() { return analyzer; } @Override public TokenCountFieldMapper build(BuilderContext context) { fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f); TokenCountFieldMapper fieldMapper = new TokenCountFieldMapper(buildNames(context), precisionStep, boost, fieldType, docValues, nullValue, ignoreMalformed(context), coerce(context), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, context.indexSettings(), analyzer, multiFieldsBuilder.build(this, context), copyTo); fieldMapper.includeInAll(includeInAll); return fieldMapper; } } public static class TypeParser implements Mapper.TypeParser { @Override @SuppressWarnings("unchecked") public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException { TokenCountFieldMapper.Builder builder = tokenCountField(name); parseNumberField(builder, name, node, parserContext); for (Map.Entry<String, Object> entry : node.entrySet()) { String propName = Strings.toUnderscoreCase(entry.getKey()); Object propNode = entry.getValue(); if (propName.equals("null_value")) { builder.nullValue(nodeIntegerValue(propNode)); } else if (propName.equals("analyzer")) { NamedAnalyzer analyzer = parserContext.analysisService().analyzer(propNode.toString()); if (analyzer == null) { throw new MapperParsingException( "Analyzer [" + propNode.toString() + "] not found for field [" + name + "]"); } builder.analyzer(analyzer); } } if (builder.analyzer() == null) { throw new MapperParsingException("Analyzer must be set for field [" + name + "] but wasn't."); } return builder; } } private NamedAnalyzer analyzer; protected TokenCountFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Boolean docValues, Integer nullValue, Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, SimilarityProvider similarity, Loading normsLoading, Settings fieldDataSettings, Settings indexSettings, NamedAnalyzer analyzer, MultiFields multiFields, CopyTo copyTo) { super(names, precisionStep, boost, fieldType, docValues, nullValue, ignoreMalformed, coerce, postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, indexSettings, multiFields, copyTo); this.analyzer = analyzer; } @Override protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException { ValueAndBoost valueAndBoost = StringFieldMapper.parseCreateFieldForString(context, null /* Out null value is an int so we convert*/, boost); if (valueAndBoost.value() == null && nullValue() == null) { return; } if (fieldType.indexed() || fieldType.stored() || hasDocValues()) { int count; if (valueAndBoost.value() == null) { count = nullValue(); } else { count = countPositions(analyzer.analyzer().tokenStream(name(), valueAndBoost.value())); } addIntegerFields(context, fields, count, valueAndBoost.boost()); } if (fields.isEmpty()) { context.ignoredValue(names.indexName(), valueAndBoost.value()); } } /** * Count position increments in a token stream. Package private for testing. * @param tokenStream token stream to count * @return number of position increments in a token stream * @throws IOException if tokenStream throws it */ static int countPositions(TokenStream tokenStream) throws IOException { try { int count = 0; PositionIncrementAttribute position = tokenStream.addAttribute(PositionIncrementAttribute.class); tokenStream.reset(); while (tokenStream.incrementToken()) { count += position.getPositionIncrement(); } tokenStream.end(); count += position.getPositionIncrement(); return count; } finally { tokenStream.close(); } } /** * Name of analyzer. * @return name of analyzer */ public String analyzer() { return analyzer.name(); } @Override protected String contentType() { return CONTENT_TYPE; } @Override public void merge(Mapper mergeWith, MergeContext mergeContext) throws MergeMappingException { super.merge(mergeWith, mergeContext); if (!this.getClass().equals(mergeWith.getClass())) { return; } if (!mergeContext.mergeFlags().simulate()) { this.analyzer = ((TokenCountFieldMapper) mergeWith).analyzer; } } @Override protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { super.doXContentBody(builder, includeDefaults, params); builder.field("analyzer", analyzer()); } }