Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.solr.schema; import java.io.IOException; import java.io.InputStream; import java.text.Collator; import java.text.ParseException; import java.text.RuleBasedCollator; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Map; import org.apache.commons.io.IOUtils; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; import org.apache.lucene.analysis.util.ResourceLoader; import org.apache.lucene.collation.CollationKeyAnalyzer; import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.Query; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.util.BytesRef; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.SolrException; import org.apache.solr.response.TextResponseWriter; import org.apache.solr.search.QParser; import org.apache.solr.uninverting.UninvertingReader.Type; /** * Field for collated sort keys. * These can be used for locale-sensitive sort and range queries. * <p> * This field can be created in two ways: * <ul> * <li>Based upon a system collator associated with a Locale. * <li>Based upon a tailored ruleset. * </ul> * <p> * Using a System collator: * <ul> * <li>language: ISO-639 language code (mandatory) * <li>country: ISO-3166 country code (optional) * <li>variant: vendor or browser-specific code (optional) * <li>strength: 'primary','secondary','tertiary', or 'identical' (optional) * <li>decomposition: 'no','canonical', or 'full' (optional) * </ul> * <p> * Using a Tailored ruleset: * <ul> * <li>custom: UTF-8 text file containing rules supported by RuleBasedCollator (mandatory) * <li>strength: 'primary','secondary','tertiary', or 'identical' (optional) * <li>decomposition: 'no','canonical', or 'full' (optional) * </ul> * * @see Collator * @see Locale * @see RuleBasedCollator * @since solr 4.0 */ public class CollationField extends FieldType { private Analyzer analyzer; @Override protected void init(IndexSchema schema, Map<String, String> args) { properties |= TOKENIZED; // this ensures our analyzer gets hit setup(schema.getResourceLoader(), args); super.init(schema, args); } /** * Setup the field according to the provided parameters */ private void setup(ResourceLoader loader, Map<String, String> args) { String custom = args.remove("custom"); String language = args.remove("language"); String country = args.remove("country"); String variant = args.remove("variant"); String strength = args.remove("strength"); String decomposition = args.remove("decomposition"); final Collator collator; if (custom == null && language == null) throw new SolrException(ErrorCode.SERVER_ERROR, "Either custom or language is required."); if (custom != null && (language != null || country != null || variant != null)) throw new SolrException(ErrorCode.SERVER_ERROR, "Cannot specify both language and custom. " + "To tailor rules for a built-in language, see the javadocs for RuleBasedCollator. " + "Then save the entire customized ruleset to a file, and use with the custom parameter"); if (language != null) { // create from a system collator, based on Locale. collator = createFromLocale(language, country, variant); } else { // create from a custom ruleset collator = createFromRules(custom, loader); } // set the strength flag, otherwise it will be the default. if (strength != null) { if (strength.equalsIgnoreCase("primary")) collator.setStrength(Collator.PRIMARY); else if (strength.equalsIgnoreCase("secondary")) collator.setStrength(Collator.SECONDARY); else if (strength.equalsIgnoreCase("tertiary")) collator.setStrength(Collator.TERTIARY); else if (strength.equalsIgnoreCase("identical")) collator.setStrength(Collator.IDENTICAL); else throw new SolrException(ErrorCode.SERVER_ERROR, "Invalid strength: " + strength); } // set the decomposition flag, otherwise it will be the default. if (decomposition != null) { if (decomposition.equalsIgnoreCase("no")) collator.setDecomposition(Collator.NO_DECOMPOSITION); else if (decomposition.equalsIgnoreCase("canonical")) collator.setDecomposition(Collator.CANONICAL_DECOMPOSITION); else if (decomposition.equalsIgnoreCase("full")) collator.setDecomposition(Collator.FULL_DECOMPOSITION); else throw new SolrException(ErrorCode.SERVER_ERROR, "Invalid decomposition: " + decomposition); } analyzer = new CollationKeyAnalyzer(collator); } /** * Create a locale from language, with optional country and variant. * Then return the appropriate collator for the locale. */ private Collator createFromLocale(String language, String country, String variant) { Locale locale; if (language != null && country == null && variant != null) throw new SolrException(ErrorCode.SERVER_ERROR, "To specify variant, country is required"); else if (language != null && country != null && variant != null) locale = new Locale(language, country, variant); else if (language != null && country != null) locale = new Locale(language, country); else locale = new Locale(language); return Collator.getInstance(locale); } /** * Read custom rules from a file, and create a RuleBasedCollator * The file cannot support comments, as # might be in the rules! */ private Collator createFromRules(String fileName, ResourceLoader loader) { InputStream input = null; try { input = loader.openResource(fileName); String rules = IOUtils.toString(input, "UTF-8"); return new RuleBasedCollator(rules); } catch (IOException | ParseException e) { // io error or invalid rules throw new RuntimeException(e); } finally { IOUtils.closeQuietly(input); } } @Override public void write(TextResponseWriter writer, String name, IndexableField f) throws IOException { writer.writeStr(name, f.stringValue(), true); } @Override public SortField getSortField(SchemaField field, boolean top) { return getStringSort(field, top); } @Override public Type getUninversionType(SchemaField sf) { if (sf.multiValued()) { return Type.SORTED_SET_BINARY; } else { return Type.SORTED; } } @Override public Analyzer getIndexAnalyzer() { return analyzer; } @Override public Analyzer getQueryAnalyzer() { return analyzer; } /** * analyze the range with the analyzer, instead of the collator. * because jdk collators might not be thread safe (when they are * it's just that all methods are synced), this keeps things * simple (we already have a threadlocal clone in the reused TS) */ private BytesRef getCollationKey(String field, String text) { try (TokenStream source = analyzer.tokenStream(field, text)) { source.reset(); TermToBytesRefAttribute termAtt = source.getAttribute(TermToBytesRefAttribute.class); // we control the analyzer here: most errors are impossible if (!source.incrementToken()) throw new IllegalArgumentException("analyzer returned no terms for text: " + text); BytesRef bytes = BytesRef.deepCopyOf(termAtt.getBytesRef()); assert !source.incrementToken(); source.end(); return bytes; } catch (IOException e) { throw new RuntimeException("Unable to analyze text: " + text, e); } } @Override public Query getRangeQuery(QParser parser, SchemaField field, String part1, String part2, boolean minInclusive, boolean maxInclusive) { String f = field.getName(); BytesRef low = part1 == null ? null : getCollationKey(f, part1); BytesRef high = part2 == null ? null : getCollationKey(f, part2); if (!field.indexed() && field.hasDocValues()) { return SortedSetDocValuesField.newRangeQuery(field.getName(), low, high, minInclusive, maxInclusive); } else { return new TermRangeQuery(field.getName(), low, high, minInclusive, maxInclusive); } } @Override public void checkSchemaField(SchemaField field) { // no-op } @Override public List<IndexableField> createFields(SchemaField field, Object value, float boost) { if (field.hasDocValues()) { List<IndexableField> fields = new ArrayList<>(); fields.add(createField(field, value, boost)); final BytesRef bytes = getCollationKey(field.getName(), value.toString()); if (field.multiValued()) { fields.add(new SortedSetDocValuesField(field.getName(), bytes)); } else { fields.add(new SortedDocValuesField(field.getName(), bytes)); } return fields; } else { return Collections.singletonList(createField(field, value, boost)); } } @Override public Object marshalSortValue(Object value) { return marshalBase64SortValue(value); } @Override public Object unmarshalSortValue(Object value) { return unmarshalBase64SortValue(value); } }