Java tutorial
package mj.ocraptor.database; /** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import java.io.File; import java.io.IOException; import java.io.Reader; import java.util.Set; import org.apache.lucene.analysis.CharReader; import org.apache.lucene.analysis.LowerCaseFilter; import org.apache.lucene.analysis.MappingCharFilter; import org.apache.lucene.analysis.NormalizeCharMap; import org.apache.lucene.analysis.StopAnalyzer; import org.apache.lucene.analysis.StopFilter; import org.apache.lucene.analysis.StopwordAnalyzerBase; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.WordlistLoader; import org.apache.lucene.analysis.standard.ClassicAnalyzer; import org.apache.lucene.analysis.standard.ClassicTokenizer; import org.apache.lucene.analysis.standard.StandardFilter; import org.apache.lucene.analysis.standard.StandardTokenizer; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.Version; /** * Filters {@link StandardTokenizer} with {@link StandardFilter}, * {@link LowerCaseFilter} and {@link StopFilter}, using a list of English stop * words. * * <a name="version"/> * <p> * You must specify the required {@link Version} compatibility when creating * StandardAnalyzer: * <ul> * <li>As of 3.4, Hiragana and Han characters are no longer wrongly split from * their combining characters. If you use a previous version number, you get the * exact broken behavior for backwards compatibility. * <li>As of 3.1, StandardTokenizer implements Unicode text segmentation, and * StopFilter correctly handles Unicode 4.0 supplementary characters in * stopwords. {@link ClassicTokenizer} and {@link ClassicAnalyzer} are the * pre-3.1 implementations of StandardTokenizer and StandardAnalyzer. * <li>As of 2.9, StopFilter preserves position increments * <li>As of 2.4, Tokens incorrectly identified as acronyms are corrected (see * <a href="https://issues.apache.org/jira/browse/LUCENE-1068">LUCENE-1068</a>) * </ul> */ public final class StandardAnalyzer extends StopwordAnalyzerBase { /** Default maximum allowed token length */ public static final int DEFAULT_MAX_TOKEN_LENGTH = 255; private int maxTokenLength = DEFAULT_MAX_TOKEN_LENGTH; final static NormalizeCharMap map = new NormalizeCharMap(); static { // ------------------------------------------------ // map.add(".", " "); map.add(":", " "); map.add(",", " "); map.add("_", " "); map.add("=", " "); map.add("-", " "); map.add("&", " "); // ------------------------------------------------ // } /** * An unmodifiable set containing some common English words that are usually * not useful for searching. */ public static final Set<?> STOP_WORDS_SET = StopAnalyzer.ENGLISH_STOP_WORDS_SET; /** * Builds an analyzer with the given stop words. * * @param matchVersion * Lucene version to match See {@link <a href="#version">above</a>} * @param stopWords * stop words */ public StandardAnalyzer(Version matchVersion, Set<?> stopWords) { super(matchVersion, stopWords); // replaceInvalidAcronym = matchVersion.onOrAfter(Version.LUCENE_24); } /** * Builds an analyzer with the default stop words ({@link #STOP_WORDS_SET}). * * @param matchVersion * Lucene version to match See {@link <a href="#version">above</a>} */ public StandardAnalyzer(Version matchVersion) { this(matchVersion, STOP_WORDS_SET); } /** * Builds an analyzer with the stop words from the given file. * * @see WordlistLoader#getWordSet(Reader, Version) * @param matchVersion * Lucene version to match See {@link <a href="#version">above</a>} * @param stopwords * File to read stop words from * @deprecated Use {@link #StandardAnalyzer(Version, Reader)} instead. */ @Deprecated public StandardAnalyzer(Version matchVersion, File stopwords) throws IOException { this(matchVersion, WordlistLoader.getWordSet(IOUtils.getDecodingReader(stopwords, IOUtils.CHARSET_UTF_8), matchVersion)); } /** * Builds an analyzer with the stop words from the given reader. * * @see WordlistLoader#getWordSet(Reader, Version) * @param matchVersion * Lucene version to match See {@link <a href="#version">above</a>} * @param stopwords * Reader to read stop words from */ public StandardAnalyzer(Version matchVersion, Reader stopwords) throws IOException { this(matchVersion, WordlistLoader.getWordSet(stopwords, matchVersion)); } /** * Set maximum allowed token length. If a token is seen that exceeds this * length then it is discarded. This setting only takes effect the next time * tokenStream or reusableTokenStream is called. */ public void setMaxTokenLength(int length) { maxTokenLength = length; } /** * @see #setMaxTokenLength */ public int getMaxTokenLength() { return maxTokenLength; } @Override protected TokenStreamComponents createComponents(final String fieldName, final Reader reader) { final StandardTokenizer src = new StandardTokenizer(matchVersion, reader); src.setMaxTokenLength(maxTokenLength); TokenStream tok = new StandardFilter(matchVersion, src); tok = new LowerCaseFilter(matchVersion, tok); tok = new StopFilter(matchVersion, tok, stopwords); return new TokenStreamComponents(src, tok) { @Override protected boolean reset(final Reader reader) throws IOException { Reader reader2 = new MappingCharFilter(map, CharReader.get(reader)); src.setMaxTokenLength(StandardAnalyzer.this.maxTokenLength); return super.reset(reader2); } }; } }