Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.lucene.analysis.util; import java.io.IOException; import java.util.Objects; import java.util.function.IntPredicate; import org.apache.lucene.analysis.CharacterUtils; import org.apache.lucene.analysis.CharacterUtils.CharacterBuffer; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.LetterTokenizer; import org.apache.lucene.analysis.core.WhitespaceTokenizer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.util.AttributeFactory; import static org.apache.lucene.analysis.standard.StandardTokenizer.MAX_TOKEN_LENGTH_LIMIT; /** * An abstract base class for simple, character-oriented tokenizers. * <p> * The base class also provides factories to create instances of * {@code CharTokenizer} using Java 8 lambdas or method references. * It is possible to create an instance which behaves exactly like * {@link LetterTokenizer}: * <pre class="prettyprint lang-java"> * Tokenizer tok = CharTokenizer.fromTokenCharPredicate(Character::isLetter); * </pre> */ public abstract class CharTokenizer extends Tokenizer { /** * Creates a new {@link CharTokenizer} instance */ public CharTokenizer() { this.maxTokenLen = DEFAULT_MAX_WORD_LEN; } /** * Creates a new {@link CharTokenizer} instance * * @param factory * the attribute factory to use for this {@link Tokenizer} */ public CharTokenizer(AttributeFactory factory) { super(factory); this.maxTokenLen = DEFAULT_MAX_WORD_LEN; } /** * Creates a new {@link CharTokenizer} instance * * @param factory the attribute factory to use for this {@link Tokenizer} * @param maxTokenLen maximum token length the tokenizer will emit. * Must be greater than 0 and less than MAX_TOKEN_LENGTH_LIMIT (1024*1024) * @throws IllegalArgumentException if maxTokenLen is invalid. */ public CharTokenizer(AttributeFactory factory, int maxTokenLen) { super(factory); if (maxTokenLen > MAX_TOKEN_LENGTH_LIMIT || maxTokenLen <= 0) { throw new IllegalArgumentException("maxTokenLen must be greater than 0 and less than " + MAX_TOKEN_LENGTH_LIMIT + " passed: " + maxTokenLen); } this.maxTokenLen = maxTokenLen; } /** * Creates a new instance of CharTokenizer using a custom predicate, supplied as method reference or lambda expression. * The predicate should return {@code true} for all valid token characters. * <p> * This factory is intended to be used with lambdas or method references. E.g., an elegant way * to create an instance which behaves exactly as {@link LetterTokenizer} is: * <pre class="prettyprint lang-java"> * Tokenizer tok = CharTokenizer.fromTokenCharPredicate(Character::isLetter); * </pre> */ public static CharTokenizer fromTokenCharPredicate(final IntPredicate tokenCharPredicate) { return fromTokenCharPredicate(DEFAULT_TOKEN_ATTRIBUTE_FACTORY, tokenCharPredicate); } /** * Creates a new instance of CharTokenizer with the supplied attribute factory using a custom predicate, supplied as method reference or lambda expression. * The predicate should return {@code true} for all valid token characters. * <p> * This factory is intended to be used with lambdas or method references. E.g., an elegant way * to create an instance which behaves exactly as {@link LetterTokenizer} is: * <pre class="prettyprint lang-java"> * Tokenizer tok = CharTokenizer.fromTokenCharPredicate(factory, Character::isLetter); * </pre> */ public static CharTokenizer fromTokenCharPredicate(AttributeFactory factory, final IntPredicate tokenCharPredicate) { Objects.requireNonNull(tokenCharPredicate, "predicate must not be null."); return new CharTokenizer(factory) { @Override protected boolean isTokenChar(int c) { return tokenCharPredicate.test(c); } }; } /** * Creates a new instance of CharTokenizer using a custom predicate, supplied as method reference or lambda expression. * The predicate should return {@code true} for all valid token separator characters. * This method is provided for convenience to easily use predicates that are negated * (they match the separator characters, not the token characters). * <p> * This factory is intended to be used with lambdas or method references. E.g., an elegant way * to create an instance which behaves exactly as {@link WhitespaceTokenizer} is: * <pre class="prettyprint lang-java"> * Tokenizer tok = CharTokenizer.fromSeparatorCharPredicate(Character::isWhitespace); * </pre> */ public static CharTokenizer fromSeparatorCharPredicate(final IntPredicate separatorCharPredicate) { return fromSeparatorCharPredicate(DEFAULT_TOKEN_ATTRIBUTE_FACTORY, separatorCharPredicate); } /** * Creates a new instance of CharTokenizer with the supplied attribute factory using a custom predicate, supplied as method reference or lambda expression. * The predicate should return {@code true} for all valid token separator characters. * <p> * This factory is intended to be used with lambdas or method references. E.g., an elegant way * to create an instance which behaves exactly as {@link WhitespaceTokenizer} is: * <pre class="prettyprint lang-java"> * Tokenizer tok = CharTokenizer.fromSeparatorCharPredicate(factory, Character::isWhitespace); * </pre> */ public static CharTokenizer fromSeparatorCharPredicate(AttributeFactory factory, final IntPredicate separatorCharPredicate) { return fromTokenCharPredicate(factory, separatorCharPredicate.negate()); } private int offset = 0, bufferIndex = 0, dataLen = 0, finalOffset = 0; public static final int DEFAULT_MAX_WORD_LEN = 255; private static final int IO_BUFFER_SIZE = 4096; private final int maxTokenLen; private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class); private final CharacterBuffer ioBuffer = CharacterUtils.newCharacterBuffer(IO_BUFFER_SIZE); /** * Returns true iff a codepoint should be included in a token. This tokenizer * generates as tokens adjacent sequences of codepoints which satisfy this * predicate. Codepoints for which this is false are used to define token * boundaries and are not included in tokens. */ protected abstract boolean isTokenChar(int c); @Override public final boolean incrementToken() throws IOException { clearAttributes(); int length = 0; int start = -1; // this variable is always initialized int end = -1; char[] buffer = termAtt.buffer(); while (true) { if (bufferIndex >= dataLen) { offset += dataLen; CharacterUtils.fill(ioBuffer, input); // read supplementary char aware with CharacterUtils if (ioBuffer.getLength() == 0) { dataLen = 0; // so next offset += dataLen won't decrement offset if (length > 0) { break; } else { finalOffset = correctOffset(offset); return false; } } dataLen = ioBuffer.getLength(); bufferIndex = 0; } // use CharacterUtils here to support < 3.1 UTF-16 code unit behavior if the char based methods are gone final int c = Character.codePointAt(ioBuffer.getBuffer(), bufferIndex, ioBuffer.getLength()); final int charCount = Character.charCount(c); bufferIndex += charCount; if (isTokenChar(c)) { // if it's a token char if (length == 0) { // start of token assert start == -1; start = offset + bufferIndex - charCount; end = start; } else if (length >= buffer.length - 1) { // check if a supplementary could run out of bounds buffer = termAtt.resizeBuffer(2 + length); // make sure a supplementary fits in the buffer } end += charCount; length += Character.toChars(c, buffer, length); // buffer it, normalized if (length >= maxTokenLen) { // buffer overflow! make sure to check for >= surrogate pair could break == test break; } } else if (length > 0) { // at non-Letter w/ chars break; // return 'em } } termAtt.setLength(length); assert start != -1; offsetAtt.setOffset(correctOffset(start), finalOffset = correctOffset(end)); return true; } @Override public final void end() throws IOException { super.end(); // set final offset offsetAtt.setOffset(finalOffset, finalOffset); } @Override public void reset() throws IOException { super.reset(); bufferIndex = 0; offset = 0; dataLen = 0; finalOffset = 0; ioBuffer.reset(); // make sure to reset the IO buffer!! } }