Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.lucene.analysis.icu.segmentation; import java.io.IOException; import java.io.Reader; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.icu.tokenattributes.ScriptAttribute; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.TypeAttribute; import org.apache.lucene.util.AttributeFactory; import com.ibm.icu.lang.UCharacter; import com.ibm.icu.text.BreakIterator; /** * Breaks text into words according to UAX #29: Unicode Text Segmentation * (http://www.unicode.org/reports/tr29/) * <p> * Words are broken across script boundaries, then segmented according to * the BreakIterator and typing provided by the {@link ICUTokenizerConfig} * </p> * @see ICUTokenizerConfig * @lucene.experimental */ public final class ICUTokenizer extends Tokenizer { private static final int IOBUFFER = 4096; private final char buffer[] = new char[IOBUFFER]; /** true length of text in the buffer */ private int length = 0; /** length in buffer that can be evaluated safely, up to a safe end point */ private int usableLength = 0; /** accumulated offset of previous buffers for this reader, for offsetAtt */ private int offset = 0; private final CompositeBreakIterator breaker; /* tokenizes a char[] of text */ private final ICUTokenizerConfig config; private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class); private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class); private final ScriptAttribute scriptAtt = addAttribute(ScriptAttribute.class); /** * Construct a new ICUTokenizer that breaks text into words from the given * Reader. * <p> * The default script-specific handling is used. * <p> * The default attribute factory is used. * * @see DefaultICUTokenizerConfig */ public ICUTokenizer() { this(new DefaultICUTokenizerConfig(true, true)); } /** * Construct a new ICUTokenizer that breaks text into words from the given * Reader, using a tailored BreakIterator configuration. * <p> * The default attribute factory is used. * * @param config Tailored BreakIterator configuration */ public ICUTokenizer(ICUTokenizerConfig config) { this(DEFAULT_TOKEN_ATTRIBUTE_FACTORY, config); } /** * Construct a new ICUTokenizer that breaks text into words from the given * Reader, using a tailored BreakIterator configuration. * * @param factory AttributeFactory to use * @param config Tailored BreakIterator configuration */ public ICUTokenizer(AttributeFactory factory, ICUTokenizerConfig config) { super(factory); this.config = config; breaker = new CompositeBreakIterator(config); } @Override public boolean incrementToken() throws IOException { clearAttributes(); if (length == 0) refill(); while (!incrementTokenBuffer()) { refill(); if (length <= 0) // no more bytes to read; return false; } return true; } @Override public void reset() throws IOException { super.reset(); breaker.setText(buffer, 0, 0); length = usableLength = offset = 0; } @Override public void end() throws IOException { super.end(); final int finalOffset = (length < 0) ? offset : offset + length; offsetAtt.setOffset(correctOffset(finalOffset), correctOffset(finalOffset)); } /* * This tokenizes text based upon the longest matching rule, and because of * this, isn't friendly to a Reader. * * Text is read from the input stream in 4kB chunks. Within a 4kB chunk of * text, the last unambiguous break point is found (in this implementation: * white space character) Any remaining characters represent possible partial * words, so are appended to the front of the next chunk. * * There is the possibility that there are no unambiguous break points within * an entire 4kB chunk of text (binary data). So there is a maximum word limit * of 4kB since it will not try to grow the buffer in this case. */ /** * Returns the last unambiguous break position in the text. * * @return position of character, or -1 if one does not exist */ private int findSafeEnd() { for (int i = length - 1; i >= 0; i--) if (UCharacter.isWhitespace(buffer[i])) return i + 1; return -1; } /** * Refill the buffer, accumulating the offset and setting usableLength to the * last unambiguous break position * * @throws IOException If there is a low-level I/O error. */ private void refill() throws IOException { offset += usableLength; int leftover = length - usableLength; System.arraycopy(buffer, usableLength, buffer, 0, leftover); int requested = buffer.length - leftover; int returned = read(input, buffer, leftover, requested); length = returned + leftover; if (returned < requested) /* reader has been emptied, process the rest */ usableLength = length; else { /* still more data to be read, find a safe-stopping place */ usableLength = findSafeEnd(); if (usableLength < 0) usableLength = length; /* * more than IOBUFFER of text without space, * gonna possibly truncate tokens */ } breaker.setText(buffer, 0, Math.max(0, usableLength)); } // TODO: refactor to a shared readFully somewhere // (NGramTokenizer does this too): /** commons-io's readFully, but without bugs if offset != 0 */ private static int read(Reader input, char[] buffer, int offset, int length) throws IOException { assert length >= 0 : "length must not be negative: " + length; int remaining = length; while (remaining > 0) { int location = length - remaining; int count = input.read(buffer, offset + location, remaining); if (-1 == count) { // EOF break; } remaining -= count; } return length - remaining; } /* * return true if there is a token from the buffer, or null if it is * exhausted. */ private boolean incrementTokenBuffer() { int start = breaker.current(); assert start != BreakIterator.DONE; // find the next set of boundaries, skipping over non-tokens (rule status 0) int end = breaker.next(); while (end != BreakIterator.DONE && breaker.getRuleStatus() == 0) { start = end; end = breaker.next(); } if (end == BreakIterator.DONE) { return false; // BreakIterator exhausted } termAtt.copyBuffer(buffer, start, end - start); offsetAtt.setOffset(correctOffset(offset + start), correctOffset(offset + end)); typeAtt.setType(config.getType(breaker.getScriptCode(), breaker.getRuleStatus())); scriptAtt.setCode(breaker.getScriptCode()); return true; } }