Java tutorial
/* * Created on 28-Oct-2004 */ package edu.mit.ll.vizlinc.highlight; /** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import java.io.IOException; import java.io.StringReader; import java.util.ArrayList; import java.util.Comparator; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.TermFreqVector; import org.apache.lucene.index.TermPositionVector; import org.apache.lucene.index.TermVectorOffsetInfo; import org.apache.lucene.util.ArrayUtil; /** * Hides implementation issues associated with obtaining a TokenStream for use * with the higlighter - can obtain from TermFreqVectors with offsets and * (optionally) positions or from Analyzer class reparsing the stored content. */ public class TokenSources { /** * A convenience method that tries to first get a TermPositionVector for the * specified docId, then, falls back to using the passed in * {@link org.apache.lucene.document.Document} to retrieve the TokenStream. * This is useful when you already have the document, but would prefer to use * the vector first. * * @param reader The {@link org.apache.lucene.index.IndexReader} to use to try * and get the vector from * @param docId The docId to retrieve. * @param field The field to retrieve on the document * @param doc The document to fall back on * @param analyzer The analyzer to use for creating the TokenStream if the * vector doesn't exist * @return The {@link org.apache.lucene.analysis.TokenStream} for the * {@link org.apache.lucene.document.Fieldable} on the * {@link org.apache.lucene.document.Document} * @throws IOException if there was an error loading */ public static TokenStream getAnyTokenStream(IndexReader reader, int docId, String field, Document doc, Analyzer analyzer) throws IOException { TokenStream ts = null; TermFreqVector tfv = reader.getTermFreqVector(docId, field); if (tfv != null) { if (tfv instanceof TermPositionVector) { ts = getTokenStream((TermPositionVector) tfv); } } // No token info stored so fall back to analyzing raw content if (ts == null) { ts = getTokenStream(doc, field, analyzer); } return ts; } /** * A convenience method that tries a number of approaches to getting a token * stream. The cost of finding there are no termVectors in the index is * minimal (1000 invocations still registers 0 ms). So this "lazy" (flexible?) * approach to coding is probably acceptable * * @param reader * @param docId * @param field * @param analyzer * @return null if field not stored correctly * @throws IOException */ public static TokenStream getAnyTokenStream(IndexReader reader, int docId, String field, Analyzer analyzer) throws IOException { TokenStream ts = null; TermFreqVector tfv = reader.getTermFreqVector(docId, field); if (tfv != null) { if (tfv instanceof TermPositionVector) { ts = getTokenStream((TermPositionVector) tfv); } } // No token info stored so fall back to analyzing raw content if (ts == null) { ts = getTokenStream(reader, docId, field, analyzer); } return ts; } public static TokenStream getTokenStream(TermPositionVector tpv) { // assumes the worst and makes no assumptions about token position // sequences. return getTokenStream(tpv, false); } /** * Low level api. Returns a token stream or null if no offset info available * in index. This can be used to feed the highlighter with a pre-parsed token * stream * * In my tests the speeds to recreate 1000 token streams using this method * are: - with TermVector offset only data stored - 420 milliseconds - with * TermVector offset AND position data stored - 271 milliseconds (nb timings * for TermVector with position data are based on a tokenizer with contiguous * positions - no overlaps or gaps) The cost of not using TermPositionVector * to store pre-parsed content and using an analyzer to re-parse the original * content: - reanalyzing the original content - 980 milliseconds * * The re-analyze timings will typically vary depending on - 1) The complexity * of the analyzer code (timings above were using a * stemmer/lowercaser/stopword combo) 2) The number of other fields (Lucene * reads ALL fields off the disk when accessing just one document field - can * cost dear!) 3) Use of compression on field storage - could be faster due to * compression (less disk IO) or slower (more CPU burn) depending on the * content. * * @param tpv * @param tokenPositionsGuaranteedContiguous true if the token position * numbers have no overlaps or gaps. If looking to eek out the last * drops of performance, set to true. If in doubt, set to false. */ public static TokenStream getTokenStream(TermPositionVector tpv, boolean tokenPositionsGuaranteedContiguous) { if (!tokenPositionsGuaranteedContiguous && tpv.getTermPositions(0) != null) { return new TokenStreamFromTermPositionVector(tpv); } // an object used to iterate across an array of tokens final class StoredTokenStream extends TokenStream { Token tokens[]; int currentToken = 0; CharTermAttribute termAtt; OffsetAttribute offsetAtt; PositionIncrementAttribute posincAtt; StoredTokenStream(Token tokens[]) { this.tokens = tokens; termAtt = addAttribute(CharTermAttribute.class); offsetAtt = addAttribute(OffsetAttribute.class); posincAtt = addAttribute(PositionIncrementAttribute.class); } @Override public boolean incrementToken() throws IOException { if (currentToken >= tokens.length) { return false; } Token token = tokens[currentToken++]; clearAttributes(); termAtt.setEmpty().append(token); offsetAtt.setOffset(token.startOffset(), token.endOffset()); posincAtt.setPositionIncrement(currentToken <= 1 || tokens[currentToken - 1].startOffset() > tokens[currentToken - 2].startOffset() ? 1 : 0); return true; } } // code to reconstruct the original sequence of Tokens String[] terms = tpv.getTerms(); int[] freq = tpv.getTermFrequencies(); int totalTokens = 0; for (int t = 0; t < freq.length; t++) { totalTokens += freq[t]; } Token tokensInOriginalOrder[] = new Token[totalTokens]; ArrayList<Token> unsortedTokens = null; for (int t = 0; t < freq.length; t++) { TermVectorOffsetInfo[] offsets = tpv.getOffsets(t); if (offsets == null) { throw new IllegalArgumentException("Required TermVector Offset information was not found"); } int[] pos = null; if (tokenPositionsGuaranteedContiguous) { // try get the token position info to speed up assembly of tokens into // sorted sequence pos = tpv.getTermPositions(t); } if (pos == null) { // tokens NOT stored with positions or not guaranteed contiguous - must // add to list and sort later if (unsortedTokens == null) { unsortedTokens = new ArrayList<Token>(); } for (int tp = 0; tp < offsets.length; tp++) { Token token = new Token(terms[t], offsets[tp].getStartOffset(), offsets[tp].getEndOffset()); unsortedTokens.add(token); } } else { // We have positions stored and a guarantee that the token position // information is contiguous // This may be fast BUT wont work if Tokenizers used which create >1 // token in same position or // creates jumps in position numbers - this code would fail under those // circumstances // tokens stored with positions - can use this to index straight into // sorted array for (int tp = 0; tp < pos.length; tp++) { Token token = new Token(terms[t], offsets[tp].getStartOffset(), offsets[tp].getEndOffset()); tokensInOriginalOrder[pos[tp]] = token; } } } // If the field has been stored without position data we must perform a sort if (unsortedTokens != null) { tokensInOriginalOrder = unsortedTokens.toArray(new Token[unsortedTokens.size()]); ArrayUtil.mergeSort(tokensInOriginalOrder, new Comparator<Token>() { public int compare(Token t1, Token t2) { if (t1.startOffset() == t2.startOffset()) return t1.endOffset() - t2.endOffset(); else return t1.startOffset() - t2.startOffset(); } }); } return new StoredTokenStream(tokensInOriginalOrder); } public static TokenStream getTokenStream(IndexReader reader, int docId, String field) throws IOException { TermFreqVector tfv = reader.getTermFreqVector(docId, field); if (tfv == null) { throw new IllegalArgumentException( field + " in doc #" + docId + "does not have any term position data stored"); } if (tfv instanceof TermPositionVector) { TermPositionVector tpv = (TermPositionVector) reader.getTermFreqVector(docId, field); return getTokenStream(tpv); } throw new IllegalArgumentException( field + " in doc #" + docId + "does not have any term position data stored"); } // convenience method public static TokenStream getTokenStream(IndexReader reader, int docId, String field, Analyzer analyzer) throws IOException { Document doc = reader.document(docId); return getTokenStream(doc, field, analyzer); } public static TokenStream getTokenStream(Document doc, String field, Analyzer analyzer) { String contents = doc.get(field); if (contents == null) { throw new IllegalArgumentException( "Field " + field + " in document is not stored and cannot be analyzed"); } return getTokenStream(field, contents, analyzer); } // convenience method public static TokenStream getTokenStream(String field, String contents, Analyzer analyzer) { try { return analyzer.reusableTokenStream(field, new StringReader(contents)); } catch (IOException ex) { throw new RuntimeException(ex); } } }