Java tutorial
/* * Licensed to Elasticsearch under one or more contributor * license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright * ownership. Elasticsearch licenses this file to you under * the Apache License, Version 2.0 (the "License"); you may * not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.elasticsearch.action.termvector; import com.carrotsearch.hppc.ObjectLongOpenHashMap; import com.carrotsearch.hppc.cursors.ObjectLongCursor; import org.apache.lucene.index.*; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamInput; import java.io.IOException; import java.util.Comparator; import java.util.Iterator; import static org.apache.lucene.util.ArrayUtil.grow; /** * This class represents the result of a {@link TermVectorRequest}. It works * exactly like the {@link Fields} class except for one thing: It can return * offsets and payloads even if positions are not present. You must call * nextPosition() anyway to move the counter although this method only returns * <tt>-1,</tt>, if no positions were returned by the {@link TermVectorRequest}. * <p/> * The data is stored in two byte arrays ({@code headerRef} and * {@code termVectors}, both {@link ByteRef}) that have the following format: * <p/> * {@code headerRef}: Stores offsets per field in the {@code termVectors} array * and some header information as {@link BytesRef}. Format is * <ul> * <li>String : "TV"</li> * <li>vint: version (=-1)</li> * <li>boolean: hasTermStatistics (are the term statistics stored?)</li> * <li>boolean: hasFieldStatitsics (are the field statistics stored?)</li> * <li>vint: number of fields</li> * <ul> * <li>String: field name 1</li> * <li>vint: offset in {@code termVectors} for field 1</li> * <li>...</li> * <li>String: field name last field</li> * <li>vint: offset in {@code termVectors} for last field</li> * </ul> * </ul> * <p/> * termVectors: Stores the actual term vectors as a {@link BytesRef}. * <p/> * Term vectors for each fields are stored in blocks, one for each field. The * offsets in {@code headerRef} are used to find where the block for a field * starts. Each block begins with a * <ul> * <li>vint: number of terms</li> * <li>boolean: positions (has it positions stored?)</li> * <li>boolean: offsets (has it offsets stored?)</li> * <li>boolean: payloads (has it payloads stored?)</li> * </ul> * If the field statistics were requested ({@code hasFieldStatistics} is true, * see {@code headerRef}), the following numbers are stored: * <ul> * <li>vlong: sum of total term freqencies of the field (sumTotalTermFreq)</li> * <li>vlong: sum of document frequencies for each term (sumDocFreq)</li> * <li>vint: number of documents in the shard that has an entry for this field * (docCount)</li> * </ul> * <p/> * After that, for each term it stores * <ul> * <ul> * <li>vint: term lengths</li> * <li>BytesRef: term name</li> * </ul> * <p/> * If term statistics are requested ({@code hasTermStatistics} is true, see * {@code headerRef}): * <ul> * <li>vint: document frequency, how often does this term appear in documents?</li> * <li>vlong: total term frequency. Sum of terms in this field.</li> * </ul> * After that * <ul> * <li>vint: frequency (always returned)</li> * <ul> * <li>vint: position_1 (if positions == true)</li> * <li>vint: startOffset_1 (if offset == true)</li> * <li>vint: endOffset_1 (if offset == true)</li> * <li>BytesRef: payload_1 (if payloads == true)</li> * <li>...</li> * <li>vint: endOffset_freqency (if offset == true)</li> * <li>BytesRef: payload_freqency (if payloads == true)</li> * <ul> * </ul> </ul> */ public final class TermVectorFields extends Fields { private final ObjectLongOpenHashMap<String> fieldMap; private final BytesReference termVectors; final boolean hasTermStatistic; final boolean hasFieldStatistic; /** * @param headerRef Stores offsets per field in the {@code termVectors} and some * header information as {@link BytesRef}. * @param termVectors Stores the actual term vectors as a {@link BytesRef}. */ public TermVectorFields(BytesReference headerRef, BytesReference termVectors) throws IOException { BytesStreamInput header = new BytesStreamInput(headerRef); fieldMap = new ObjectLongOpenHashMap<String>(); // here we read the header to fill the field offset map String headerString = header.readString(); assert headerString.equals("TV"); int version = header.readInt(); assert version == -1; hasTermStatistic = header.readBoolean(); hasFieldStatistic = header.readBoolean(); final int numFields = header.readVInt(); for (int i = 0; i < numFields; i++) { fieldMap.put((header.readString()), header.readVLong()); } header.close(); // reference to the term vector data this.termVectors = termVectors; } @Override public Iterator<String> iterator() { final Iterator<ObjectLongCursor<String>> iterator = fieldMap.iterator(); return new Iterator<String>() { @Override public boolean hasNext() { return iterator.hasNext(); } @Override public String next() { return iterator.next().key; } @Override public void remove() { throw new UnsupportedOperationException(); } }; } @Override public Terms terms(String field) throws IOException { // first, find where in the termVectors bytes the actual term vector for // this field is stored if (!fieldMap.containsKey(field)) { return null; // we don't have it. } long offset = fieldMap.lget(); final BytesStreamInput perFieldTermVectorInput = new BytesStreamInput(this.termVectors); perFieldTermVectorInput.reset(); perFieldTermVectorInput.skip(offset); // read how many terms.... final long numTerms = perFieldTermVectorInput.readVLong(); // ...if positions etc. were stored.... final boolean hasPositions = perFieldTermVectorInput.readBoolean(); final boolean hasOffsets = perFieldTermVectorInput.readBoolean(); final boolean hasPayloads = perFieldTermVectorInput.readBoolean(); // read the field statistics final long sumTotalTermFreq = hasFieldStatistic ? readPotentiallyNegativeVLong(perFieldTermVectorInput) : -1; final long sumDocFreq = hasFieldStatistic ? readPotentiallyNegativeVLong(perFieldTermVectorInput) : -1; final int docCount = hasFieldStatistic ? readPotentiallyNegativeVInt(perFieldTermVectorInput) : -1; return new Terms() { @Override public TermsEnum iterator(TermsEnum reuse) throws IOException { // convert bytes ref for the terms to actual data return new TermsEnum() { int currentTerm = 0; int freq = 0; int docFreq = -1; long totalTermFrequency = -1; int[] positions = new int[1]; int[] startOffsets = new int[1]; int[] endOffsets = new int[1]; BytesRef[] payloads = new BytesRef[1]; final BytesRef spare = new BytesRef(); @Override public BytesRef next() throws IOException { if (currentTerm++ < numTerms) { // term string. first the size... int termVectorSize = perFieldTermVectorInput.readVInt(); spare.grow(termVectorSize); // ...then the value. perFieldTermVectorInput.readBytes(spare.bytes, 0, termVectorSize); spare.length = termVectorSize; if (hasTermStatistic) { docFreq = readPotentiallyNegativeVInt(perFieldTermVectorInput); totalTermFrequency = readPotentiallyNegativeVLong(perFieldTermVectorInput); } freq = readPotentiallyNegativeVInt(perFieldTermVectorInput); // grow the arrays to read the values. this is just // for performance reasons. Re-use memory instead of // realloc. growBuffers(); // finally, read the values into the arrays // curentPosition etc. so that we can just iterate // later writeInfos(perFieldTermVectorInput); return spare; } else { return null; } } private void writeInfos(final BytesStreamInput input) throws IOException { for (int i = 0; i < freq; i++) { if (hasPositions) { positions[i] = input.readVInt(); } if (hasOffsets) { startOffsets[i] = input.readVInt(); endOffsets[i] = input.readVInt(); } if (hasPayloads) { int payloadLength = input.readVInt(); if (payloads[i] == null) { payloads[i] = new BytesRef(payloadLength); } else { payloads[i].grow(payloadLength); } input.readBytes(payloads[i].bytes, 0, payloadLength); payloads[i].length = payloadLength; payloads[i].offset = 0; } } } private void growBuffers() { if (hasPositions) { positions = grow(positions, freq); } if (hasOffsets) { startOffsets = grow(startOffsets, freq); endOffsets = grow(endOffsets, freq); } if (hasPayloads) { if (payloads.length < freq) { final BytesRef[] newArray = new BytesRef[ArrayUtil.oversize(freq, RamUsageEstimator.NUM_BYTES_OBJECT_REF)]; System.arraycopy(payloads, 0, newArray, 0, payloads.length); payloads = newArray; } } } @Override public Comparator<BytesRef> getComparator() { return BytesRef.getUTF8SortedAsUnicodeComparator(); } @Override public SeekStatus seekCeil(BytesRef text) throws IOException { throw new UnsupportedOperationException(); } @Override public void seekExact(long ord) throws IOException { throw new UnsupportedOperationException("Seek is not supported"); } @Override public BytesRef term() throws IOException { return spare; } @Override public long ord() throws IOException { throw new UnsupportedOperationException("ordinals are not supported"); } @Override public int docFreq() throws IOException { return docFreq; } @Override public long totalTermFreq() throws IOException { return totalTermFrequency; } @Override public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException { return docsAndPositions(liveDocs, reuse instanceof DocsAndPositionsEnum ? (DocsAndPositionsEnum) reuse : null, 0); } @Override public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { final TermVectorsDocsAndPosEnum retVal = (reuse instanceof TermVectorsDocsAndPosEnum ? (TermVectorsDocsAndPosEnum) reuse : new TermVectorsDocsAndPosEnum()); return retVal.reset(hasPositions ? positions : null, hasOffsets ? startOffsets : null, hasOffsets ? endOffsets : null, hasPayloads ? payloads : null, freq); } }; } @Override public Comparator<BytesRef> getComparator() { return BytesRef.getUTF8SortedAsUnicodeComparator(); } @Override public long size() throws IOException { return numTerms; } @Override public long getSumTotalTermFreq() throws IOException { return sumTotalTermFreq; } @Override public long getSumDocFreq() throws IOException { return sumDocFreq; } @Override public int getDocCount() throws IOException { return docCount; } @Override public boolean hasFreqs() { return true; } @Override public boolean hasOffsets() { return hasOffsets; } @Override public boolean hasPositions() { return hasPositions; } @Override public boolean hasPayloads() { return hasPayloads; } }; } @Override public int size() { return fieldMap.size(); } private final class TermVectorsDocsAndPosEnum extends DocsAndPositionsEnum { private boolean hasPositions; private boolean hasOffsets; private boolean hasPayloads; int curPos = -1; int doc = -1; private int freq; private int[] startOffsets; private int[] positions; private BytesRef[] payloads; private int[] endOffsets; private DocsAndPositionsEnum reset(int[] positions, int[] startOffsets, int[] endOffsets, BytesRef[] payloads, int freq) { curPos = -1; doc = -1; this.hasPositions = positions != null; this.hasOffsets = startOffsets != null; this.hasPayloads = payloads != null; this.freq = freq; this.startOffsets = startOffsets; this.endOffsets = endOffsets; this.payloads = payloads; this.positions = positions; return this; } @Override public int nextDoc() throws IOException { return doc = (doc == -1 ? 0 : NO_MORE_DOCS); } @Override public int docID() { return doc; } @Override public int advance(int target) throws IOException { while (nextDoc() < target && doc != NO_MORE_DOCS) { } return doc; } @Override public int freq() throws IOException { return freq; } // call nextPosition once before calling this one // because else counter is not advanced @Override public int startOffset() throws IOException { assert curPos < freq && curPos >= 0; return hasOffsets ? startOffsets[curPos] : -1; } @Override // can return -1 if posistions were not requested or // stored but offsets were stored and requested public int nextPosition() throws IOException { assert curPos + 1 < freq; ++curPos; // this is kind of cheating but if you don't need positions // we safe lots fo space on the wire return hasPositions ? positions[curPos] : -1; } @Override public BytesRef getPayload() throws IOException { assert curPos < freq && curPos >= 0; return hasPayloads ? payloads[curPos] : null; } @Override public int endOffset() throws IOException { assert curPos < freq && curPos >= 0; return hasOffsets ? endOffsets[curPos] : -1; } @Override public long cost() { return 1; } } // read a vInt. this is used if the integer might be negative. In this case, // the writer writes a 0 for -1 or value +1 and accordingly we have to // substract 1 again // adds one to mock not existing term freq int readPotentiallyNegativeVInt(BytesStreamInput stream) throws IOException { return stream.readVInt() - 1; } // read a vLong. this is used if the integer might be negative. In this // case, the writer writes a 0 for -1 or value +1 and accordingly we have to // substract 1 again // adds one to mock not existing term freq long readPotentiallyNegativeVLong(BytesStreamInput stream) throws IOException { return stream.readVLong() - 1; } }