Example usage for org.apache.lucene.store DataInput readVInt

List of usage examples for org.apache.lucene.store DataInput readVInt

  1. HOME
  2. Java
  3. org.apache.lucene
  4. org.apache.lucene.store.*
  5. DataInput
  6. readVInt

Introduction

In this page you can find the example usage for org.apache.lucene.store DataInput readVInt.

Prototype

public int readVInt() throws IOException

Source Link

Document

Reads an int stored in variable-length format.

Usage

From source file:com.browseengine.bobo.geosearch.impl.MappedFieldNameFilterConverter.java

License:Apache License

@Override
public void loadFromInput(DataInput input) throws IOException {
    int version = input.readVInt(); //read version

    int mapSize = input.readVInt();
    bitmasks = new HashMap<String, Byte>(mapSize);
    for (int i = 0; i < mapSize; i++) {
        String fieldName = input.readString();
        Byte filterByte = input.readByte();

        bitmasks.put(fieldName, filterByte);
    }// w w w.j  a v  a2s .  co m
}

From source file:com.browseengine.bobo.geosearch.merge.impl.BufferedGeoMerger.java

License:Apache License

/**
 * /*from   w ww  .  j  a v a 2  s. c om*/
 * @param directory
 * @param geoFileName
 * @param fieldNameFilterConverter
 * @return true iff successful
 * @throws IOException
 */
protected boolean loadFieldNameFilterConverter(Directory directory, String geoFileName,
        IFieldNameFilterConverter fieldNameFilterConverter) throws IOException {
    try {
        DataInput input = directory.openInput(geoFileName);
        input.readVInt(); //read version
        input.readInt(); //throw out tree position
        input.readVInt(); //throw out tree size
        input.readVInt(); //throw out record length

        fieldNameFilterConverter.loadFromInput(input);

        return true;
    } catch (FileNotFoundException e) {
        LOGGER.warn("suppressing missing geo file pair, treating as no field names: " + e);
        return false;
    }
}

From source file:com.github.cstoku.neologd.unidic.lucene.analysis.ja.dict.BinaryDictionary.java

License:Apache License

protected BinaryDictionary() throws IOException {
    InputStream mapIS = null, dictIS = null, posIS = null;
    int[] targetMapOffsets = null, targetMap = null;
    String[] posDict = null;/*from   w  ww  .  j  ava2s .c  o  m*/
    String[] inflFormDict = null;
    String[] inflTypeDict = null;
    ByteBuffer buffer = null;
    boolean success = false;
    try {
        mapIS = getResource(TARGETMAP_FILENAME_SUFFIX);
        mapIS = new BufferedInputStream(mapIS);
        DataInput in = new InputStreamDataInput(mapIS);
        CodecUtil.checkHeader(in, TARGETMAP_HEADER, VERSION, VERSION);
        targetMap = new int[in.readVInt()];
        targetMapOffsets = new int[in.readVInt()];
        int accum = 0, sourceId = 0;
        for (int ofs = 0; ofs < targetMap.length; ofs++) {
            final int val = in.readVInt();
            if ((val & 0x01) != 0) {
                targetMapOffsets[sourceId] = ofs;
                sourceId++;
            }
            accum += val >>> 1;
            targetMap[ofs] = accum;
        }
        if (sourceId + 1 != targetMapOffsets.length)
            throw new IOException("targetMap file format broken");
        targetMapOffsets[sourceId] = targetMap.length;
        mapIS.close();
        mapIS = null;

        posIS = getResource(POSDICT_FILENAME_SUFFIX);
        posIS = new BufferedInputStream(posIS);
        in = new InputStreamDataInput(posIS);
        CodecUtil.checkHeader(in, POSDICT_HEADER, VERSION, VERSION);
        int posSize = in.readVInt();
        posDict = new String[posSize];
        inflTypeDict = new String[posSize];
        inflFormDict = new String[posSize];
        for (int j = 0; j < posSize; j++) {
            posDict[j] = in.readString();
            inflTypeDict[j] = in.readString();
            inflFormDict[j] = in.readString();
            // this is how we encode null inflections
            if (inflTypeDict[j].length() == 0) {
                inflTypeDict[j] = null;
            }
            if (inflFormDict[j].length() == 0) {
                inflFormDict[j] = null;
            }
        }
        posIS.close();
        posIS = null;

        dictIS = getResource(DICT_FILENAME_SUFFIX);
        // no buffering here, as we load in one large buffer
        in = new InputStreamDataInput(dictIS);
        CodecUtil.checkHeader(in, DICT_HEADER, VERSION, VERSION);
        final int size = in.readVInt();
        final ByteBuffer tmpBuffer = ByteBuffer.allocateDirect(size);
        final ReadableByteChannel channel = Channels.newChannel(dictIS);
        final int read = channel.read(tmpBuffer);
        if (read != size) {
            throw new EOFException("Cannot read whole dictionary");
        }
        dictIS.close();
        dictIS = null;
        buffer = tmpBuffer.asReadOnlyBuffer();
        success = true;
    } finally {
        if (success) {
            IOUtils.close(mapIS, posIS, dictIS);
        } else {
            IOUtils.closeWhileHandlingException(mapIS, posIS, dictIS);
        }
    }

    this.targetMap = targetMap;
    this.targetMapOffsets = targetMapOffsets;
    this.posDict = posDict;
    this.inflTypeDict = inflTypeDict;
    this.inflFormDict = inflFormDict;
    this.buffer = buffer;
}

From source file:com.github.cstoku.neologd.unidic.lucene.analysis.ja.dict.ConnectionCosts.java

License:Apache License

private ConnectionCosts() throws IOException {
    InputStream is = null;//from w  w  w.j  a va2s.  c  om
    short[][] costs = null;
    boolean success = false;
    try {
        is = BinaryDictionary.getClassResource(getClass(), FILENAME_SUFFIX);
        is = new BufferedInputStream(is);
        final DataInput in = new InputStreamDataInput(is);
        CodecUtil.checkHeader(in, HEADER, VERSION, VERSION);
        int forwardSize = in.readVInt();
        int backwardSize = in.readVInt();
        costs = new short[backwardSize][forwardSize];
        int accum = 0;
        for (int j = 0; j < costs.length; j++) {
            final short[] a = costs[j];
            for (int i = 0; i < a.length; i++) {
                accum += in.readZInt();
                a[i] = (short) accum;
            }
        }
        success = true;
    } finally {
        if (success) {
            IOUtils.close(is);
        } else {
            IOUtils.closeWhileHandlingException(is);
        }
    }

    this.costs = costs;
}

From source file:com.lucure.core.codec.CompressingStoredFieldsReader.java

License:Apache License

private static void readField(DataInput in, RestrictedStoredFieldVisitor visitor, FieldInfo info, int bits,
        FieldVisibility fieldVisibility) throws IOException {

    switch (bits & TYPE_MASK) {
    case BYTE_ARR:
        int length = in.readVInt();
        byte[] data = new byte[length];
        in.readBytes(data, 0, length);//from  w w  w  .  j a  va  2  s.co m
        visitor.binaryField(info, data, fieldVisibility);
        break;
    case STRING:
        length = in.readVInt();
        data = new byte[length];
        in.readBytes(data, 0, length);
        visitor.stringField(info, new String(data, IOUtils.CHARSET_UTF_8), fieldVisibility);
        break;
    case NUMERIC_INT:
        int intValue = in.readInt();
        visitor.intField(info, intValue, fieldVisibility);
        break;
    case NUMERIC_FLOAT:
        float floatValue = Float.intBitsToFloat(in.readInt());
        visitor.floatField(info, floatValue, fieldVisibility);
        break;
    case NUMERIC_LONG:
        long longValue = in.readLong();
        visitor.longField(info, longValue, fieldVisibility);
        break;
    case NUMERIC_DOUBLE:
        double doubleValue = Double.longBitsToDouble(in.readLong());
        visitor.doubleField(info, doubleValue, fieldVisibility);
        break;
    default:
        throw new AssertionError("Unknown type flag: " + Integer.toHexString(bits));
    }
}

From source file:com.lucure.core.codec.CompressingStoredFieldsReader.java

License:Apache License

private static void skipField(DataInput in, int bits, FieldVisibility cv) throws IOException {
    switch (bits & TYPE_MASK) {
    case BYTE_ARR:
    case STRING:/*from w w w  .  j a  v  a  2 s .  co m*/
        final int length = in.readVInt();
        in.skipBytes(length);
        break;
    case NUMERIC_INT:
    case NUMERIC_FLOAT:
        in.readInt();
        break;
    case NUMERIC_LONG:
    case NUMERIC_DOUBLE:
        in.readLong();
        break;
    default:
        throw new AssertionError("Unknown type flag: " + Integer.toHexString(bits));
    }
}

From source file:com.lucure.core.codec.CompressingStoredFieldsReader.java

License:Apache License

@Override
public void visitDocument(int docID, StoredFieldVisitor visitor) throws IOException {
    fieldsStream.seek(indexReader.getStartPointer(docID));

    final int docBase = fieldsStream.readVInt();
    final int chunkDocs = fieldsStream.readVInt();
    if (docID < docBase || docID >= docBase + chunkDocs || docBase + chunkDocs > numDocs) {
        throw new CorruptIndexException("Corrupted: docID=" + docID + ", docBase=" + docBase + ", chunkDocs="
                + chunkDocs + ", numDocs=" + numDocs + " (resource=" + fieldsStream + ")");
    }/*from  ww w . j  a v  a  2 s.  c  om*/

    final int numStoredFields, offset, length, totalLength;
    if (chunkDocs == 1) {
        numStoredFields = fieldsStream.readVInt();
        offset = 0;
        length = fieldsStream.readVInt();
        totalLength = length;
    } else {
        final int bitsPerStoredFields = fieldsStream.readVInt();
        if (bitsPerStoredFields == 0) {
            numStoredFields = fieldsStream.readVInt();
        } else if (bitsPerStoredFields > 31) {
            throw new CorruptIndexException(
                    "bitsPerStoredFields=" + bitsPerStoredFields + " (resource=" + fieldsStream + ")");
        } else {
            final long filePointer = fieldsStream.getFilePointer();
            final PackedInts.Reader reader = PackedInts.getDirectReaderNoHeader(fieldsStream,
                    PackedInts.Format.PACKED, packedIntsVersion, chunkDocs, bitsPerStoredFields);
            numStoredFields = (int) (reader.get(docID - docBase));
            fieldsStream.seek(filePointer
                    + PackedInts.Format.PACKED.byteCount(packedIntsVersion, chunkDocs, bitsPerStoredFields));
        }

        final int bitsPerLength = fieldsStream.readVInt();
        if (bitsPerLength == 0) {
            length = fieldsStream.readVInt();
            offset = (docID - docBase) * length;
            totalLength = chunkDocs * length;
        } else if (bitsPerStoredFields > 31) {
            throw new CorruptIndexException(
                    "bitsPerLength=" + bitsPerLength + " (resource=" + fieldsStream + ")");
        } else {
            final PackedInts.ReaderIterator it = PackedInts.getReaderIteratorNoHeader(fieldsStream,
                    PackedInts.Format.PACKED, packedIntsVersion, chunkDocs, bitsPerLength, 1);
            int off = 0;
            for (int i = 0; i < docID - docBase; ++i) {
                off += it.next();
            }
            offset = off;
            length = (int) it.next();
            off += length;
            for (int i = docID - docBase + 1; i < chunkDocs; ++i) {
                off += it.next();
            }
            totalLength = off;
        }
    }

    if ((length == 0) != (numStoredFields == 0)) {
        throw new CorruptIndexException("length=" + length + ", numStoredFields=" + numStoredFields
                + " (resource=" + fieldsStream + ")");
    }
    if (numStoredFields == 0) {
        // nothing to do
        return;
    }

    final DataInput documentInput;
    if (version >= VERSION_BIG_CHUNKS && totalLength >= 2 * chunkSize) {
        assert chunkSize > 0;
        assert offset < chunkSize;

        decompressor.decompress(fieldsStream, chunkSize, offset, Math.min(length, chunkSize - offset), bytes);
        documentInput = new DataInput() {

            int decompressed = bytes.length;

            void fillBuffer() throws IOException {
                assert decompressed <= length;
                if (decompressed == length) {
                    throw new EOFException();
                }
                final int toDecompress = Math.min(length - decompressed, chunkSize);
                decompressor.decompress(fieldsStream, toDecompress, 0, toDecompress, bytes);
                decompressed += toDecompress;
            }

            @Override
            public byte readByte() throws IOException {
                if (bytes.length == 0) {
                    fillBuffer();
                }
                --bytes.length;
                return bytes.bytes[bytes.offset++];
            }

            @Override
            public void readBytes(byte[] b, int offset, int len) throws IOException {
                while (len > bytes.length) {
                    System.arraycopy(bytes.bytes, bytes.offset, b, offset, bytes.length);
                    len -= bytes.length;
                    offset += bytes.length;
                    fillBuffer();
                }
                System.arraycopy(bytes.bytes, bytes.offset, b, offset, len);
                bytes.offset += len;
                bytes.length -= len;
            }

        };
    } else {
        final BytesRef bytes = totalLength <= BUFFER_REUSE_THRESHOLD ? this.bytes : new BytesRef();
        decompressor.decompress(fieldsStream, totalLength, offset, length, bytes);
        assert bytes.length == length;
        documentInput = new ByteArrayDataInput(bytes.bytes, bytes.offset, bytes.length);
    }

    for (int fieldIDX = 0; fieldIDX < numStoredFields; fieldIDX++) {
        final long infoAndBits = documentInput.readVLong();
        final int fieldNumber = (int) (infoAndBits >>> TYPE_BITS);
        final FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldNumber);

        final int bits = (int) (infoAndBits & TYPE_MASK);
        assert bits <= NUMERIC_DOUBLE : "bits=" + Integer.toHexString(bits);

        //get restricted
        FieldVisibility cv = RestrictedStoredFieldVisitor.EMPTY;
        boolean isRestricted = documentInput.readByte() == 1;
        if (isRestricted) {
            int cv_length = documentInput.readVInt();
            byte[] cv_bytes = new byte[cv_length];
            documentInput.readBytes(cv_bytes, 0, cv_length);
            cv = new FieldVisibility(cv_bytes);
        }

        RestrictedStoredFieldVisitor restrictedStoredFieldVisitor = DelegatingRestrictedFieldVisitor
                .wrap(visitor);
        if (evaluate(cv)) {
            switch (restrictedStoredFieldVisitor.needsField(fieldInfo, cv)) {
            case YES:
                readField(documentInput, restrictedStoredFieldVisitor, fieldInfo, bits, cv);
                break;
            case NO:
                skipField(documentInput, bits, cv);
                break;
            case STOP:
                return;
            }
        } else {
            skipField(documentInput, bits, cv);
        }
    }
}

From source file:com.lucure.core.codec.ForUtil.java

License:Apache License

/**
 * Restore a {@link ForUtil} from a {@link DataInput}.
 *//*from   w ww.ja v a2 s  . c  o  m*/
ForUtil(DataInput in) throws IOException {
    int packedIntsVersion = in.readVInt();
    PackedInts.checkVersion(packedIntsVersion);
    encodedSizes = new int[33];
    encoders = new PackedInts.Encoder[33];
    decoders = new PackedInts.Decoder[33];
    iterations = new int[33];

    for (int bpv = 1; bpv <= 32; ++bpv) {
        final int code = in.readVInt();
        final int formatId = code >>> 5;
        final int bitsPerValue = (code & 31) + 1;

        final PackedInts.Format format = PackedInts.Format.byId(formatId);
        assert format.isSupported(bitsPerValue);
        encodedSizes[bpv] = encodedSize(format, packedIntsVersion, bitsPerValue);
        encoders[bpv] = PackedInts.getEncoder(format, packedIntsVersion, bitsPerValue);
        decoders[bpv] = PackedInts.getDecoder(format, packedIntsVersion, bitsPerValue);
        iterations[bpv] = computeIterations(decoders[bpv]);
    }
}

From source file:com.sindicetech.siren.index.codecs.siren10.Siren10PostingsReader.java

License:Open Source License

@Override
public void decodeTerm(long[] empty, DataInput in, FieldInfo fieldInfo, BlockTermState _termState,
        boolean absolute) throws IOException {
    final Siren10TermState termState = (Siren10TermState) _termState;

    termState.blockCount = in.readVInt();

    termState.docIndex.read(in, absolute);

    if (termState.blockCount >= blockSkipMinimum) {
        if (absolute) {
            termState.skipFP = in.readVLong();
        } else {/*from   w w  w .ja va  2 s  . c o m*/
            termState.skipFP += in.readVLong();
        }
    } else if (absolute) {
        termState.skipFP = 0;
    }
}