Example usage for org.apache.hadoop.io WritableUtils readVInt

Introduction

In this page you can find the example usage for org.apache.hadoop.io WritableUtils readVInt.

Prototype

public static int readVInt(DataInput stream) throws IOException

Source Link

Document

Reads a zero-compressed encoded integer from input stream and returns it.

Usage

From source file:ivory.core.data.index.PostingsListDocSortedNonPositional.java

License:Apache License

@Override
public void readFields(DataInput in) throws IOException {
    postingsAdded = WritableUtils.readVInt(in);
    numPostings = postingsAdded;/*www.  j  a  va  2 s  . co  m*/

    df = WritableUtils.readVInt(in);
    cf = WritableUtils.readVLong(in);
    sumOfPostingsScore = cf;

    rawBytes = new byte[WritableUtils.readVInt(in)];
    in.readFully(rawBytes);
}

From source file:ivory.core.data.index.PostingsListDocSortedPositionalPForDelta.java

License:Apache License

@Override
public void readFields(DataInput in) throws IOException {
    postingsAdded = WritableUtils.readVInt(in);
    numPostings = postingsAdded;/*from ww w .j  av a 2s. c o m*/

    df = WritableUtils.readVInt(in);
    cf = WritableUtils.readVLong(in);
    sumOfPostingsScore = cf;

    lastBlockSize = in.readInt();
    docidCompressed = new int[in.readInt()][];
    tfCompressed = new int[docidCompressed.length][];
    offsetCompressed = new int[docidCompressed.length][];
    for (int i = 0; i < docidCompressed.length; i++) {
        docidCompressed[i] = new int[in.readInt()];
        for (int j = 0; j < docidCompressed[i].length; j++) {
            docidCompressed[i][j] = in.readInt();
        }
    }

    for (int i = 0; i < tfCompressed.length; i++) {
        tfCompressed[i] = new int[in.readInt()];
        for (int j = 0; j < tfCompressed[i].length; j++) {
            tfCompressed[i][j] = in.readInt();
        }
    }

    for (int i = 0; i < offsetCompressed.length; i++) {
        offsetCompressed[i] = new int[in.readInt()];
        for (int j = 0; j < offsetCompressed[i].length; j++) {
            offsetCompressed[i][j] = in.readInt();
        }
    }

    positionsLastBlockSize = in.readInt();
    positionsCompressed = new int[in.readInt()][];
    for (int i = 0; i < positionsCompressed.length; i++) {
        positionsCompressed[i] = new int[in.readInt()];
        for (int j = 0; j < positionsCompressed[i].length; j++) {
            positionsCompressed[i][j] = in.readInt();
        }
    }
}

From source file:ivory.core.data.stat.DfTableArray.java

License:Apache License

/**
 * Creates a {@code DfTableArray} object.
 * @param file collection frequency data file path
 * @param fs FileSystem to read from//  w w  w .j a  va 2  s  .co  m
 * @throws IOException
 */
public DfTableArray(Path file, FileSystem fs) throws IOException {
    Preconditions.checkNotNull(file);
    Preconditions.checkNotNull(fs);

    FSDataInputStream in = fs.open(file);

    this.numTerms = in.readInt();

    dfs = new int[numTerms];

    for (int i = 0; i < numTerms; i++) {
        int df = WritableUtils.readVInt(in);

        dfs[i] = df;
        if (df > maxDf) {
            maxDf = df;
            maxDfTerm = i + 1;
        }

        if (df == 1) {
            numSingletonTerms++;
        }
    }

    in.close();
}

From source file:ivory.core.data.stat.PrefixEncodedGlobalStats.java

License:Apache License

public void loadDFStats(Path dfStatsPath, FileSystem fs) throws IOException {
    dfStatsInput = fs.open(dfStatsPath);

    int l = dfStatsInput.readInt();
    if (l != prefixSet.size()) {
        throw new RuntimeException("df length mismatch: " + l + "\t" + prefixSet.size());
    }/*from   ww  w.j  a  v a  2  s.  c om*/
    df = new int[l];
    for (int i = 0; i < l; i++)
        //df[i] = dfStatsInput.readInt();
        df[i] = WritableUtils.readVInt(dfStatsInput);

    dfStatsInput.close();
}

From source file:ivory.core.data.stat.PrefixEncodedGlobalStatsWithIndex.java

License:Apache License

private void loadDfs(Path dfStatsPath) throws IOException {
    if (dfs != null)
        return;//  w ww .  j  a  v a  2  s .  com
    FSDataInputStream dfStatsInput = fileSys.open(dfStatsPath);
    int l = dfStatsInput.readInt();
    if (l != prefixSet.size()) {
        throw new RuntimeException("df length mismatch: " + l + "\t" + prefixSet.size());
    }
    dfs = new int[l];
    for (int i = 0; i < l; i++)
        dfs[i] = WritableUtils.readVInt(dfStatsInput);
    dfStatsInput.close();
}

From source file:ivory.data.DfTableArray.java

License:Apache License

/**
 * Creates a <code>DfTableArray</code> object.
 *
 * @param file collection frequency data file path
 * @param fs   FileSystem to read from//w  ww .  ja  v a2 s  . co m
 * @throws IOException
 */
public DfTableArray(Path file, FileSystem fs) throws IOException {
    FSDataInputStream in = fs.open(file);

    this.numTerms = in.readInt();

    dfs = new int[numTerms];

    for (int i = 0; i < numTerms; i++) {
        int df = WritableUtils.readVInt(in);

        dfs[i] = df;
        if (df > maxDf) {
            maxDf = df;
            maxDfTerm = i + 1;
        }

        if (df == 1) {
            numSingletonTerms++;
        }
    }

    in.close();
}

From source file:ivory.data.LazyIntDocVector.java

License:Apache License

public void readFields(DataInput in) throws IOException {
    numTerms = WritableUtils.readVInt(in);
    if (numTerms == 0) {
        bytes = null;/*from   w  ww.  ja  va 2 s .c o  m*/
        return;
    }
    bytes = new byte[WritableUtils.readVInt(in)];
    in.readFully(bytes);
}

From source file:ivory.data.LazyTermDocVector.java

License:Apache License

public void readFields(DataInput in) throws IOException {
    read = true;//from   w  w w  .  jav a2  s  . co  m
    nTerms = WritableUtils.readVInt(in);
    if (nTerms == 0) {
        mRawBytes = null;
        mTerms = null;
        return;
    }
    mTerms = new String[nTerms];
    for (int i = 0; i < nTerms; i++) {
        mTerms[i] = in.readUTF();
    }
    mRawBytes = new byte[WritableUtils.readVInt(in)];
    in.readFully(mRawBytes);
}

From source file:ivory.data.PostingsListDocSortedPositional.java

License:Apache License

public void readFields(DataInput in) throws IOException {
    mPostingsAdded = WritableUtils.readVInt(in);
    mNumPostings = mPostingsAdded;/*from  w w w.jav  a2  s. c  om*/

    mDf = WritableUtils.readVInt(in);
    mCf = WritableUtils.readVLong(in);
    mSumOfPostingsScore = mCf;

    mRawBytes = new byte[WritableUtils.readVInt(in)];
    in.readFully(mRawBytes);
}

From source file:ivory.data.PrefixEncodedGlobalStats.java

License:Apache License

public void loadDFStats(Path dfStatsPath, FileSystem fs) throws IOException {
    dfStatsInput = fs.open(dfStatsPath);

    int l = dfStatsInput.readInt();
    if (l != prefixSet.length()) {
        throw new RuntimeException("df length mismatch: " + l + "\t" + prefixSet.length());
    }/*  w  w  w.  java 2s  . c  o m*/
    df = new int[l];
    for (int i = 0; i < l; i++)
        //df[i] = dfStatsInput.readInt();
        df[i] = WritableUtils.readVInt(dfStatsInput);

    dfStatsInput.close();
}