Example usage for org.apache.hadoop.io WritableUtils writeVInt

Introduction

In this page you can find the example usage for org.apache.hadoop.io WritableUtils writeVInt.

Prototype

public static void writeVInt(DataOutput stream, int i) throws IOException 

Source Link

Document

Serializes an integer to a binary stream with zero-compressed encoding.

Usage

From source file:ivory.core.data.index.PostingsListDocSortedPositionalPForDelta.java

License:Apache License

@Override
public void write(DataOutput out) throws IOException {
    WritableUtils.writeVInt(out, postingsAdded);
    WritableUtils.writeVInt(out, df == 0 ? postingsAdded : df);
    WritableUtils.writeVLong(out, cf == 0 ? sumOfPostingsScore : cf);

    out.writeInt(lastBlockSize);/*  w ww .  j  ava 2 s.  co m*/
    out.writeInt(docidCompressed.length);
    for (int i = 0; i < docidCompressed.length; i++) {
        out.writeInt(docidCompressed[i].length);
        for (int j = 0; j < docidCompressed[i].length; j++) {
            out.writeInt(docidCompressed[i][j]);
        }
    }

    for (int i = 0; i < tfCompressed.length; i++) {
        out.writeInt(tfCompressed[i].length);
        for (int j = 0; j < tfCompressed[i].length; j++) {
            out.writeInt(tfCompressed[i][j]);
        }
    }

    for (int i = 0; i < offsetCompressed.length; i++) {
        out.writeInt(offsetCompressed[i].length);
        for (int j = 0; j < offsetCompressed[i].length; j++) {
            out.writeInt(offsetCompressed[i][j]);
        }
    }

    out.writeInt(positionsLastBlockSize);
    out.writeInt(positionsCompressed.length);
    for (int i = 0; i < positionsCompressed.length; i++) {
        out.writeInt(positionsCompressed[i].length);
        for (int j = 0; j < positionsCompressed[i].length; j++) {
            out.writeInt(positionsCompressed[i][j]);
        }
    }
}

---

From source file:ivory.data.LazyIntDocVector.java

License:Apache License

private void writeRawBytes(DataOutput out) {
    LOG.debug("in write), writing mRawBytes to out: " + out);
    try {/*  w  w w.ja  v a  2  s  .c  o m*/
        WritableUtils.writeVInt(out, bytes.length);
        out.write(bytes);
    } catch (IOException e) {
        throw new RuntimeException("Error writing LazyIntDocVector raw bytes");
    }
}

---

From source file:ivory.data.LazyIntDocVector.java

License:Apache License

private void writeTermPositionsMap(DataOutput out) {
    LOG.debug("in write(), writing termPositionsMap to out: " + out);
    try {/*from   w  ww.  j a  v a2s. c o m*/
        numTerms = termPositionsMap.size();

        // Write # of terms.
        WritableUtils.writeVInt(out, numTerms);
        if (numTerms == 0)
            return;

        bytesOut = new ByteArrayOutputStream();
        bitsOut = new BitOutputStream(bytesOut);

        Iterator<Map.Entry<Integer, int[]>> it = termPositionsMap.entrySet().iterator();
        Map.Entry<Integer, int[]> posting = it.next();
        int[] positions = posting.getValue();
        TermPositions tp = new TermPositions();
        // Write out the first termid.
        int lastTerm = posting.getKey().intValue();
        bitsOut.writeBinary(32, lastTerm);
        // Write out the tf value.
        bitsOut.writeGamma((short) positions.length);
        tp.set(positions, (short) positions.length);
        // Write out the positions.
        writePositions(bitsOut, tp);

        int curTerm;
        while (it.hasNext()) {
            posting = it.next();
            curTerm = posting.getKey().intValue();
            positions = posting.getValue();
            int tgap = curTerm - lastTerm;
            if (tgap <= 0) {
                throw new RuntimeException("Error: encountered invalid t-gap. termid=" + curTerm);
            }
            // Write out the gap.
            bitsOut.writeGamma(tgap);
            tp.set(positions, (short) positions.length);
            // Write out the tf value.
            bitsOut.writeGamma((short) positions.length);
            // Write out the positions.
            writePositions(bitsOut, tp);
            lastTerm = curTerm;
        }

        bitsOut.padAndFlush();
        bitsOut.close();
        byte[] bytes = bytesOut.toByteArray();
        WritableUtils.writeVInt(out, bytes.length);
        out.write(bytes);
    } catch (IOException e) {
        throw new RuntimeException("Error writing LazyIntDocVector term positions map", e);
    } catch (ArithmeticException e) {
        throw new RuntimeException(e);
    }
}

---

From source file:ivory.data.LazyTermDocVector.java

License:Apache License

public void write(DataOutput out) throws IOException {

    if (!read) {/* ww  w .ja va 2 s.com*/

        nTerms = termPositionsMap.size();
        // write # of terms
        WritableUtils.writeVInt(out, nTerms);
        if (nTerms == 0)
            return;

        try {
            mBytesOut = new ByteArrayOutputStream();
            mBitsOut = new BitOutputStream(mBytesOut);

            ArrayListOfInts positions;
            TermPositions tp = new TermPositions();
            String term;

            for (Map.Entry<String, ArrayListOfInts> posting : termPositionsMap.entrySet()) {
                term = posting.getKey();
                positions = posting.getValue();
                tp.set(positions.getArray(), (short) positions.size());

                // write the term
                out.writeUTF(term);
                // write out the tf value
                mBitsOut.writeGamma((short) positions.size());
                // write out the positions
                LazyIntDocVector.writePositions(mBitsOut, tp);
            }
            mBitsOut.padAndFlush();
            mBitsOut.close();
            byte[] bytes = mBytesOut.toByteArray();
            WritableUtils.writeVInt(out, bytes.length);
            out.write(bytes);
        } catch (IOException e) {
            e.printStackTrace();
            throw new RuntimeException("Error adding postings.");
        } catch (ArithmeticException e) {
            e.printStackTrace();
            throw new RuntimeException("ArithmeticException caught \"" + e.getMessage());
        }

    } else {
        WritableUtils.writeVInt(out, nTerms);
        if (nTerms == 0)
            return;

        for (int i = 0; i < nTerms; i++)
            out.writeUTF(mTerms[i]);

        WritableUtils.writeVInt(out, mRawBytes.length);
        out.write(mRawBytes);
    }
}

---

From source file:ivory.data.PostingsListDocSortedPositional.java

License:Apache License

public void write(DataOutput out) throws IOException {
    if (mRawBytes != null) {
        // this would happen if we're reading in an already-encoded
        // postings; if that's the case, simply write out the byte array
        WritableUtils.writeVInt(out, mPostingsAdded);
        WritableUtils.writeVInt(out, mDf == 0 ? mPostingsAdded : mDf); // df
        WritableUtils.writeVLong(out, mCf == 0 ? mSumOfPostingsScore : mCf); // cf
        WritableUtils.writeVInt(out, mRawBytes.length);
        out.write(mRawBytes);//from w  w w  . j  a  v a  2 s. c  om
    } else {
        try {
            mBitsOut.padAndFlush();
            mBitsOut.close();

            if (mNumPostings != mPostingsAdded) {
                throw new RuntimeException(
                        "Error, number of postings added doesn't match number of expected postings.  Expected "
                                + mNumPostings + ", got " + mPostingsAdded);
            }

            WritableUtils.writeVInt(out, mPostingsAdded);
            WritableUtils.writeVInt(out, mDf == 0 ? mPostingsAdded : mDf); // df
            WritableUtils.writeVLong(out, mCf == 0 ? mSumOfPostingsScore : mCf); // cf
            byte[] bytes = mBytesOut.toByteArray();
            WritableUtils.writeVInt(out, bytes.length);
            out.write(bytes);
        } catch (ArithmeticException e) {
            throw new RuntimeException("ArithmeticException caught \"" + e.getMessage()
                    + "\": check to see if collection size or df is set properly.");
        }

        sLogger.info("writing postings: cf=" + mSumOfPostingsScore + ", df=" + mNumPostings);
    }
}

---

From source file:ml.shifu.guagua.yarn.GuaguaSplitWriter.java

License:Apache License

private static void writeJobSplitMetaInfo(FileSystem fs, Path filename, FsPermission p,
        int splitMetaInfoVersion, JobSplit.SplitMetaInfo[] allSplitMetaInfo) throws IOException {
    // write the splits meta-info to a file for the job tracker
    FSDataOutputStream out = null;//from  w  w w. j  a  v  a2 s  .  co  m
    try {
        out = FileSystem.create(fs, filename, p);
        out.write(META_SPLIT_FILE_HEADER);
        WritableUtils.writeVInt(out, splitMetaInfoVersion);
        WritableUtils.writeVInt(out, allSplitMetaInfo.length);
        for (JobSplit.SplitMetaInfo splitMetaInfo : allSplitMetaInfo) {
            splitMetaInfo.write(out);
        }
    } finally {
        IOUtils.closeStream(out);
    }
}

---

From source file:org.apache.accumulo.core.client.IteratorSetting.java

License:Apache License

/**
 * @since 1.5.0//from w  w w  . jav  a 2  s .  com
 * @see Writable
 */
@Override
public void write(DataOutput dout) throws IOException {
    WritableUtils.writeVInt(dout, priority);
    WritableUtils.writeString(dout, name);
    WritableUtils.writeString(dout, iteratorClass);
    WritableUtils.writeVInt(dout, properties.size());
    for (Entry<String, String> e : properties.entrySet()) {
        WritableUtils.writeString(dout, e.getKey());
        WritableUtils.writeString(dout, e.getValue());
    }
}

---

From source file:org.apache.accumulo.core.data.Key.java

License:Apache License

@Override
public void write(DataOutput out) throws IOException {

    int colFamilyOffset = row.length;
    int colQualifierOffset = colFamilyOffset + colFamily.length;
    int colVisibilityOffset = colQualifierOffset + colQualifier.length;
    int totalLen = colVisibilityOffset + colVisibility.length;

    WritableUtils.writeVInt(out, colFamilyOffset);
    WritableUtils.writeVInt(out, colQualifierOffset);
    WritableUtils.writeVInt(out, colVisibilityOffset);

    WritableUtils.writeVInt(out, totalLen);

    out.write(row);/*from   w  w w.ja  va  2  s.  com*/
    out.write(colFamily);
    out.write(colQualifier);
    out.write(colVisibility);

    WritableUtils.writeVLong(out, timestamp);
    out.writeBoolean(deleted);
}

---

From source file:org.apache.accumulo.core.data.Mutation.java

License:Apache License

@Override
public void write(DataOutput out) throws IOException {
    serialize();//from  w ww. ja v a 2s .  c  o m
    byte hasValues = (values == null) ? 0 : (byte) 1;
    if (!replicationSources.isEmpty()) {
        // Use 2nd least-significant bit for whether or not we have replication sources
        hasValues = (byte) (0x02 | hasValues);
    }
    out.write((byte) (0x80 | hasValues));

    WritableUtils.writeVInt(out, row.length);
    out.write(row);

    WritableUtils.writeVInt(out, data.length);
    out.write(data);
    WritableUtils.writeVInt(out, entries);

    if (0x01 == (0x01 & hasValues)) {
        WritableUtils.writeVInt(out, values.size());
        for (int i = 0; i < values.size(); i++) {
            byte val[] = values.get(i);
            WritableUtils.writeVInt(out, val.length);
            out.write(val);
        }
    }
    if (0x02 == (0x02 & hasValues)) {
        WritableUtils.writeVInt(out, replicationSources.size());
        for (String source : replicationSources) {
            WritableUtils.writeString(out, source);
        }
    }
}

---

From source file:org.apache.accumulo.core.file.rfile.RelativeKey.java

License:Apache License

private static void write(DataOutput out, ByteSequence bs) throws IOException {
    WritableUtils.writeVInt(out, bs.length());
    out.write(bs.getBackingArray(), bs.offset(), bs.length());
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• »