Example usage for org.apache.hadoop.io WritableUtils writeVInt

Introduction

In this page you can find the example usage for org.apache.hadoop.io WritableUtils writeVInt.

Prototype

public static void writeVInt(DataOutput stream, int i) throws IOException

Source Link

Document

Serializes an integer to a binary stream with zero-compressed encoding.

Usage

From source file:it.crs4.pydoop.pipes.BinaryProtocol.java

License:Apache License

public void runMap(InputSplit split, int numReduces, boolean pipedInput) throws IOException {
    WritableUtils.writeVInt(stream, MessageType.RUN_MAP.code);
    writeObject(split);/*  ww  w  . j  av  a2  s.c o  m*/
    WritableUtils.writeVInt(stream, numReduces);
    WritableUtils.writeVInt(stream, pipedInput ? 1 : 0);
}

From source file:it.crs4.pydoop.pipes.BinaryProtocol.java

License:Apache License

public void mapItem(WritableComparable key, Writable value) throws IOException {
    WritableUtils.writeVInt(stream, MessageType.MAP_ITEM.code);
    writeObject(key);//from  w ww.  j a v a  2 s. c  om
    writeObject(value);
}

From source file:it.crs4.pydoop.pipes.BinaryProtocol.java

License:Apache License

public void reduceKey(WritableComparable key) throws IOException {
    WritableUtils.writeVInt(stream, MessageType.REDUCE_KEY.code);
    writeObject(key);
}

From source file:it.crs4.pydoop.pipes.BinaryProtocol.java

License:Apache License

/**
 * Write the given object to the stream. If it is a Text or BytesWritable,
 * write it directly. Otherwise, write it to a buffer and then write the
 * length and data to the stream./*from w w  w. ja v a  2  s .c o  m*/
 * @param obj the object to write
 * @throws IOException
 */
private void writeObject(Writable obj) throws IOException {
    // For Text and BytesWritable, encode them directly, so that they end up
    // in C++ as the natural translations.
    if (obj instanceof Text) {
        Text t = (Text) obj;
        int len = t.getLength();
        WritableUtils.writeVInt(stream, len);
        stream.write(t.getBytes(), 0, len);
    } else if (obj instanceof BytesWritable) {
        BytesWritable b = (BytesWritable) obj;
        int len = b.getLength();
        WritableUtils.writeVInt(stream, len);
        stream.write(b.getBytes(), 0, len);
    } else {
        buffer.reset();
        obj.write(buffer);
        int length = buffer.getLength();
        WritableUtils.writeVInt(stream, length);
        stream.write(buffer.getData(), 0, length);
    }
}

From source file:ivory.core.data.document.LazyIntDocVector.java

License:Apache License

private void writeRawBytes(DataOutput out) {
    try {//w  w w  .ja v a 2 s . co m
        WritableUtils.writeVInt(out, bytes.length);
        out.write(bytes);
    } catch (IOException e) {
        throw new RuntimeException("Error writing LazyIntDocVector raw bytes");
    }
}

From source file:ivory.core.data.document.LazyIntDocVector.java

License:Apache License

private void writeTermPositionsMap(DataOutput out) {
    try {//from   ww  w.  j  av a2s . c om
        numTerms = termPositionsMap.size();

        // Write # of terms.
        WritableUtils.writeVInt(out, numTerms);
        if (numTerms == 0)
            return;

        bytesOut = new ByteArrayOutputStream();
        bitsOut = new BitOutputStream(bytesOut);

        Iterator<Map.Entry<Integer, int[]>> it = termPositionsMap.entrySet().iterator();
        Map.Entry<Integer, int[]> posting = it.next();
        int[] positions = posting.getValue();
        TermPositions tp = new TermPositions();
        // Write out the first termid.
        int lastTerm = posting.getKey().intValue();
        bitsOut.writeBinary(32, lastTerm);
        // Write out the tf value.
        bitsOut.writeGamma((short) positions.length);
        tp.set(positions, (short) positions.length);
        // Write out the positions.
        writePositions(bitsOut, tp);

        int curTerm;
        while (it.hasNext()) {
            posting = it.next();
            curTerm = posting.getKey().intValue();
            positions = posting.getValue();
            int tgap = curTerm - lastTerm;
            if (tgap <= 0) {
                throw new RuntimeException("Error: encountered invalid t-gap. termid=" + curTerm);
            }
            // Write out the gap.
            bitsOut.writeGamma(tgap);
            tp.set(positions, (short) positions.length);
            // Write out the tf value.
            bitsOut.writeGamma((short) positions.length);
            // Write out the positions.
            writePositions(bitsOut, tp);
            lastTerm = curTerm;
        }

        bitsOut.padAndFlush();
        bitsOut.close();
        byte[] bytes = bytesOut.toByteArray();
        WritableUtils.writeVInt(out, bytes.length);
        out.write(bytes);
    } catch (IOException e) {
        throw new RuntimeException("Error writing LazyIntDocVector term positions map", e);
    } catch (ArithmeticException e) {
        throw new RuntimeException(e);
    }
}

From source file:ivory.core.data.document.LazyTermDocVector.java

License:Apache License

@Override
public void write(DataOutput out) throws IOException {
    if (!read) {/*from  ww  w  . j  a va  2s .co  m*/
        numTerms = termPositionsMap.size();
        // write # of terms
        WritableUtils.writeVInt(out, numTerms);
        if (numTerms == 0)
            return;

        try {
            bytesOut = new ByteArrayOutputStream();
            bitsOut = new BitOutputStream(bytesOut);

            ArrayListOfInts positions;
            TermPositions tp = new TermPositions();
            String term;

            for (Map.Entry<String, ArrayListOfInts> posting : termPositionsMap.entrySet()) {
                term = posting.getKey();
                positions = posting.getValue();
                tp.set(positions.getArray(), (short) positions.size());

                // Write the term.
                out.writeUTF(term);
                // Write out the tf value.
                bitsOut.writeGamma((short) positions.size());
                // Write out the positions.
                LazyIntDocVector.writePositions(bitsOut, tp);
            }
            bitsOut.padAndFlush();
            bitsOut.close();
            byte[] bytes = bytesOut.toByteArray();
            WritableUtils.writeVInt(out, bytes.length);
            out.write(bytes);
        } catch (IOException e) {
            e.printStackTrace();
            throw new RuntimeException("Error adding postings.");
        } catch (ArithmeticException e) {
            e.printStackTrace();
            throw new RuntimeException(e.getMessage());
        }

    } else {
        WritableUtils.writeVInt(out, numTerms);
        if (numTerms == 0)
            return;

        for (int i = 0; i < numTerms; i++)
            out.writeUTF(terms[i]);

        WritableUtils.writeVInt(out, rawBytes.length);
        out.write(rawBytes);
    }
}

From source file:ivory.core.data.index.PostingsAccumulator.java

License:Apache License

public void write(DataOutput out) throws IOException {
    docnos.trimToSize();//from w  ww  .j a va 2  s  . co  m
    out.writeInt(docnos.size());
    for (int i : docnos.getArray())
        out.writeInt(i);
    for (int[] tp : positions) {
        out.writeShort((short) tp.length);
        for (int i : tp)
            WritableUtils.writeVInt(out, i);
    }

}

From source file:ivory.core.data.index.PostingsListDocSortedNonPositional.java

License:Apache License

@Override
public void write(DataOutput out) throws IOException {
    if (rawBytes != null) {
        // this would happen if we're reading in an already-encoded
        // postings; if that's the case, simply write out the byte array
        WritableUtils.writeVInt(out, postingsAdded);
        WritableUtils.writeVInt(out, df == 0 ? postingsAdded : df); // df
        WritableUtils.writeVLong(out, cf == 0 ? sumOfPostingsScore : cf); // cf
        WritableUtils.writeVInt(out, rawBytes.length);
        out.write(rawBytes);//from  w w w.j a  v  a2  s  .c  o  m
    } else {
        try {
            bitOut.padAndFlush();
            bitOut.close();

            if (numPostings != postingsAdded) {
                throw new RuntimeException(
                        "Error, number of postings added doesn't match number of expected postings. Expected "
                                + numPostings + ", got " + postingsAdded);
            }

            WritableUtils.writeVInt(out, postingsAdded);
            WritableUtils.writeVInt(out, df == 0 ? postingsAdded : df); // df
            WritableUtils.writeVLong(out, cf == 0 ? sumOfPostingsScore : cf); // cf
            byte[] bytes = bytesOut.toByteArray();
            WritableUtils.writeVInt(out, bytes.length);
            out.write(bytes);
        } catch (ArithmeticException e) {
            throw new RuntimeException("ArithmeticException caught \"" + e.getMessage()
                    + "\": check to see if collection size or df is set properly.");
        }
    }
}

From source file:ivory.core.data.index.PostingsListDocSortedPositional.java

License:Apache License

@Override
public void write(DataOutput out) throws IOException {
    if (rawBytes != null) {
        // This would happen if we're reading in an already-encoded postings; if that's the case,
        // simply write out the byte array.
        WritableUtils.writeVInt(out, postingsAdded);
        WritableUtils.writeVInt(out, df == 0 ? postingsAdded : df);
        WritableUtils.writeVLong(out, cf == 0 ? sumOfPostingsScore : cf);
        WritableUtils.writeVInt(out, rawBytes.length);
        out.write(rawBytes);/*w ww  .  j  a va  2  s .  co m*/
    } else {
        try {
            bitsOut.padAndFlush();
            bitsOut.close();

            if (numPostings != postingsAdded) {
                throw new RuntimeException(
                        "Error: number of postings added doesn't match number of expected postings. Expected "
                                + numPostings + ", got " + postingsAdded);
            }

            WritableUtils.writeVInt(out, postingsAdded);
            WritableUtils.writeVInt(out, df == 0 ? postingsAdded : df);
            WritableUtils.writeVLong(out, cf == 0 ? sumOfPostingsScore : cf);
            byte[] bytes = bytesOut.toByteArray();
            WritableUtils.writeVInt(out, bytes.length);
            out.write(bytes);
        } catch (ArithmeticException e) {
            throw new RuntimeException("ArithmeticException caught \"" + e.getMessage()
                    + "\": check to see if collection size or df is set properly.");
        }
    }
}