Example usage for org.apache.hadoop.io WritableUtils writeVInt

Introduction

In this page you can find the example usage for org.apache.hadoop.io WritableUtils writeVInt.

Prototype

public static void writeVInt(DataOutput stream, int i) throws IOException

Source Link

Document

Serializes an integer to a binary stream with zero-compressed encoding.

Usage

From source file:org.apache.phoenix.util.ByteUtil.java

License:Apache License

/**
 * Serialize an array of byte arrays into a single byte array.  Used
 * to pass through a set of bytes arrays as an attribute of a Scan.
 * Use {@link #toByteArrays(byte[], int)} to convert the serialized
 * byte array back to the array of byte arrays.
 * @param byteArrays the array of byte arrays to serialize
 * @return the byte array/*from  w  w  w . j a v  a2s . c  om*/
 */
public static byte[] toBytes(byte[][] byteArrays) {
    int size = 0;
    for (byte[] b : byteArrays) {
        if (b == null) {
            size++;
        } else {
            size += b.length;
            size += WritableUtils.getVIntSize(b.length);
        }
    }
    TrustedByteArrayOutputStream bytesOut = new TrustedByteArrayOutputStream(size);
    DataOutputStream out = new DataOutputStream(bytesOut);
    try {
        for (byte[] b : byteArrays) {
            if (b == null) {
                WritableUtils.writeVInt(out, 0);
            } else {
                WritableUtils.writeVInt(out, b.length);
                out.write(b);
            }
        }
    } catch (IOException e) {
        throw new RuntimeException(e); // not possible
    } finally {
        try {
            out.close();
        } catch (IOException e) {
            throw new RuntimeException(e); // not possible
        }
    }
    return bytesOut.getBuffer();
}

From source file:org.apache.phoenix.util.ByteUtil.java

License:Apache License

/**
 * Allows additional stuff to be encoded in length
 * @param output/*ww w.  j  a  v  a  2 s.  c  om*/
 * @param intArray
 * @param encodedLength
 * @throws IOException
 */
public static void serializeVIntArray(DataOutput output, int[] intArray, int encodedLength) throws IOException {
    WritableUtils.writeVInt(output, encodedLength);
    for (int i = 0; i < intArray.length; i++) {
        WritableUtils.writeVInt(output, intArray[i]);
    }
}

From source file:org.apache.phoenix.util.PrefixByteEncoder.java

License:Apache License

/**
 * Prefix encodes the byte array from offset to length into output stream. 
 * Instead of writing the entire byte array, only the portion of the byte array
 * that differs from the beginning of the previous byte array written is written.
 *  //from ww w.  j a v a  2  s .  c om
 * @param out output stream to encode into
 * @param b byte array buffer
 * @param offset offset into byte array to start encoding
 * @param length length of byte array to encode
 * @throws IOException
 */
public void encode(DataOutput out, byte[] b, int offset, int length) throws IOException {
    int i = 0;
    int prevOffset = previous.getOffset();
    byte[] prevBytes = previous.get();
    int prevLength = previous.getLength();
    int minLength = prevLength < b.length ? prevLength : b.length;
    for (i = 0; (i < minLength) && (prevBytes[prevOffset + i] == b[offset + i]); i++)
        ;
    WritableUtils.writeVInt(out, i);
    Bytes.writeByteArray(out, b, offset + i, length - i);
    previous.set(b, offset, length);
    if (length > maxLength) {
        maxLength = length;
    }
}

From source file:org.apache.phoenix.util.PrefixByteEncoderDecoderTest.java

License:Apache License

@Test
public void testEncode() throws IOException {
    List<byte[]> listOfBytes = Arrays.asList(Bytes.toBytes("aaaaa"), Bytes.toBytes("aaaabb"));
    ImmutableBytesWritable ptr = new ImmutableBytesWritable();
    int maxLength = PrefixByteCodec.encodeBytes(listOfBytes, ptr);
    assertEquals(6, maxLength);/*from   ww w  .  ja v  a  2s  . com*/
    TrustedByteArrayOutputStream stream = new TrustedByteArrayOutputStream(
            PrefixByteCodec.calculateSize(listOfBytes));
    DataOutput output = new DataOutputStream(stream);
    WritableUtils.writeVInt(output, 0);
    WritableUtils.writeVInt(output, 5);
    output.write(Bytes.toBytes("aaaaa")); // No space savings on first key
    WritableUtils.writeVInt(output, 4);
    WritableUtils.writeVInt(output, 2);
    output.write(Bytes.toBytes("bb")); // Only writes part of second key that's different
    assertArrayEquals(stream.toByteArray(), ptr.copyBytes());
}

From source file:org.apache.phoenix.util.TupleUtil.java

License:Apache License

@SuppressWarnings("deprecation")
public static int write(Tuple result, DataOutput out) throws IOException {
    int size = 0;
    for (int i = 0; i < result.size(); i++) {
        KeyValue kv = org.apache.hadoop.hbase.KeyValueUtil.ensureKeyValue(result.getValue(i));
        size += kv.getLength();//  ww  w.ja v a  2  s .co  m
        size += Bytes.SIZEOF_INT; // kv.getLength
    }

    WritableUtils.writeVInt(out, size);
    for (int i = 0; i < result.size(); i++) {
        KeyValue kv = org.apache.hadoop.hbase.KeyValueUtil.ensureKeyValue(result.getValue(i));
        out.writeInt(kv.getLength());
        out.write(kv.getBuffer(), kv.getOffset(), kv.getLength());
    }
    return size;
}

From source file:org.apache.pig.backend.hadoop.executionengine.spark_streaming.Text.java

License:Apache License

/** serialize
 * write this object to out//  w w w . ja  v a 2 s .  co  m
 * length uses zero-compressed encoding
 * @see Writable#write(DataOutput)
 */
public void write(DataOutput out) throws IOException {
    WritableUtils.writeVInt(out, length);
    out.write(bytes, 0, length);
}

From source file:org.apache.pig.backend.hadoop.executionengine.spark_streaming.Text.java

License:Apache License

/** Write a UTF8 encoded string to out
 *//*from  www  . jav a  2  s.co  m*/
public static int writeString(DataOutput out, String s) throws IOException {
    ByteBuffer bytes = encode(s);
    int length = bytes.limit();
    WritableUtils.writeVInt(out, length);
    out.write(bytes.array(), 0, length);
    return length;
}

From source file:org.apache.sqoop.job.io.Data.java

License:Apache License

private void writeType(DataOutput out, int type) throws IOException {
    WritableUtils.writeVInt(out, type);
}

From source file:org.apache.tez.common.counters.AbstractCounterGroup.java

License:Apache License

/**
 * GenericGroup ::= displayName #counter counter*
 *//*  ww  w .  j  a v a 2s  . c om*/
@Override
public synchronized void write(DataOutput out) throws IOException {
    Text.writeString(out, displayName);
    WritableUtils.writeVInt(out, counters.size());
    for (TezCounter counter : counters.values()) {
        counter.write(out);
    }
}

From source file:org.apache.tez.common.counters.AbstractCounters.java

License:Apache License

/**
 * Write the set of groups./*from   w ww .ja v a  2s .co m*/
 * Counters ::= version #fgroups (groupId, group)* #groups (group)*
 */
@Override
public synchronized void write(DataOutput out) throws IOException {
    WritableUtils.writeVInt(out, groupFactory.version());
    WritableUtils.writeVInt(out, fgroups.size()); // framework groups first
    for (G group : fgroups.values()) {
        if (group.getUnderlyingGroup() instanceof FrameworkCounterGroup<?, ?>) {
            WritableUtils.writeVInt(out, GroupType.FRAMEWORK.ordinal());
            WritableUtils.writeVInt(out, getFrameworkGroupId(group.getName()));
            group.write(out);
        } else if (group.getUnderlyingGroup() instanceof FileSystemCounterGroup<?>) {
            WritableUtils.writeVInt(out, GroupType.FILESYSTEM.ordinal());
            group.write(out);
        }
    }
    if (writeAllCounters) {
        WritableUtils.writeVInt(out, groups.size());
        for (G group : groups.values()) {
            Text.writeString(out, group.getName());
            group.write(out);
        }
    } else {
        WritableUtils.writeVInt(out, 0);
    }
}