Example usage for org.apache.hadoop.io WritableUtils getVIntSize

List of usage examples for org.apache.hadoop.io WritableUtils getVIntSize

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.io.*
  5. WritableUtils
  6. getVIntSize

Introduction

In this page you can find the example usage for org.apache.hadoop.io WritableUtils getVIntSize.

Prototype

public static int getVIntSize(long i)

Source Link

Document

Get the encoded length if an integer is stored in a variable-length format

Usage

From source file:org.apache.phoenix.schema.ValueSchema.java

License:Apache License

public int getEstimatedByteSize() {
    int size = 0;
    size += WritableUtils.getVIntSize(minNullable);
    size += WritableUtils.getVIntSize(fields.size());
    size += fields.size() * 3;// ww w .j a v  a  2  s .  c  om
    return size;
}

From source file:org.apache.phoenix.util.ByteUtil.java

License:Apache License

/**
 * Serialize an array of byte arrays into a single byte array.  Used
 * to pass through a set of bytes arrays as an attribute of a Scan.
 * Use {@link #toByteArrays(byte[], int)} to convert the serialized
 * byte array back to the array of byte arrays.
 * @param byteArrays the array of byte arrays to serialize
 * @return the byte array//from   ww w . j  av a2s .  c  om
 */
public static byte[] toBytes(byte[][] byteArrays) {
    int size = 0;
    for (byte[] b : byteArrays) {
        if (b == null) {
            size++;
        } else {
            size += b.length;
            size += WritableUtils.getVIntSize(b.length);
        }
    }
    TrustedByteArrayOutputStream bytesOut = new TrustedByteArrayOutputStream(size);
    DataOutputStream out = new DataOutputStream(bytesOut);
    try {
        for (byte[] b : byteArrays) {
            if (b == null) {
                WritableUtils.writeVInt(out, 0);
            } else {
                WritableUtils.writeVInt(out, b.length);
                out.write(b);
            }
        }
    } catch (IOException e) {
        throw new RuntimeException(e); // not possible
    } finally {
        try {
            out.close();
        } catch (IOException e) {
            throw new RuntimeException(e); // not possible
        }
    }
    return bytesOut.getBuffer();
}

From source file:org.apache.phoenix.util.ByteUtil.java

License:Apache License

public static byte[] serializeVIntArray(int[] intArray, int encodedLength) {
    int size = WritableUtils.getVIntSize(encodedLength);
    for (int i = 0; i < intArray.length; i++) {
        size += WritableUtils.getVIntSize(intArray[i]);
    }//www. ja v a2s  .  com
    int offset = 0;
    byte[] out = new byte[size];
    offset += ByteUtil.vintToBytes(out, offset, size);
    for (int i = 0; i < intArray.length; i++) {
        offset += ByteUtil.vintToBytes(out, offset, intArray[i]);
    }
    return out;
}

From source file:org.apache.tephra.AbstractTransactionAwareTable.java

License:Apache License

/**
 * @param vint long to make a vint of./*from www  .j  ava2  s .  c o  m*/
 * @return long in vint byte array representation
 * We could alternatively make this abstract and
 * implement this method as Bytes.vintToBytes(long) in
 * every compat module. 
 */
protected byte[] getVIntBytes(final long vint) {
    long i = vint;
    int size = WritableUtils.getVIntSize(i);
    byte[] result = new byte[size];
    int offset = 0;
    if (i >= -112 && i <= 127) {
        result[offset] = (byte) i;
        return result;
    }

    int len = -112;
    if (i < 0) {
        i ^= -1L; // take one's complement'
        len = -120;
    }

    long tmp = i;
    while (tmp != 0) {
        tmp = tmp >> 8;
        len--;
    }

    result[offset++] = (byte) len;

    len = (len < -120) ? -(len + 120) : -(len + 112);

    for (int idx = len; idx != 0; idx--) {
        int shiftbits = (idx - 1) * 8;
        long mask = 0xFFL << shiftbits;
        result[offset++] = (byte) ((i & mask) >> shiftbits);
    }
    return result;
}

From source file:org.apache.tez.engine.common.sort.impl.dflt.InMemoryShuffleSorter.java

License:Apache License

@Override
protected void spill(int mstart, int mend) throws IOException, InterruptedException {
    // Start the shuffleHandler
    shuffleHandler.start();/*  w w  w.  ja va  2 s  . co  m*/

    // Don't spill!

    // Make a copy
    this.kvbuffer = super.kvbuffer;
    this.kvmeta = super.kvmeta;

    // Just save spill-indices for serving later
    int spindex = mstart;
    for (int i = 0; i < partitions; ++i) {
        spillIndices.add(spindex);

        int length = 0;
        while (spindex < mend && kvmeta.get(offsetFor(spindex) + PARTITION) == i) {

            final int kvoff = offsetFor(spindex);
            int keyLen = kvmeta.get(kvoff + VALSTART) - kvmeta.get(kvoff + KEYSTART);
            int valLen = getInMemVBytesLength(kvoff);
            length += (keyLen + WritableUtils.getVIntSize(keyLen))
                    + (valLen + WritableUtils.getVIntSize(valLen));

            ++spindex;
        }
        length += IFILE_EOF_LENGTH;

        shuffleHeaders.add(new ShuffleHeader(task.getTaskAttemptId().toString(), length + IFILE_CHECKSUM_LENGTH,
                length, i));
        LOG.info("shuffleHeader[" + i + "]:" + " rawLen=" + length + " partLen="
                + (length + IFILE_CHECKSUM_LENGTH) + " spillIndex=" + spillIndices.get(i));
    }

    LOG.info("Saved " + spillIndices.size() + " spill-indices and " + shuffleHeaders.size()
            + " shuffle headers");
}

From source file:org.apache.tez.engine.common.sort.impl.dflt.SortBufferInputStream.java

License:Apache License

@Override
public int read(byte[] b, int off, int len) throws IOException {
    if (available() == 0) {
        return -1;
    }//  w  w w  .  ja  va 2  s.  c o  m

    int currentOffset = off;
    int currentLength = len;
    int currentReadBytes = 0;

    // Check if there is residual data in the dualBuf
    int residualLen = out.getCurrent();
    if (residualLen > 0) {
        int readable = Math.min(currentLength, residualLen);
        System.arraycopy(dualBuf, 0, b, currentOffset, readable);
        currentOffset += readable;
        currentReadBytes += readable;
        out.setCurrentPointer(-readable);

        // buffer has less capacity
        currentLength -= readable;

        if (LOG.isDebugEnabled()) {
            LOG.debug("XXX read_residual:" + " readable=" + readable + " readBytes=" + readBytes);
        }
    }

    // Now, use the provided buffer
    if (LOG.isDebugEnabled()) {
        LOG.debug("XXX read: out.reset" + " b=" + b + " currentOffset=" + currentOffset + " currentLength="
                + currentLength + " recIndex=" + recIndex);
    }
    out.reset(b, currentOffset, currentLength);

    // Read from sort-buffer into the provided buffer, space permitting
    DataInputBuffer key = new DataInputBuffer();
    final InMemValBytes value = sorter.createInMemValBytes();

    int kvPartition = 0;
    int numRec = 0;
    for (; currentLength > 0 && recIndex < mend
            && (kvPartition = getKVPartition(recIndex)) == partition; ++recIndex) {

        final int kvoff = sorter.offsetFor(recIndex);

        int keyLen = (kvmeta.get(kvoff + InMemoryShuffleSorter.VALSTART)
                - kvmeta.get(kvoff + InMemoryShuffleSorter.KEYSTART));
        key.reset(kvbuffer, kvmeta.get(kvoff + InMemoryShuffleSorter.KEYSTART), keyLen);

        int valLen = sorter.getVBytesForOffset(kvoff, value);

        int recLen = (keyLen + WritableUtils.getVIntSize(keyLen))
                + (valLen + WritableUtils.getVIntSize(valLen));

        currentReadBytes += recLen;
        currentOffset += recLen;
        currentLength -= recLen;

        // Write out key/value into the in-mem ifile
        if (LOG.isDebugEnabled()) {
            LOG.debug("XXX read: sortOutput.append" + " #rec=" + ++numRec + " recIndex=" + recIndex + " kvoff="
                    + kvoff + " keyLen=" + keyLen + " valLen=" + valLen + " recLen=" + recLen + " readBytes="
                    + readBytes + " currentReadBytes=" + currentReadBytes + " currentLength=" + currentLength);
        }
        sortOutput.append(key, value);
    }

    // If we are at the end of the segment, close the ifile
    if (currentLength > 0 && (recIndex == mend || kvPartition != partition)) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("XXX About to call close:" + " currentLength=" + currentLength + " recIndex=" + recIndex
                    + " mend=" + mend + " kvPartition=" + kvPartition + " partitino=" + partition);
        }
        sortOutput.close();
        currentReadBytes += (InMemoryShuffleSorter.IFILE_EOF_LENGTH
                + InMemoryShuffleSorter.IFILE_CHECKSUM_LENGTH);
    } else {
        if (LOG.isDebugEnabled()) {
            LOG.debug("XXX Hmm..." + " currentLength=" + currentLength + " recIndex=" + recIndex + " mend="
                    + mend + " kvPartition=" + kvPartition + " partitino=" + partition);
        }
    }

    int retVal = Math.min(currentReadBytes, len);
    readBytes += retVal;
    if (LOG.isDebugEnabled()) {
        LOG.debug("XXX read: done" + " retVal=" + retVal + " currentReadBytes=" + currentReadBytes + " len="
                + len + " readBytes=" + readBytes + " partitionBytes=" + partitionBytes + " residualBytes="
                + out.getCurrent());
    }
    return retVal;
}

From source file:org.apache.tez.runtime.library.common.sort.impl.dflt.InMemoryShuffleSorter.java

License:Apache License

@Override
protected void spill(int mstart, int mend) throws IOException, InterruptedException {
    // Start the shuffleHandler
    shuffleHandler.start();// w w w .ja v  a  2  s.  c  om

    // Don't spill!

    // Make a copy
    this.kvbuffer = super.kvbuffer;
    this.kvmeta = super.kvmeta;

    // Just save spill-indices for serving later
    int spindex = mstart;
    for (int i = 0; i < partitions; ++i) {
        spillIndices.add(spindex);

        int length = 0;
        while (spindex < mend && kvmeta.get(offsetFor(spindex) + PARTITION) == i) {

            final int kvoff = offsetFor(spindex);
            int keyLen = kvmeta.get(kvoff + VALSTART) - kvmeta.get(kvoff + KEYSTART);
            int valLen = getInMemVBytesLength(kvoff);
            length += (keyLen + WritableUtils.getVIntSize(keyLen))
                    + (valLen + WritableUtils.getVIntSize(valLen));

            ++spindex;
        }
        length += IFILE_EOF_LENGTH;

        shuffleHeaders.add(new ShuffleHeader(outputContext.getUniqueIdentifier(), // TODO Verify that this is correct. 
                length + IFILE_CHECKSUM_LENGTH, length, i));
        LOG.info("shuffleHeader[" + i + "]:" + " rawLen=" + length + " partLen="
                + (length + IFILE_CHECKSUM_LENGTH) + " spillIndex=" + spillIndices.get(i));
    }

    LOG.info("Saved " + spillIndices.size() + " spill-indices and " + shuffleHeaders.size()
            + " shuffle headers");
}

From source file:org.terrier.compression.integer.ByteInputStream.java

License:Mozilla Public License

@Override
public final int readVInt() throws IOException {

    int i = WritableUtils.readVInt(di);
    byteOffset += WritableUtils.getVIntSize(i);

    return i;//from w w w.  j av  a 2  s .  c  om
}

From source file:org.terrier.compression.integer.ByteInputStream.java

License:Mozilla Public License

@Override
public final long readVLong() throws IOException {

    long i = WritableUtils.readVLong(di);
    byteOffset += WritableUtils.getVIntSize(i);
    return i;//from  w  w w .jav  a 2 s. c o m
}

From source file:org.terrier.compression.integer.ByteInputStream.java

License:Mozilla Public License

@Override
public int getVSize(long x) throws IOException {

    return WritableUtils.getVIntSize(x);
}