Example usage for org.apache.hadoop.io WritableComparator readVInt

List of usage examples for org.apache.hadoop.io WritableComparator readVInt

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.io.*
  5. WritableComparator
  6. readVInt

Introduction

In this page you can find the example usage for org.apache.hadoop.io WritableComparator readVInt.

Prototype

public static int readVInt(byte[] bytes, int start) throws IOException

Source Link

Document

Reads a zero-compressed encoded integer from a byte array and returns it.

Usage

From source file:com.asakusafw.runtime.io.util.WritableRawComparableUnion.java

License:Apache License

@Override
public int getSizeInBytes(byte[] buf, int offset) throws IOException {
    int pos = WritableComparator.readVInt(buf, offset);
    WritableRawComparable object = objects[pos];
    int meta = WritableUtils.decodeVIntSize(buf[offset]);
    return meta + object.getSizeInBytes(buf, offset + meta);
}

From source file:com.asakusafw.runtime.io.util.WritableRawComparableUnion.java

License:Apache License

@Override
public int compareInBytes(byte[] b1, int o1, byte[] b2, int o2) throws IOException {
    int p1 = WritableComparator.readVInt(b1, o1);
    int p2 = WritableComparator.readVInt(b2, o2);
    if (p1 < p2) {
        return -1;
    } else if (p1 > p2) {
        return +1;
    }/*from   ww w . ja va  2  s  . c  o  m*/
    WritableRawComparable object = objects[p1];
    int meta = WritableUtils.decodeVIntSize(b1[o1]);
    return object.compareInBytes(b1, o1 + meta, b2, o2 + meta);
}

From source file:com.asakusafw.runtime.stage.directio.StringTemplate.java

License:Apache License

/**
 * Computes and returns size in bytes./*from w ww .  java 2 s  .  c  o  m*/
 * @param buf bytes array
 * @param offset bytes offset
 * @return size in bytes
 * @throws IOException if failed to compute size
 */
@Override
public final int getSizeInBytes(byte[] buf, int offset) throws IOException {
    int cursor = 0;
    for (int i = 0; i < formatters.length; i++) {
        int metaSize = WritableUtils.decodeVIntSize(buf[offset + cursor]);
        int bodySize = WritableComparator.readVInt(buf, offset + cursor);
        cursor += metaSize + bodySize;
    }
    return cursor;
}

From source file:com.asakusafw.runtime.value.DecimalOption.java

License:Apache License

@SuppressWarnings("deprecation")
@Override//  ww  w  .j  a  v  a 2  s .  c om
public int restore(byte[] bytes, int offset, int limit) throws IOException {
    int cursor = offset;
    int head = bytes[cursor++] & 0xff;
    if ((head & MASK_PRESENT) == 0) {
        setNull();
    } else {
        boolean plus = (head & MASK_PLUS) != 0;
        int scale = WritableComparator.readVInt(bytes, cursor);
        cursor += WritableUtils.decodeVIntSize(bytes[cursor]);
        int length = WritableComparator.readVInt(bytes, cursor);
        cursor += WritableUtils.decodeVIntSize(bytes[cursor]);
        DecimalBuffer buffer = BUFFER_MAIN.get();
        buffer.set(plus, scale, bytes, cursor, length);
        cursor += length;
        modify(buffer.toBigDecimal());
    }
    return cursor - offset;
}

From source file:com.asakusafw.runtime.value.DecimalOption.java

License:Apache License

/**
 * Returns the actual number of bytes from the serialized byte array.
 * @param bytes the target byte array//from  w  ww .  jav  a2s .  c  om
 * @param offset the beginning index in the byte array (inclusive)
 * @param length the limit length of the byte array
 * @return the comparison result
 */
public static int getBytesLength(byte[] bytes, int offset, int length) {
    try {
        int cursor = offset;
        int head = bytes[cursor++] & 0xff;
        if ((head & MASK_PRESENT) != 0) {
            cursor += WritableUtils.decodeVIntSize(bytes[cursor]);
            int bytesLength = WritableComparator.readVInt(bytes, cursor);
            cursor += WritableUtils.decodeVIntSize(bytes[cursor]);
            cursor += bytesLength;
        }
        return cursor - offset;
    } catch (IOException e) {
        throw new IllegalStateException(e);
    }
}

From source file:com.asakusafw.runtime.value.DecimalOption.java

License:Apache License

/**
 * Compares between the two objects in serialized form.
 * @param b1 the first byte array to be compared
 * @param s1 the beginning index in {@code b1}
 * @param l1 the limit byte size in {@code b1}
 * @param b2 the second byte array to be compared
 * @param s2 the beginning index in {@code b2}
 * @param l2 the limit byte size in {@code b2}
 * @return the comparison result//from  w w  w. ja  v a2s.  co  m
 */
public static int compareBytes(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
    try {
        // head
        int cursor1 = s1;
        int cursor2 = s2;
        int h1 = b1[cursor1++] & 0xff;
        int h2 = b2[cursor2++] & 0xff;

        // nullity
        boolean null1 = (h1 & MASK_PRESENT) == 0;
        boolean null2 = (h2 & MASK_PRESENT) == 0;
        if (null1) {
            if (null2) {
                return 0;
            } else {
                return -1;
            }
        } else if (null2) {
            return +1;
        }

        // sign
        boolean plus1 = (h1 & MASK_PLUS) != 0;
        boolean plus2 = (h2 & MASK_PLUS) != 0;
        if (plus1 && plus2 == false) {
            return +1;
        } else if (plus1 == false && plus2) {
            return -1;
        }

        // scale
        int scale1 = WritableComparator.readVInt(b1, cursor1);
        int scale2 = WritableComparator.readVInt(b2, cursor2);
        cursor1 += WritableUtils.decodeVIntSize(b1[cursor1]);
        cursor2 += WritableUtils.decodeVIntSize(b2[cursor2]);

        // bytesCount
        int bytesCount1 = WritableComparator.readVInt(b1, cursor1);
        int bytesCount2 = WritableComparator.readVInt(b2, cursor2);
        cursor1 += WritableUtils.decodeVIntSize(b1[cursor1]);
        cursor2 += WritableUtils.decodeVIntSize(b2[cursor2]);

        DecimalBuffer d1 = BUFFER_MAIN.get();
        d1.set(plus1, scale1, b1, cursor1, bytesCount1);

        DecimalBuffer d2 = BUFFER_SUB.get();
        d2.set(plus2, scale2, b2, cursor2, bytesCount2);

        return DecimalBuffer.compare(d1, d2);
    } catch (IOException e) {
        throw new IllegalStateException(e);
    }
}

From source file:com.datasalt.pangool.tuplemr.mapred.SortComparator.java

License:Apache License

protected int compare(byte[] b1, int s1, byte[] b2, int s2, Schema schema, Criteria criteria, Offsets o,
        Nulls n) throws IOException {
    o.offset1 = s1;// w ww .j a  v  a 2s  . c o m
    o.offset2 = s2;

    // Reading nulls bit field, if present
    if (schema.containsNullableFields()) {
        o.offset1 += n.nulls1.deser(b1, s1);
        o.offset2 += n.nulls2.deser(b2, s2);
    }

    for (int depth = 0; depth < criteria.getElements().size(); depth++) {
        Field field = schema.getField(depth);
        Field.Type type = field.getType();
        SortElement sortElement = criteria.getElements().get(depth);
        Order sort = sortElement.getOrder();
        RawComparator comparator = sortElement.getCustomComparator();

        // Control for nulls, if field is nullable.
        if (field.isNullable()) {
            Criteria.NullOrder nullOrder = sortElement.getNullOrder();
            if (n.nulls1.isSet(schema.getNullablePositionFromIndex(depth))) {
                if (n.nulls2.isSet(schema.getNullablePositionFromIndex(depth))) {
                    // Both are null, so both are equal. No space is used. Continue.
                    continue;
                } else {
                    // First is null
                    return (nullOrder == Criteria.NullOrder.NULL_SMALLEST && sort == Order.ASC) ? -1 : 1;
                }
            } else if (n.nulls2.isSet(schema.getNullablePositionFromIndex(depth))) {
                // Second is null
                return (nullOrder == Criteria.NullOrder.NULL_SMALLEST && sort == Order.ASC) ? 1 : -1;
            }
        }

        if (comparator != null) {
            //custom comparator for OBJECT
            int length1 = WritableComparator.readVInt(b1, o.offset1);
            int length2 = WritableComparator.readVInt(b2, o.offset2);
            o.offset1 += WritableUtils.decodeVIntSize(b1[o.offset1]);
            o.offset2 += WritableUtils.decodeVIntSize(b2[o.offset2]);
            int comparison = comparator.compare(b1, o.offset1, length1, b2, o.offset2, length2);
            o.offset1 += length1;
            o.offset2 += length2;
            if (comparison != 0) {
                return (sort == Order.ASC) ? comparison : -comparison;
            }
        } else {
            //not custom comparator
            switch (type) {
            case INT:
            case ENUM: {
                int value1 = readVInt(b1, o.offset1);
                int value2 = readVInt(b2, o.offset2);
                if (value1 > value2) {
                    return (sort == Order.ASC) ? 1 : -1;
                } else if (value1 < value2) {
                    return (sort == Order.ASC) ? -1 : 1;
                }
                int vintSize = WritableUtils.decodeVIntSize(b1[o.offset1]);
                o.offset1 += vintSize;
                o.offset2 += vintSize;
            }
                break;
            case LONG: {
                long value1 = readVLong(b1, o.offset1);
                long value2 = readVLong(b2, o.offset2);
                if (value1 > value2) {
                    return (sort == Order.ASC) ? 1 : -1;
                } else if (value1 < value2) {
                    return (sort == Order.ASC) ? -1 : 1;
                }
                int vIntSize = WritableUtils.decodeVIntSize(b1[o.offset1]);
                o.offset1 += vIntSize;
                o.offset2 += vIntSize;
            }
                break;
            case FLOAT: {
                float value1 = readFloat(b1, o.offset1);
                float value2 = readFloat(b2, o.offset2);
                int comp = Float.compare(value1, value2);
                if (comp != 0) {
                    return (sort == Order.ASC) ? comp : -comp;
                }
                o.offset1 += Float.SIZE / 8;
                o.offset2 += Float.SIZE / 8;
            }
                break;
            case DOUBLE: {
                double value1 = readDouble(b1, o.offset1);
                double value2 = readDouble(b2, o.offset2);
                int comp = Double.compare(value1, value2);
                if (comp != 0) {
                    return (sort == Order.ASC) ? comp : -comp;
                }
                o.offset1 += Double.SIZE / 8;
                o.offset2 += Double.SIZE / 8;
            }
                break;
            case BOOLEAN: {
                byte value1 = b1[o.offset1++];
                byte value2 = b2[o.offset2++];
                if (value1 > value2) {
                    return (sort == Order.ASC) ? 1 : -1;
                } else if (value1 < value2) {
                    return (sort == Order.ASC) ? -1 : 1;
                }
            }
                break;
            case STRING:
            case OBJECT:
            case BYTES: {
                int length1 = readVInt(b1, o.offset1);
                int length2 = readVInt(b2, o.offset2);
                o.offset1 += WritableUtils.decodeVIntSize(b1[o.offset1]);
                o.offset2 += WritableUtils.decodeVIntSize(b2[o.offset2]);
                int comparison = compareBytes(b1, o.offset1, length1, b2, o.offset2, length2);
                o.offset1 += length1;
                o.offset2 += length2;
                if (comparison != 0) {
                    return (sort == Order.ASC) ? comparison : (-comparison);
                }
            }
                break;
            default:
                throw new IOException("Not supported comparison for type:" + type);
            }
        }
    }
    return 0; // equals
}

From source file:org.apache.mrql.Bag.java

License:Apache License

/** compare this Bag with a given Bag by comparing their associated elements */
final public static int compare(byte[] x, int xs, int xl, byte[] y, int ys, int yl, int[] size) {
    try {/*from w w  w  .j  a va  2  s . c om*/
        int xn = WritableComparator.readVInt(x, xs);
        int xx = WritableUtils.decodeVIntSize(x[xs]);
        int yn = WritableComparator.readVInt(y, ys);
        int yy = WritableUtils.decodeVIntSize(y[ys]);
        for (int i = 0; i < xn && i < yn; i++) {
            int k = MRContainer.compare(x, xs + xx, xl - xx, y, ys + yy, yl - yy, size);
            if (k != 0)
                return k;
            xx += size[0];
            yy += size[0];
        }
        ;
        size[0] = xx + 1;
        if (xn > yn)
            return 1;
        if (xn < yn)
            return -1;
        return 0;
    } catch (IOException e) {
        throw new Error(e);
    }
}

From source file:org.apache.mrql.MR_int.java

License:Apache License

final public static int compare(byte[] x, int xs, int xl, byte[] y, int ys, int yl, int[] size) {
    try {//  www  . j a  v  a  2s  .c  om
        size[0] = 1 + WritableUtils.decodeVIntSize(x[xs]);
        int v = WritableComparator.readVInt(x, xs) - WritableComparator.readVInt(y, ys);
        return (v == 0) ? 0 : ((v > 0) ? 1 : -1);
    } catch (IOException e) {
        throw new Error(e);
    }
}

From source file:org.apache.mrql.MR_string.java

License:Apache License

final public static int compare(byte[] x, int xs, int xl, byte[] y, int ys, int yl, int[] size) {
    try {/*from w ww  .j a  v a2s.  c o m*/
        size[0] = 1 + WritableComparator.readVInt(x, xs) + WritableUtils.decodeVIntSize(x[xs]);
        return comparator.compare(x, xs, xl, y, ys, yl);
    } catch (IOException e) {
        throw new Error(e);
    }
}