List of usage examples for org.apache.hadoop.io WritableUtils decodeVIntSize
public static int decodeVIntSize(byte value)
From source file:com.asakusafw.runtime.value.StringOption.java
License:Apache License
/** * Compares between the two objects in serialized form. * @param b1 the first byte array to be compared * @param s1 the beginning index in {@code b1} * @param l1 the limit byte size in {@code b1} * @param b2 the second byte array to be compared * @param s2 the beginning index in {@code b2} * @param l2 the limit byte size in {@code b2} * @return the comparison result/* ww w.j a va 2 s . c o m*/ */ public static int compareBytes(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { if (b1[s1] == 0 || b2[s2] == 0) { return ByteArrayUtil.compare(b1[s1], b2[s2]); } int n1 = WritableUtils.decodeVIntSize(b1[s1 + 1]); int n2 = WritableUtils.decodeVIntSize(b2[s2 + 1]); int len1 = (int) ByteArrayUtil.readVLong(b1, s1 + 1); int len2 = (int) ByteArrayUtil.readVLong(b2, s2 + 1); return ByteArrayUtil.compare(b1, s1 + 1 + n1, len1, b2, s2 + 1 + n2, len2); }
From source file:com.chinamobile.bcbsp.util.Bytes.java
License:Apache License
/** * Converts a byte array to a long.// w ww. jav a2s. co m * * @param buffer * buffer to convert * @return vint bytes as an integer. */ public static long bytesToVint(final byte[] buffer) { int offset = 0; byte firstByte = buffer[offset++]; int len = WritableUtils.decodeVIntSize(firstByte); if (len == 1) { return firstByte; } long i = 0; for (int idx = 0; idx < len - 1; idx++) { byte b = buffer[offset++]; i = i << 8; i = i | (b & 0xFF); } return WritableUtils.isNegativeVInt(firstByte) ? ~i : i; }
From source file:com.chinamobile.bcbsp.util.Bytes.java
License:Apache License
/** * Reads a zero-compressed encoded long from input stream and returns it. * * @param buffer//from w w w .j a v a 2 s . c o m * Binary array * @param offset * Offset into array at which vint begins. * @throws java.io.IOException * e * @return deserialized long from stream. */ public static long readVLong(final byte[] buffer, final int offset) throws IOException { byte firstByte = buffer[offset]; int len = WritableUtils.decodeVIntSize(firstByte); if (len == 1) { return firstByte; } long i = 0; for (int idx = 0; idx < len - 1; idx++) { byte b = buffer[offset + 1 + idx]; i = i << 8; i = i | (b & 0xFF); } return WritableUtils.isNegativeVInt(firstByte) ? ~i : i; }
From source file:com.datasalt.pangool.tuplemr.mapred.SortComparator.java
License:Apache License
protected int compareMultipleSources(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) throws IOException { Schema commonSchema = serInfo.getCommonSchema(); Criteria commonOrder = tupleMRConf.getCommonCriteria(); int comparison = compare(b1, s1, b2, s2, commonSchema, commonOrder, offsets, nulls); if (comparison != 0) { return comparison; }// ww w .j a va 2 s .c o m int schemaId1 = readVInt(b1, offsets.offset1); int schemaId2 = readVInt(b2, offsets.offset2); if (schemaId1 != schemaId2) { int r = schemaId1 - schemaId2; return (tupleMRConf.getSchemasOrder() == Order.ASC) ? r : -r; } int vintSize = WritableUtils.decodeVIntSize(b1[offsets.offset1]); offsets.offset1 += vintSize; offsets.offset2 += vintSize; // sources are the same Criteria criteria = tupleMRConf.getSpecificOrderBys().get(schemaId1); if (criteria == null) { return 0; } Schema specificSchema = serInfo.getSpecificSchema(schemaId1); return compare(b1, offsets.offset1, b2, offsets.offset2, specificSchema, criteria, offsets, nulls); }
From source file:com.datasalt.pangool.tuplemr.mapred.SortComparator.java
License:Apache License
protected int compare(byte[] b1, int s1, byte[] b2, int s2, Schema schema, Criteria criteria, Offsets o, Nulls n) throws IOException { o.offset1 = s1;/* w w w . ja v a 2 s .co m*/ o.offset2 = s2; // Reading nulls bit field, if present if (schema.containsNullableFields()) { o.offset1 += n.nulls1.deser(b1, s1); o.offset2 += n.nulls2.deser(b2, s2); } for (int depth = 0; depth < criteria.getElements().size(); depth++) { Field field = schema.getField(depth); Field.Type type = field.getType(); SortElement sortElement = criteria.getElements().get(depth); Order sort = sortElement.getOrder(); RawComparator comparator = sortElement.getCustomComparator(); // Control for nulls, if field is nullable. if (field.isNullable()) { Criteria.NullOrder nullOrder = sortElement.getNullOrder(); if (n.nulls1.isSet(schema.getNullablePositionFromIndex(depth))) { if (n.nulls2.isSet(schema.getNullablePositionFromIndex(depth))) { // Both are null, so both are equal. No space is used. Continue. continue; } else { // First is null return (nullOrder == Criteria.NullOrder.NULL_SMALLEST && sort == Order.ASC) ? -1 : 1; } } else if (n.nulls2.isSet(schema.getNullablePositionFromIndex(depth))) { // Second is null return (nullOrder == Criteria.NullOrder.NULL_SMALLEST && sort == Order.ASC) ? 1 : -1; } } if (comparator != null) { //custom comparator for OBJECT int length1 = WritableComparator.readVInt(b1, o.offset1); int length2 = WritableComparator.readVInt(b2, o.offset2); o.offset1 += WritableUtils.decodeVIntSize(b1[o.offset1]); o.offset2 += WritableUtils.decodeVIntSize(b2[o.offset2]); int comparison = comparator.compare(b1, o.offset1, length1, b2, o.offset2, length2); o.offset1 += length1; o.offset2 += length2; if (comparison != 0) { return (sort == Order.ASC) ? comparison : -comparison; } } else { //not custom comparator switch (type) { case INT: case ENUM: { int value1 = readVInt(b1, o.offset1); int value2 = readVInt(b2, o.offset2); if (value1 > value2) { return (sort == Order.ASC) ? 1 : -1; } else if (value1 < value2) { return (sort == Order.ASC) ? -1 : 1; } int vintSize = WritableUtils.decodeVIntSize(b1[o.offset1]); o.offset1 += vintSize; o.offset2 += vintSize; } break; case LONG: { long value1 = readVLong(b1, o.offset1); long value2 = readVLong(b2, o.offset2); if (value1 > value2) { return (sort == Order.ASC) ? 1 : -1; } else if (value1 < value2) { return (sort == Order.ASC) ? -1 : 1; } int vIntSize = WritableUtils.decodeVIntSize(b1[o.offset1]); o.offset1 += vIntSize; o.offset2 += vIntSize; } break; case FLOAT: { float value1 = readFloat(b1, o.offset1); float value2 = readFloat(b2, o.offset2); int comp = Float.compare(value1, value2); if (comp != 0) { return (sort == Order.ASC) ? comp : -comp; } o.offset1 += Float.SIZE / 8; o.offset2 += Float.SIZE / 8; } break; case DOUBLE: { double value1 = readDouble(b1, o.offset1); double value2 = readDouble(b2, o.offset2); int comp = Double.compare(value1, value2); if (comp != 0) { return (sort == Order.ASC) ? comp : -comp; } o.offset1 += Double.SIZE / 8; o.offset2 += Double.SIZE / 8; } break; case BOOLEAN: { byte value1 = b1[o.offset1++]; byte value2 = b2[o.offset2++]; if (value1 > value2) { return (sort == Order.ASC) ? 1 : -1; } else if (value1 < value2) { return (sort == Order.ASC) ? -1 : 1; } } break; case STRING: case OBJECT: case BYTES: { int length1 = readVInt(b1, o.offset1); int length2 = readVInt(b2, o.offset2); o.offset1 += WritableUtils.decodeVIntSize(b1[o.offset1]); o.offset2 += WritableUtils.decodeVIntSize(b2[o.offset2]); int comparison = compareBytes(b1, o.offset1, length1, b2, o.offset2, length2); o.offset1 += length1; o.offset2 += length2; if (comparison != 0) { return (sort == Order.ASC) ? comparison : (-comparison); } } break; default: throw new IOException("Not supported comparison for type:" + type); } } } return 0; // equals }
From source file:com.datasalt.pangool.tuplemr.serialization.SingleFieldDeserializer.java
License:Apache License
/** * Deserialize an individual field from a byte array position that is encoded with the * {@link TupleSerialization}./*from w w w .j a va2 s . c om*/ * * @param bytes The byte array. * @param offset The place to start reading. * @return A deserialized instance. * */ public Object deserialize(byte[] bytes, int offset) throws IOException { //TODO repeated code from TupleDeserializer. REFACTOR!! switch (fieldType) { case INT: return readVInt(bytes, offset); case LONG: return readVLong(bytes, offset); case FLOAT: return readFloat(bytes, offset); case DOUBLE: return readDouble(bytes, offset); case BOOLEAN: return bytes[offset] != 0; case ENUM: int value1 = readVInt(bytes, offset); return objectClazz.getEnumConstants()[value1]; case STRING: int length = readVInt(bytes, offset); offset += WritableUtils.decodeVIntSize(bytes[offset]); ((Utf8) instance).set(bytes, offset, length); return instance; case OBJECT: length = readVInt(bytes, offset); //read prepended length offset += WritableUtils.decodeVIntSize(bytes[offset]); return ser.deser(instance, bytes, offset, length); default: throw new IOException("Not supported type:" + fieldType); } }
From source file:edu.uci.ics.hivesterix.serde.lazy.LazyUtils.java
License:Apache License
/** * Check a particular field and set its size and offset in bytes based on * the field type and the bytes arrays./*from ww w.j ava2 s . c o m*/ * For void, boolean, byte, short, int, long, float and double, there is no * offset and the size is fixed. For string, map, list, struct, the first * four bytes are used to store the size. So the offset is 4 and the size is * computed by concating the first four bytes together. The first four bytes * are defined with respect to the offset in the bytes arrays. * * @param objectInspector * object inspector of the field * @param bytes * bytes arrays store the table row * @param offset * offset of this field * @param recordInfo * modify this byteinfo object and return it */ public static void checkObjectByteInfo(ObjectInspector objectInspector, byte[] bytes, int offset, RecordInfo recordInfo) { Category category = objectInspector.getCategory(); switch (category) { case PRIMITIVE: PrimitiveCategory primitiveCategory = ((PrimitiveObjectInspector) objectInspector) .getPrimitiveCategory(); switch (primitiveCategory) { case VOID: recordInfo.elementOffset = 0; recordInfo.elementSize = 0; break; case BOOLEAN: case BYTE: recordInfo.elementOffset = 0; recordInfo.elementSize = 1; break; case SHORT: recordInfo.elementOffset = 0; recordInfo.elementSize = 2; break; case FLOAT: recordInfo.elementOffset = 0; recordInfo.elementSize = 4; break; case DOUBLE: recordInfo.elementOffset = 0; recordInfo.elementSize = 8; break; case INT: recordInfo.elementOffset = 0; recordInfo.elementSize = WritableUtils.decodeVIntSize(bytes[offset]); break; case LONG: recordInfo.elementOffset = 0; recordInfo.elementSize = WritableUtils.decodeVIntSize(bytes[offset]); break; case STRING: // using vint instead of 4 bytes LazyUtils.readVInt(bytes, offset, vInt); recordInfo.elementOffset = vInt.length; recordInfo.elementSize = vInt.value; break; default: { throw new RuntimeException("Unrecognized primitive type: " + primitiveCategory); } } break; case LIST: case MAP: case STRUCT: recordInfo.elementOffset = 4; recordInfo.elementSize = LazyUtils.byteArrayToInt(bytes, offset); break; default: { throw new RuntimeException("Unrecognized non-primitive type: " + category); } } }
From source file:edu.uci.ics.hivesterix.serde.lazy.LazyUtils.java
License:Apache License
/** * Reads a zero-compressed encoded long from a byte array and returns it. * // ww w. j a va 2 s. c o m * @param bytes * the byte array * @param offset * offset of the array to read from * @param vlong * storing the deserialized long and its size in byte */ public static void readVLong(byte[] bytes, int offset, VLong vlong) { byte firstByte = bytes[offset]; vlong.length = (byte) WritableUtils.decodeVIntSize(firstByte); if (vlong.length == 1) { vlong.value = firstByte; return; } long i = 0; for (int idx = 0; idx < vlong.length - 1; idx++) { byte b = bytes[offset + 1 + idx]; i = i << 8; i = i | (b & 0xFF); } vlong.value = (WritableUtils.isNegativeVInt(firstByte) ? (i ^ -1L) : i); }
From source file:edu.uci.ics.hivesterix.serde.lazy.LazyUtils.java
License:Apache License
/** * Reads a zero-compressed encoded int from a byte array and returns it. * /*from www.j a v a 2 s .c o m*/ * @param bytes * the byte array * @param offset * offset of the array to read from * @param vInt * storing the deserialized int and its size in byte */ public static void readVInt(byte[] bytes, int offset, VInt vInt) { byte firstByte = bytes[offset]; vInt.length = (byte) WritableUtils.decodeVIntSize(firstByte); if (vInt.length == 1) { vInt.value = firstByte; return; } int i = 0; for (int idx = 0; idx < vInt.length - 1; idx++) { byte b = bytes[offset + 1 + idx]; i = i << 8; i = i | (b & 0xFF); } vInt.value = (WritableUtils.isNegativeVInt(firstByte) ? (i ^ -1) : i); }
From source file:it.crs4.seal.common.GroupByLocationComparator.java
License:Open Source License
@Override public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { int sizeVint1 = WritableUtils.decodeVIntSize(b1[s1]); int sizeVint2 = WritableUtils.decodeVIntSize(b2[s2]); int retval = WritableComparator.compareBytes(b1, s1 + sizeVint1, l1 - sizeVint1 - Byte.SIZE / 8, // Byte.SIZE is the byte size in bits b2, s2 + sizeVint2, l2 - sizeVint2 - Byte.SIZE / 8); return retval; }