Example usage for org.apache.hadoop.io WritableUtils readVInt

Introduction

In this page you can find the example usage for org.apache.hadoop.io WritableUtils readVInt.

Prototype

public static int readVInt(DataInput stream) throws IOException

Source Link

Document

Reads a zero-compressed encoded integer from input stream and returns it.

Usage

From source file:com.datasalt.pangool.tuplemr.serialization.SimpleTupleDeserializer.java

License:Apache License

public void readFields(ITuple tuple, Deserializer[] customDeserializers) throws IOException {
    Schema schema = tuple.getSchema();
    // If there are fields with nulls, read the bit field and set the values that are null
    if (schema.containsNullableFields()) {
        List<Integer> nullableFields = schema.getNullableFieldsIdx();
        nullsAbsolute.ensureSize(schema.getFields().size());
        nullsAbsolute.clear(nullableFields);
        nullsRelative.deser(input);/*from w ww.j  a  v  a  2 s  . c om*/
        for (int i = 0; i < nullableFields.size(); i++) {
            if (nullsRelative.isSet(i)) {
                int field = nullableFields.get(i);
                tuple.set(field, null);
                nullsAbsolute.flags[field] = true;
            }
        }
    }

    // Field by field deseralization
    for (int index = 0; index < schema.getFields().size(); index++) {
        Deserializer customDeser = customDeserializers[index];
        Field field = schema.getField(index);

        // Nulls control
        if (field.isNullable() && nullsAbsolute.flags[index]) {
            // Null field. Nothing to deserialize.
            continue;
        }

        switch (field.getType()) {
        case INT:
            tuple.set(index, WritableUtils.readVInt(input));
            break;
        case LONG:
            tuple.set(index, WritableUtils.readVLong(input));
            break;
        case DOUBLE:
            tuple.set(index, input.readDouble());
            break;
        case FLOAT:
            tuple.set(index, input.readFloat());
            break;
        case STRING:
            readUtf8(input, tuple, index);
            break;
        case BOOLEAN:
            byte b = input.readByte();
            tuple.set(index, (b != 0));
            break;
        case ENUM:
            readEnum(input, tuple, field.getObjectClass(), index);
            break;
        case BYTES:
            readBytes(input, tuple, index);
            break;
        case OBJECT:
            readCustomObject(input, tuple, field.getObjectClass(), index, customDeser);
            break;
        default:
            throw new IOException("Not supported type:" + field.getType());
        }
    }
}

From source file:com.datasalt.pangool.tuplemr.serialization.SimpleTupleDeserializer.java

License:Apache License

protected void readCustomObject(DataInputStream input, ITuple tuple, Class<?> expectedType, int index,
        Deserializer customDeser) throws IOException {
    int size = WritableUtils.readVInt(input);
    if (size >= 0) {
        Object object = tuple.get(index);
        if (customDeser != null) {
            customDeser.open(input);//from   ww w  .j a v  a2  s  . c  o  m
            object = customDeser.deserialize(object);
            customDeser.close();
            tuple.set(index, object);
        } else {
            if (object == null) {
                tuple.set(index, ReflectionUtils.newInstance(expectedType, conf));
            }
            tmpInputBuffer.setSize(size);
            input.readFully(tmpInputBuffer.getBytes(), 0, size);
            Object ob = ser.deser(tuple.get(index), tmpInputBuffer.getBytes(), 0, size);
            tuple.set(index, ob);
        }
    } else {
        throw new IOException("Error deserializing, custom object serialized with negative length : " + size);
    }
}

From source file:com.datasalt.pangool.tuplemr.serialization.SimpleTupleDeserializer.java

License:Apache License

public void readBytes(DataInputStream input, ITuple tuple, int index) throws IOException {
    int length = WritableUtils.readVInt(input);
    ByteBuffer old = (ByteBuffer) tuple.get(index);
    ByteBuffer result;//  w  w w  .j av  a 2 s .c  o m
    if (old != null && length <= old.capacity()) {
        result = old;
        result.clear();
    } else {
        result = ByteBuffer.allocate(length);
        tuple.set(index, result);
    }
    input.readFully(result.array(), result.position(), length);
    result.limit(length);
}

From source file:com.datasalt.pangool.tuplemr.serialization.SimpleTupleDeserializer.java

License:Apache License

protected void readEnum(DataInputStream input, ITuple tuple, Class<?> fieldType, int index) throws IOException {
    int ordinal = WritableUtils.readVInt(input);
    try {//  w  ww. j av  a  2  s. c om
        Object[] enums = fieldType.getEnumConstants();
        tuple.set(index, enums[ordinal]);
    } catch (ArrayIndexOutOfBoundsException e) {
        throw new IOException("Ordinal index out of bounds for " + fieldType + " ordinal=" + ordinal);
    }
}

From source file:com.datasalt.pangool.tuplemr.serialization.TupleDeserializer.java

License:Apache License

private ITuple deserializeMultipleSources() throws IOException {
    CachedTuples tuples = cachedTuples.datum();
    ITuple commonTuple = tuples.commonTuple;

    simpleTupleDeSer.readFields(commonTuple, serInfo.getCommonSchemaDeserializers());
    int schemaId = WritableUtils.readVInt(simpleTupleDeSer.getInput());
    ITuple specificTuple = tuples.specificTuples.get(schemaId);
    simpleTupleDeSer.readFields(specificTuple, serInfo.getSpecificSchemaDeserializers().get(schemaId));
    ITuple result = tuples.resultTuples.get(schemaId);
    mixIntermediateIntoResult(commonTuple, specificTuple, result, schemaId);
    return result;
}

From source file:com.datasalt.utils.io.IdDatumBase.java

License:Apache License

@Override
public void readFields(DataInput in) throws IOException {
    identifier = WritableUtils.readVInt(in);
    item1.readFields(in);
}

From source file:com.datasalt.utils.io.IdDatumPairBase.java

License:Apache License

@Override
public void readFields(DataInput in) throws IOException {
    identifier = WritableUtils.readVInt(in);
    datumPair.readFields(in);
}

From source file:com.datasalt.utils.mapred.joiner.MultiJoinDatum.java

License:Apache License

@Override
public void readFields(DataInput in) throws IOException {
    channelId = WritableUtils.readVInt(in);
    datum.readFields(in);
}

From source file:com.datasalt.utils.mapred.joiner.MultiJoinPair.java

License:Apache License

@Override
public void readFields(DataInput in) throws IOException {
    group.readFields(in);/*from www  .j a va2s. c  o  m*/
    channelId = WritableUtils.readVInt(in);
    if (secondarySortClass != null) {
        secondarySort.readFields(in);
    }
}

From source file:com.emadbarsoum.lib.Tuple.java

License:Apache License

/**
 * {@inheritDoc}// www. j a  v a 2 s .com
 */
@SuppressWarnings("unchecked") // No static typeinfo on Tuples
public void readFields(DataInput in) throws IOException {
    int card = WritableUtils.readVInt(in);
    values = new Writable[card];
    written = WritableUtils.readVLong(in);
    Class<? extends Writable>[] cls = new Class[card];
    try {
        for (int i = 0; i < card; ++i) {
            cls[i] = Class.forName(Text.readString(in)).asSubclass(Writable.class);
        }
        for (int i = 0; i < card; ++i) {
            values[i] = cls[i].newInstance();
            if (has(i)) {
                values[i].readFields(in);
            }
        }
    } catch (ClassNotFoundException e) {
        throw (IOException) new IOException("Failed tuple init").initCause(e);
    } catch (IllegalAccessException e) {
        throw (IOException) new IOException("Failed tuple init").initCause(e);
    } catch (InstantiationException e) {
        throw (IOException) new IOException("Failed tuple init").initCause(e);
    }
}