Example usage for org.apache.hadoop.io WritableUtils writeVInt

Introduction

In this page you can find the example usage for org.apache.hadoop.io WritableUtils writeVInt.

Prototype

public static void writeVInt(DataOutput stream, int i) throws IOException

Source Link

Document

Serializes an integer to a binary stream with zero-compressed encoding.

Usage

From source file:com.cloudera.science.matching.graph.VertexState.java

License:Open Source License

@Override
public void write(DataOutput out) throws IOException {
    out.writeBoolean(bidder);/*from   w w w .  j a  va2  s.c o m*/
    if (!bidder) {
        out.writeUTF(price.toString());
    }
    matchId.write(out);
    WritableUtils.writeVInt(out, priceIndex.size());
    for (Map.Entry<Text, BigDecimal> e : priceIndex.entrySet()) {
        e.getKey().write(out);
        out.writeUTF(e.getValue().toString());
    }
}

From source file:com.datasalt.pangool.io.BytesWritable.java

License:Apache License

public void write(DataOutput out) throws IOException {
    WritableUtils.writeVInt(out, size);
    out.write(bytes, 0, size);
}

From source file:com.datasalt.pangool.tuplemr.serialization.SimpleTupleSerializer.java

License:Apache License

void write(Schema destinationSchema, ITuple tuple, int[] translationTable, Serializer[] customSerializers)
        throws IOException {
    // If can be null values, we compose a bit set with the null information and write it the first.
    if (destinationSchema.containsNullableFields()) {
        List<Integer> nullableFields = destinationSchema.getNullableFieldsIdx();
        nulls.clear();//w  w w. j  a v  a 2s. c  o  m
        for (int i = 0; i < nullableFields.size(); i++) {
            int nField = nullableFields.get(i);
            if (valueAt(nField, tuple, translationTable) == null) {
                nulls.set(i);
            }
        }
        nulls.ser(out);
    }

    for (int i = 0; i < destinationSchema.getFields().size(); i++) {
        Field field = destinationSchema.getField(i);
        Type fieldType = field.getType();
        Object element = valueAt(i, tuple, translationTable);
        if (element == null) {
            if (field.isNullable()) {
                // Nullable null fields don't need serialization.
                continue;
            } else {
                raiseUnexpectedNullException(field, element);
            }
        }
        try {
            switch (fieldType) {
            case INT:
                WritableUtils.writeVInt(out, (Integer) element);
                break;
            case LONG:
                WritableUtils.writeVLong(out, (Long) element);
                break;
            case DOUBLE:
                out.writeDouble((Double) element);
                break;
            case FLOAT:
                out.writeFloat((Float) element);
                break;
            case STRING:
                if (element instanceof Text) {
                    ((Text) element).write(out);
                } else if (element instanceof String) {
                    HELPER_TEXT.set((String) element);
                    HELPER_TEXT.write(out);
                } else {
                    raisedClassCastException(null, field, element);
                }
                break;
            case BOOLEAN:
                out.write((Boolean) element ? 1 : 0);
                break;
            case ENUM:
                writeEnum((Enum<?>) element, field, out);
                break;
            case OBJECT:
                writeCustomObject(element, out, customSerializers[i]);
                break;
            case BYTES:
                writeBytes(element, out);
                break;
            default:
                throw new IOException("Not supported type:" + fieldType);
            }
        } catch (ClassCastException e) {
            raisedClassCastException(e, field, element);
        } catch (CustomObjectSerializationException e) {
            raisedCustomObjectException(e, field, element, customSerializers[i]);
        }
    } // End for
}

From source file:com.datasalt.pangool.tuplemr.serialization.SimpleTupleSerializer.java

License:Apache License

private void writeCustomObject(Object element, DataOutput output, Serializer customSer)
        throws CustomObjectSerializationException {
    try {//from w w w .  j  a v a  2 s. com
        tmpOutputBuffer.reset();
        if (customSer != null) {
            customSer.open(tmpOutputBuffer);
            customSer.serialize(element);
            customSer.close();
        } else {
            // If no custom serializer defined then use Hadoop Serialization by default
            ser.ser(element, tmpOutputBuffer);
        }
        WritableUtils.writeVInt(output, tmpOutputBuffer.getLength());
        output.write(tmpOutputBuffer.getData(), 0, tmpOutputBuffer.getLength());
    } catch (Throwable e) {
        throw new CustomObjectSerializationException(e);
    }
}

From source file:com.datasalt.pangool.tuplemr.serialization.SimpleTupleSerializer.java

License:Apache License

private void writeBytes(Object bytes, DataOutput output) throws IOException {
    if (bytes instanceof byte[]) {
        WritableUtils.writeVInt(output, ((byte[]) bytes).length);
        output.write((byte[]) bytes);
    } else if (bytes instanceof ByteBuffer) {
        ByteBuffer buffer = (ByteBuffer) bytes;
        int pos = buffer.position();
        int start = buffer.arrayOffset() + pos;
        int len = buffer.limit() - pos;
        WritableUtils.writeVInt(output, len);
        output.write(buffer.array(), start, len);
    } else {/*from ww w.ja  v a 2s  . c om*/
        throw new IOException("Not allowed " + bytes.getClass() + " for type " + Type.BYTES);
    }
}

From source file:com.datasalt.pangool.tuplemr.serialization.SimpleTupleSerializer.java

License:Apache License

private void writeEnum(Enum<?> element, Field field, DataOutput output) throws IOException {
    Enum<?> e = (Enum<?>) element;
    Class<?> expectedType = field.getObjectClass();
    if (e.getClass() != expectedType) {
        throw new IOException("Field '" + field.getName() + "' contains '" + element + "' which is "
                + element.getClass().getName() + ".The expected type is " + expectedType.getName());
    }/*from  w ww  .  j  a va  2  s.  c om*/
    WritableUtils.writeVInt(output, e.ordinal());
}

From source file:com.datasalt.pangool.tuplemr.serialization.TupleSerializer.java

License:Apache License

private void multipleSourcesSerialization(ITuple tuple) throws IOException {
    String schemaName = tuple.getSchema().getName();
    Integer schemaId = tupleMRConfig.getSchemaIdByName(schemaName);
    if (schemaId == null) {
        throw new IOException("Schema '" + tuple.getSchema() + "' is not a valid intermediate schema");
    }/*from  w ww  .j a  v  a 2s.  c om*/
    if (inputSchemaValidation) {
        Schema expectedSchema = tupleMRConfig.getIntermediateSchema(schemaId);
        if (!expectedSchema.equals(tuple.getSchema())) {
            throw new IOException("Tuple '" + tuple + "' " + "contains not expected schema."
                    + "Expected schema '" + expectedSchema + " and actual: " + tuple.getSchema());
        }
    }
    int[] commonTranslation = serInfo.getCommonSchemaIndexTranslation(schemaId);
    // Serialize common
    tupleSerializer.write(commonSchema, tuple, commonTranslation, serInfo.getCommonSchemaSerializers());
    // Serialize schema id
    WritableUtils.writeVInt(tupleSerializer.getOut(), schemaId);
    // Serialize rest of the fields
    Schema specificSchema = serInfo.getSpecificSchema(schemaId);
    int[] specificTranslation = serInfo.getSpecificSchemaIndexTranslation(schemaId);
    tupleSerializer.write(specificSchema, tuple, specificTranslation,
            serInfo.getSpecificSchemaSerializers().get(schemaId));
}

From source file:com.datasalt.utils.io.IdDatumBase.java

License:Apache License

@Override
public void write(DataOutput out) throws IOException {
    WritableUtils.writeVInt(out, identifier);
    item1.write(out);
}

From source file:com.datasalt.utils.io.IdDatumPairBase.java

License:Apache License

@Override
public void write(DataOutput out) throws IOException {
    WritableUtils.writeVInt(out, identifier);
    datumPair.write(out);
}

From source file:com.datasalt.utils.mapred.joiner.MultiJoinDatum.java

License:Apache License

@Override
public void write(DataOutput out) throws IOException {
    WritableUtils.writeVInt(out, channelId);
    datum.write(out);
}