List of usage examples for org.apache.hadoop.io WritableUtils writeVLong
public static void writeVLong(DataOutput stream, long i) throws IOException
From source file:com.cloudera.crunch.type.writable.TupleWritable.java
License:Apache License
/** * Writes each Writable to <code>out</code>. TupleWritable format: * {@code/*www . j a v a 2 s. co m*/ * <count><type1><type2>...<typen><obj1><obj2>...<objn> * } */ public void write(DataOutput out) throws IOException { WritableUtils.writeVInt(out, values.length); WritableUtils.writeVLong(out, written); for (int i = 0; i < values.length; ++i) { if (has(i)) { Text.writeString(out, values[i].getClass().getName()); } } for (int i = 0; i < values.length; ++i) { if (has(i)) { values[i].write(out); } } }
From source file:com.datasalt.pangool.tuplemr.serialization.SimpleTupleSerializer.java
License:Apache License
void write(Schema destinationSchema, ITuple tuple, int[] translationTable, Serializer[] customSerializers) throws IOException { // If can be null values, we compose a bit set with the null information and write it the first. if (destinationSchema.containsNullableFields()) { List<Integer> nullableFields = destinationSchema.getNullableFieldsIdx(); nulls.clear();//from w ww . jav a2s . c o m for (int i = 0; i < nullableFields.size(); i++) { int nField = nullableFields.get(i); if (valueAt(nField, tuple, translationTable) == null) { nulls.set(i); } } nulls.ser(out); } for (int i = 0; i < destinationSchema.getFields().size(); i++) { Field field = destinationSchema.getField(i); Type fieldType = field.getType(); Object element = valueAt(i, tuple, translationTable); if (element == null) { if (field.isNullable()) { // Nullable null fields don't need serialization. continue; } else { raiseUnexpectedNullException(field, element); } } try { switch (fieldType) { case INT: WritableUtils.writeVInt(out, (Integer) element); break; case LONG: WritableUtils.writeVLong(out, (Long) element); break; case DOUBLE: out.writeDouble((Double) element); break; case FLOAT: out.writeFloat((Float) element); break; case STRING: if (element instanceof Text) { ((Text) element).write(out); } else if (element instanceof String) { HELPER_TEXT.set((String) element); HELPER_TEXT.write(out); } else { raisedClassCastException(null, field, element); } break; case BOOLEAN: out.write((Boolean) element ? 1 : 0); break; case ENUM: writeEnum((Enum<?>) element, field, out); break; case OBJECT: writeCustomObject(element, out, customSerializers[i]); break; case BYTES: writeBytes(element, out); break; default: throw new IOException("Not supported type:" + fieldType); } } catch (ClassCastException e) { raisedClassCastException(e, field, element); } catch (CustomObjectSerializationException e) { raisedCustomObjectException(e, field, element, customSerializers[i]); } } // End for }
From source file:com.digitalpebble.behemoth.BehemothDocument.java
License:Apache License
protected void writeAnnotation(Annotation annot, DataOutput out, List<String> atypes) throws IOException { int typePos = atypes.indexOf(annot.getType()); IntWritable intStringPool = new IntWritable(typePos); intStringPool.write(out);//from w ww.j a va 2 s.c o m WritableUtils.writeVLong(out, annot.getStart()); WritableUtils.writeVLong(out, annot.getEnd()); out.writeInt(annot.getFeatureNum()); if (annot.getFeatures() != null) { Iterator<String> featNameIter = annot.getFeatures().keySet().iterator(); while (featNameIter.hasNext()) { String fname = featNameIter.next(); int fnamePos = atypes.indexOf(fname); intStringPool.set(fnamePos); intStringPool.write(out); WritableUtils.writeString(out, annot.getFeatures().get(fname)); } } }
From source file:com.emadbarsoum.lib.Tuple.java
License:Apache License
/** Writes each Writable to <code>out</code>. * Tuple format://from www . j av a 2 s . c o m * {@code * <count><type1><type2>...<typen><obj1><obj2>...<objn> * } */ public void write(DataOutput out) throws IOException { WritableUtils.writeVInt(out, values.length); WritableUtils.writeVLong(out, written); for (int i = 0; i < values.length; ++i) { Text.writeString(out, values[i].getClass().getName()); } for (int i = 0; i < values.length; ++i) { if (has(i)) { values[i].write(out); } } }
From source file:com.facebook.presto.rcfile.TestRcFileDecoderUtils.java
License:Apache License
private static Slice writeVintOld(SliceOutput output, long value) throws IOException { output.reset();//from w w w. j av a2 s . co m WritableUtils.writeVLong(output, value); Slice vLong = Slices.copyOf(output.slice()); if (value == (int) value) { output.reset(); WritableUtils.writeVInt(output, (int) value); Slice vInt = Slices.copyOf(output.slice()); assertEquals(vInt, vLong); } return vLong; }
From source file:com.moz.fiji.hive.io.EntityIdWritable.java
License:Apache License
@Override public void write(DataOutput out) throws IOException { WritableUtils.writeCompressedByteArray(out, mHBaseRowKey); // Write the components WritableUtils.writeVInt(out, mComponents.size()); for (Object component : mComponents) { if (component instanceof String) { WritableUtils.writeEnum(out, Component.STRING); String stringComponent = (String) component; WritableUtils.writeString(out, stringComponent); } else if (component instanceof Integer) { WritableUtils.writeEnum(out, Component.INTEGER); Integer intComponent = (Integer) component; WritableUtils.writeVInt(out, intComponent); } else if (component instanceof Long) { WritableUtils.writeEnum(out, Component.LONG); Long longComponent = (Long) component; WritableUtils.writeVLong(out, longComponent); } else if (component instanceof byte[]) { Preconditions.checkState(mComponents.size() == 1, "byte[] only valid as sole component."); WritableUtils.writeEnum(out, Component.RAW_HBASE_KEY); byte[] byteArrayComponent = (byte[]) component; WritableUtils.writeCompressedByteArray(out, byteArrayComponent); } else if (component == null) { WritableUtils.writeEnum(out, Component.NULL); } else {// w ww .ja va 2 s . c om throw new EntityIdException("Unexpected type for Component " + component.getClass().getName()); } } WritableUtils.writeString(out, mShellString); }
From source file:com.moz.fiji.hive.io.FijiCellWritable.java
License:Apache License
@Override public void write(DataOutput out) throws IOException { WritableUtils.writeVLong(out, mTimestamp); WritableUtils.writeString(out, mSchema.toString()); writeData(out, mData, mSchema);//from www. jav a 2s.c o m }
From source file:com.moz.fiji.hive.io.FijiCellWritable.java
License:Apache License
/** * Reads and converts data according to the specified schema. * * @param out DataOutput to serialize this object into. * @param data data to be serialized.//from ww w.j a va 2 s . co m * @param schema Schema to be used for serializing this data. * @throws IOException if there was an error writing. */ private static void writeData(DataOutput out, Object data, Schema schema) throws IOException { switch (schema.getType()) { case INT: Integer intData = (Integer) data; WritableUtils.writeVInt(out, intData); break; case LONG: Long longData = (Long) data; WritableUtils.writeVLong(out, longData); break; case DOUBLE: Double doubleData = (Double) data; DoubleWritable doubleWritable = new DoubleWritable(doubleData); doubleWritable.write(out); break; case ENUM: case STRING: String stringData = data.toString(); WritableUtils.writeString(out, stringData); break; case FLOAT: Float floatData = (Float) data; FloatWritable floatWritable = new FloatWritable(floatData); floatWritable.write(out); break; case ARRAY: List<Object> listData = (List<Object>) data; WritableUtils.writeVInt(out, listData.size()); for (Object listElement : listData) { writeData(out, listElement, schema.getElementType()); } break; case RECORD: IndexedRecord recordData = (IndexedRecord) data; WritableUtils.writeVInt(out, schema.getFields().size()); for (Schema.Field field : schema.getFields()) { WritableUtils.writeString(out, field.name()); writeData(out, recordData.get(field.pos()), field.schema()); } break; case MAP: Map<String, Object> mapData = (Map<String, Object>) data; WritableUtils.writeVInt(out, mapData.size()); for (Map.Entry<String, Object> entry : mapData.entrySet()) { WritableUtils.writeString(out, entry.getKey()); writeData(out, entry.getValue(), schema.getValueType()); } break; case UNION: final Integer tag = GenericData.get().resolveUnion(schema, data); WritableUtils.writeVInt(out, tag); Schema unionSubSchema = schema.getTypes().get(tag); writeData(out, data, unionSubSchema); break; case BYTES: byte[] bytesData = (byte[]) data; WritableUtils.writeCompressedByteArray(out, bytesData); break; case BOOLEAN: Boolean booleanData = (Boolean) data; BooleanWritable booleanWritable = new BooleanWritable(booleanData); booleanWritable.write(out); break; case NULL: // Don't need to write anything for null. break; case FIXED: default: throw new UnsupportedOperationException("Unsupported type: " + schema.getType()); } }
From source file:com.moz.fiji.hive.io.FijiRowDataWritable.java
License:Apache License
/** * Helper function to write a column and its associated data. * * @param out DataOutput for the Hadoop Writable to write to. * @param fijiColumnName to write//from ww w. j a v a 2s . co m * @param data to write * @throws IOException if there was an issue. */ private void writeColumn(DataOutput out, FijiColumnName fijiColumnName, NavigableMap<Long, FijiCellWritable> data) throws IOException { WritableUtils.writeString(out, fijiColumnName.getName()); WritableUtils.writeVInt(out, data.size()); // number in the timeseries for (Map.Entry<Long, FijiCellWritable> cellEntry : data.entrySet()) { WritableUtils.writeVLong(out, cellEntry.getKey()); cellEntry.getValue().write(out); } }
From source file:com.placeiq.piqconnect.BlockIndexWritable.java
License:Apache License
@Override public void write(DataOutput dataOutput) throws IOException { if (isVector) { WritableUtils.writeVLong(dataOutput, -(i + 1)); } else {/*from www . j ava 2 s .c o m*/ WritableUtils.writeVLong(dataOutput, i + 1); WritableUtils.writeVLong(dataOutput, j); } }