List of usage examples for org.apache.hadoop.io BytesWritable getBytes
@Override public byte[] getBytes()
From source file:com.gotometrics.orderly.VariableLengthBytesWritableRowKey.java
License:Apache License
@Override public int getSerializedLength(Object o) throws IOException { if (o == null) return terminate() ? fixedPrefixLength + 1 : fixedPrefixLength; final BytesWritable input = (BytesWritable) o; return fixedPrefixLength + getSerializedLength( toStringRepresentation(input.getBytes(), fixedPrefixLength, input.getLength() - fixedPrefixLength)); }
From source file:com.gotometrics.orderly.VariableLengthBytesWritableRowKey.java
License:Apache License
@Override public void serialize(Object o, ImmutableBytesWritable bytesWritable) throws IOException { byte[] bytesToWriteIn = bytesWritable.get(); int offset = bytesWritable.getOffset(); if (o == null) { if (fixedPrefixLength > 0) throw new IllegalStateException("excepted at least " + fixedPrefixLength + " bytes to write"); else if (terminate()) { // write one (masked) null byte bytesToWriteIn[offset] = mask(NULL); RowKeyUtils.seek(bytesWritable, 1); }// w w w. java 2s . c o m } else { final BytesWritable input = (BytesWritable) o; if (fixedPrefixLength > input.getLength()) throw new IllegalStateException("excepted at least " + fixedPrefixLength + " bytes to write"); else { encodeFixedPrefix(input.getBytes(), bytesWritable); encodedCustomizedReversedPackedBcd(toStringRepresentation(input.getBytes(), fixedPrefixLength, input.getLength() - fixedPrefixLength), bytesWritable); } } }
From source file:com.inmobi.messaging.consumer.databus.mapred.TestDatabusBytesWritableInputFormat.java
License:Apache License
/** * read the the given input split.// w w w . jav a2s .com * @return List : List of read messages */ private List<Message> readSplit(DatabusBytesWritableInputFormat format, InputSplit split, JobConf job, Reporter reporter) throws IOException { List<Message> result = new ArrayList<Message>(); RecordReader<LongWritable, BytesWritable> reader = format.getRecordReader(split, job, reporter); LongWritable key = ((DatabusBytesWritableRecordReader) reader).createKey(); BytesWritable value = ((DatabusBytesWritableRecordReader) reader).createValue(); while (((DatabusBytesWritableRecordReader) reader).next(key, value)) { byte[] data = new byte[value.getLength()]; System.arraycopy(value.getBytes(), 0, data, 0, value.getLength()); result.add(new Message(ByteBuffer.wrap(data))); value = (BytesWritable) ((DatabusBytesWritableRecordReader) reader).createValue(); } reader.close(); return result; }
From source file:com.jfolson.hive.serde.RBaseSerDe.java
License:Apache License
@Override public Object deserialize(Writable blob) throws SerDeException { BytesWritable data = (BytesWritable) blob; inBarrStr.reset(data.getBytes(), 0, data.getLength()); try {//from w w w . ja v a 2s . co m //Read the key if (!keepAsBytes && unwrapKeys) { // It's wrapped as a list TWICE, because R will try to c() things once deserialized RType rtype = tbIn.getInput().readType(); if (rtype == null) { throw new RuntimeException("End of stream"); } if (rtype != RType.VECTOR) { throw new RuntimeException("Error: expecting vector<vector<key>>, instead of " + rtype.name()); } int vectorLength = tbIn.getInput().readVectorHeader(); if (vectorLength != 1) { throw new RuntimeException("Hive cannot support multiple keys"); } rtype = tbIn.getInput().readType(); if (rtype == null) { throw new RuntimeException("Unexpected end of stream"); } if (rtype != RType.VECTOR) { throw new RuntimeException("Error: expecting vector<key>, instead of " + rtype.name()); } vectorLength = tbIn.getInput().readVectorHeader(); if (vectorLength != numKeys) { throw new RuntimeException( "Error: expecting " + numKeys + " values, but list only has: " + vectorLength); } } for (int i = 0; i < numKeys; i++) { //LOG.info("Deserializing column: "+i); row.set(i, deserializeField(tbIn, columnTypes.get(i), row.get(i))); } //Read the value if (!keepAsBytes && unwrapValues) { // It's wrapped as a list TWICE, because R will try to c() things once deserialized RType rtype = tbIn.getInput().readType(); if (rtype == null) { throw new RuntimeException("End of stream"); } if (rtype != RType.VECTOR) { throw new RuntimeException( "Error: expecting vector<vector<value>>, instead of " + rtype.name()); } int vectorLength = tbIn.getInput().readVectorHeader(); if (vectorLength != 1) { throw new RuntimeException("Hive cannot support multiple values"); } rtype = tbIn.getInput().readType(); if (rtype == null) { throw new RuntimeException("Unexpected end of stream"); } if (rtype != RType.VECTOR) { throw new RuntimeException("Error: expecting vector<key>, instead of " + rtype.name()); } vectorLength = tbIn.getInput().readVectorHeader(); if (vectorLength != (numColumns - numKeys)) { throw new RuntimeException("Error: expecting " + (numColumns - numKeys) + " values, but list only has: " + vectorLength); } } for (int i = numKeys; i < numColumns; i++) { //LOG.info("Deserializing column: "+i); row.set(i, deserializeField(tbIn, columnTypes.get(i), row.get(i))); } // The next byte should be the marker // R doesn't want this //assert tbIn.readTypeCode() == Type.ENDOFRECORD; } catch (IOException e) { throw new SerDeException(e); } return row; }
From source file:com.jfolson.hive.serde.RBaseSerDe.java
License:Apache License
protected void serializeField(Object o, ObjectInspector oi, Object reuse) throws IOException { //LOG.info("Serializing hive type: "+oi.getTypeName()); //LOG.info("Serializing category: "+oi.getCategory().toString()); if (o == null) { tbOut.writeNull();/* w w w . j a va 2s . c om*/ return; } switch (oi.getCategory()) { case PRIMITIVE: { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; //LOG.info("Serializing primitive: "+poi.getPrimitiveCategory().toString()); switch (poi.getPrimitiveCategory()) { case VOID: { return; } case BINARY: { BinaryObjectInspector boi = (BinaryObjectInspector) poi; TypedBytesWritable bytes = reuse == null ? new TypedBytesWritable() : (TypedBytesWritable) reuse; BytesWritable bytesWrite = boi.getPrimitiveWritableObject(o); if (bytesWrite != null) { bytes.set(bytesWrite); if (!RType.isValid(bytes)) { LOG.error("Invalid typedbytes detected with type: " + RType.getType(bytes).code); bytes.setValue(new Buffer(bytesWrite.getBytes(), 0, bytesWrite.getLength())); } //LOG.info("Writing binary primitive with class: "+bytes.getClass().getName()); tbOut.write(bytes); } return; } case BOOLEAN: { BooleanObjectInspector boi = (BooleanObjectInspector) poi; BooleanWritable r = reuse == null ? new BooleanWritable() : (BooleanWritable) reuse; r.set(boi.get(o)); tbOut.write(r); return; } case BYTE: { ByteObjectInspector boi = (ByteObjectInspector) poi; ByteWritable r = reuse == null ? new ByteWritable() : (ByteWritable) reuse; r.set(boi.get(o)); tbOut.write(r); return; } case SHORT: { ShortObjectInspector spoi = (ShortObjectInspector) poi; ShortWritable r = reuse == null ? new ShortWritable() : (ShortWritable) reuse; r.set(spoi.get(o)); tbOut.write(r); return; } case INT: { IntObjectInspector ioi = (IntObjectInspector) poi; IntWritable r = reuse == null ? new IntWritable() : (IntWritable) reuse; r.set(ioi.get(o)); tbOut.write(r); return; } case LONG: { LongObjectInspector loi = (LongObjectInspector) poi; LongWritable r = reuse == null ? new LongWritable() : (LongWritable) reuse; r.set(loi.get(o)); tbOut.write(r); return; } case FLOAT: { FloatObjectInspector foi = (FloatObjectInspector) poi; FloatWritable r = reuse == null ? new FloatWritable() : (FloatWritable) reuse; r.set(foi.get(o)); tbOut.write(r); return; } case DOUBLE: DoubleObjectInspector doi = (DoubleObjectInspector) poi; DoubleWritable r = reuse == null ? new DoubleWritable() : (DoubleWritable) reuse; r.set(doi.get(o)); tbOut.write(r); return; case STRING: { StringObjectInspector soi = (StringObjectInspector) poi; Text t = soi.getPrimitiveWritableObject(o); tbOut.write(t); return; } default: { throw new RuntimeException("Unrecognized type: " + poi.getPrimitiveCategory()); } } } case LIST: { ListObjectInspector loi = (ListObjectInspector) oi; ObjectInspector elemOI = loi.getListElementObjectInspector(); List l = loi.getList(o); // Don't use array (typecode: 144) until everything supports NA values in typedbytes if (false) {//(elemOI.getCategory()==ObjectInspector.Category.PRIMITIVE){ tbOut.writeArray(l, (PrimitiveObjectInspector) elemOI); } else { tbOut.writeVector(l, (PrimitiveObjectInspector) elemOI); } return; } case MAP: case STRUCT: { // For complex object, serialize to JSON format String s = SerDeUtils.getJSONString(o, oi); Text t = reuse == null ? new Text() : (Text) reuse; // convert to Text and write it t.set(s); tbOut.write(t); return; } default: { throw new RuntimeException("Unrecognized type: " + oi.getCategory()); } } }
From source file:com.jfolson.hive.serde.RTypedBytesRecordWriter.java
License:Apache License
public void write(Writable row) throws IOException { BytesWritable brow = (BytesWritable) row; out.write(brow.getBytes(), 0, brow.getLength()); }
From source file:com.jfolson.hive.serde.RTypedBytesSerDe.java
License:Apache License
@Override public Object deserialize(Writable blob) throws SerDeException { BytesWritable data = (BytesWritable) blob; inBarrStr.reset(data.getBytes(), 0, data.getLength()); try {/*w ww.j a v a 2s .c o m*/ for (int i = 0; i < numColumns; i++) { LOG.info("Deserializing column: " + i); row.set(i, deserializeField(tbIn, columnTypes.get(i), row.get(i))); } // The next byte should be the marker // R doesn't want this //assert tbIn.readTypeCode() == Type.ENDOFRECORD; } catch (IOException e) { throw new SerDeException(e); } return row; }
From source file:com.jfolson.hive.serde.RTypedBytesSerDe.java
License:Apache License
private void serializeField(Object o, ObjectInspector oi, Object reuse) throws IOException { //LOG.info("Serializing hive type: "+oi.getTypeName()); //LOG.info("Serializing category: "+oi.getCategory().toString()); if (o == null) { tbOut.writeNull();// w w w. java 2s . com return; } switch (oi.getCategory()) { case PRIMITIVE: { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; //LOG.info("Serializing primitive: "+poi.getPrimitiveCategory().toString()); switch (poi.getPrimitiveCategory()) { case VOID: { return; } case BINARY: { BinaryObjectInspector boi = (BinaryObjectInspector) poi; TypedBytesWritable bytes = reuse == null ? new TypedBytesWritable() : (TypedBytesWritable) reuse; BytesWritable bytesWrite = boi.getPrimitiveWritableObject(o); if (bytesWrite != null) { bytes.set(bytesWrite); if (!RType.isValid(bytes)) { LOG.error("Invalid typedbytes detected with type: " + RType.getType(bytes).code); bytes.setValue(new Buffer(bytesWrite.getBytes(), 0, bytesWrite.getLength())); } //LOG.info("Writing binary primitive with class: "+bytes.getClass().getName()); tbOut.write(bytes); } return; } case BOOLEAN: { BooleanObjectInspector boi = (BooleanObjectInspector) poi; BooleanWritable r = reuse == null ? new BooleanWritable() : (BooleanWritable) reuse; r.set(boi.get(o)); tbOut.write(r); return; } case BYTE: { ByteObjectInspector boi = (ByteObjectInspector) poi; ByteWritable r = reuse == null ? new ByteWritable() : (ByteWritable) reuse; r.set(boi.get(o)); tbOut.write(r); return; } case SHORT: { ShortObjectInspector spoi = (ShortObjectInspector) poi; ShortWritable r = reuse == null ? new ShortWritable() : (ShortWritable) reuse; r.set(spoi.get(o)); tbOut.write(r); return; } case INT: { IntObjectInspector ioi = (IntObjectInspector) poi; IntWritable r = reuse == null ? new IntWritable() : (IntWritable) reuse; r.set(ioi.get(o)); tbOut.write(r); return; } case LONG: { LongObjectInspector loi = (LongObjectInspector) poi; LongWritable r = reuse == null ? new LongWritable() : (LongWritable) reuse; r.set(loi.get(o)); tbOut.write(r); return; } case FLOAT: { FloatObjectInspector foi = (FloatObjectInspector) poi; FloatWritable r = reuse == null ? new FloatWritable() : (FloatWritable) reuse; r.set(foi.get(o)); tbOut.write(r); return; } case DOUBLE: DoubleObjectInspector doi = (DoubleObjectInspector) poi; DoubleWritable r = reuse == null ? new DoubleWritable() : (DoubleWritable) reuse; r.set(doi.get(o)); tbOut.write(r); return; case STRING: { StringObjectInspector soi = (StringObjectInspector) poi; Text t = soi.getPrimitiveWritableObject(o); tbOut.write(t); return; } default: { throw new RuntimeException("Unrecognized type: " + poi.getPrimitiveCategory()); } } } case LIST: { ListObjectInspector loi = (ListObjectInspector) oi; ObjectInspector elemOI = loi.getListElementObjectInspector(); List l = loi.getList(o); if (false) {//(elemOI.getCategory()==ObjectInspector.Category.PRIMITIVE){ tbOut.writeArray(l, (PrimitiveObjectInspector) elemOI); } else { tbOut.writeVector(l, (PrimitiveObjectInspector) elemOI); } return; } case MAP: case STRUCT: { // For complex object, serialize to JSON format String s = SerDeUtils.getJSONString(o, oi); Text t = reuse == null ? new Text() : (Text) reuse; // convert to Text and write it t.set(s); tbOut.write(t); return; } default: { throw new RuntimeException("Unrecognized type: " + oi.getCategory()); } } }
From source file:com.jfolson.hive.serde.RTypedBytesWritableOutput.java
License:Apache License
public void writeRawBytes(BytesWritable bw) throws IOException { //LOG.info("Writing a plain BytesWritable"); byte[] bytes = Arrays.copyOfRange(bw.getBytes(), 0, bw.getLength()); out.writeRawBytes(bytes);//from ww w . j av a2 s . c o m }
From source file:com.jfolson.hive.serde.RTypedBytesWritableOutput.java
License:Apache License
public void writeBytes(BytesWritable bw) throws IOException { //LOG.info("Writing a plain BytesWritable"); byte[] bytes = Arrays.copyOfRange(bw.getBytes(), 0, bw.getLength()); out.writeBytes(bytes);/*from w ww. j a va 2 s.c o m*/ }