List of usage examples for org.apache.hadoop.io BytesWritable getLength
@Override public int getLength()
From source file:com.ebay.nest.io.sede.lazy.LazySimpleSerDe.java
License:Apache License
/** * Deserialize a row from the Writable to a LazyObject. * * @param field//ww w . j ava 2s .com * the Writable that contains the data * @return The deserialized row Object. * @see SerDe#deserialize(Writable) */ @Override public Object deserialize(Writable field) throws SerDeException { if (byteArrayRef == null) { byteArrayRef = new ByteArrayRef(); } if (field instanceof BytesWritable) { BytesWritable b = (BytesWritable) field; // For backward-compatibility with hadoop 0.17 byteArrayRef.setData(b.getBytes()); cachedLazyStruct.init(byteArrayRef, 0, b.getLength()); } else if (field instanceof Text) { Text t = (Text) field; byteArrayRef.setData(t.getBytes()); cachedLazyStruct.init(byteArrayRef, 0, t.getLength()); } else { throw new SerDeException(getClass().toString() + ": expects either BytesWritable or Text object!"); } lastOperationSerialize = false; lastOperationDeserialize = true; return cachedLazyStruct; }
From source file:com.ebay.nest.io.sede.lazy.LazyUtils.java
License:Apache License
/** * Write out the text representation of a Primitive Object to a UTF8 byte * stream.// w ww .jav a 2 s.co m * * @param out * The UTF8 byte OutputStream * @param o * The primitive Object * @param needsEscape * Whether a character needs escaping. This array should have size of * 128. */ public static void writePrimitiveUTF8(OutputStream out, Object o, PrimitiveObjectInspector oi, boolean escaped, byte escapeChar, boolean[] needsEscape) throws IOException { switch (oi.getPrimitiveCategory()) { case BOOLEAN: { boolean b = ((BooleanObjectInspector) oi).get(o); if (b) { out.write(trueBytes, 0, trueBytes.length); } else { out.write(falseBytes, 0, falseBytes.length); } break; } case BYTE: { LazyInteger.writeUTF8(out, ((ByteObjectInspector) oi).get(o)); break; } case SHORT: { LazyInteger.writeUTF8(out, ((ShortObjectInspector) oi).get(o)); break; } case INT: { LazyInteger.writeUTF8(out, ((IntObjectInspector) oi).get(o)); break; } case LONG: { LazyLong.writeUTF8(out, ((LongObjectInspector) oi).get(o)); break; } case FLOAT: { float f = ((FloatObjectInspector) oi).get(o); ByteBuffer b = Text.encode(String.valueOf(f)); out.write(b.array(), 0, b.limit()); break; } case DOUBLE: { double d = ((DoubleObjectInspector) oi).get(o); ByteBuffer b = Text.encode(String.valueOf(d)); out.write(b.array(), 0, b.limit()); break; } case STRING: { Text t = ((StringObjectInspector) oi).getPrimitiveWritableObject(o); writeEscaped(out, t.getBytes(), 0, t.getLength(), escaped, escapeChar, needsEscape); break; } case VARCHAR: { HiveVarcharWritable hc = ((HiveVarcharObjectInspector) oi).getPrimitiveWritableObject(o); Text t = hc.getTextValue(); writeEscaped(out, t.getBytes(), 0, t.getLength(), escaped, escapeChar, needsEscape); break; } case BINARY: { BytesWritable bw = ((BinaryObjectInspector) oi).getPrimitiveWritableObject(o); byte[] toEncode = new byte[bw.getLength()]; System.arraycopy(bw.getBytes(), 0, toEncode, 0, bw.getLength()); byte[] toWrite = Base64.encodeBase64(toEncode); out.write(toWrite, 0, toWrite.length); break; } case DATE: { LazyDate.writeUTF8(out, ((DateObjectInspector) oi).getPrimitiveWritableObject(o)); break; } case TIMESTAMP: { LazyTimestamp.writeUTF8(out, ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o)); break; } case DECIMAL: { HiveDecimal bd = ((HiveDecimalObjectInspector) oi).getPrimitiveJavaObject(o); ByteBuffer b = Text.encode(bd.toString()); out.write(b.array(), 0, b.limit()); break; } default: { throw new RuntimeException("Hive internal error."); } } }
From source file:com.ebay.nest.io.sede.lazy.LazyUtils.java
License:Apache License
/** * gets a byte[] with copy of data from source BytesWritable * @param sourceBw - source BytesWritable *//*from w w w .j a v a 2 s . c o m*/ public static byte[] createByteArray(BytesWritable sourceBw) { //TODO should replace with BytesWritable.copyData() once Hive //removes support for the Hadoop 0.20 series. return Arrays.copyOf(sourceBw.getBytes(), sourceBw.getLength()); }
From source file:com.ebay.nest.io.sede.lazybinary.LazyBinaryBinary.java
License:Apache License
LazyBinaryBinary(LazyBinaryPrimitive<WritableBinaryObjectInspector, BytesWritable> copy) {
super(copy);/* ww w . jav a 2 s. c o m*/
BytesWritable incoming = copy.getWritableObject();
byte[] outgoing = new byte[incoming.getLength()];
System.arraycopy(incoming.getBytes(), 0, outgoing, 0, incoming.getLength());
data = new BytesWritable(outgoing);
}
From source file:com.ebay.nest.io.sede.MetadataTypedColumnsetSerDe.java
License:Apache License
@Override public Object deserialize(Writable field) throws SerDeException { String row = null;/* w w w .ja v a2 s . c o m*/ if (field instanceof BytesWritable) { BytesWritable b = (BytesWritable) field; try { row = Text.decode(b.getBytes(), 0, b.getLength()); } catch (CharacterCodingException e) { throw new SerDeException(e); } } else if (field instanceof Text) { row = field.toString(); } try { deserialize(deserializeCache, row, separator, nullString, splitLimit); if (columnNames != null) { assert (columnNames.size() == deserializeCache.col.size()); } return deserializeCache; } catch (ClassCastException e) { throw new SerDeException(this.getClass().getName() + " expects Text or BytesWritable", e); } catch (Exception e) { throw new SerDeException(e); } }
From source file:com.ebay.nest.io.sede.objectinspector.primitive.WritableBinaryObjectInspector.java
License:Apache License
@Override public BytesWritable copyObject(Object o) { if (null == o) { return null; }//from ww w. j a v a2 s .c o m BytesWritable incoming = (BytesWritable) o; byte[] bytes = new byte[incoming.getLength()]; System.arraycopy(incoming.getBytes(), 0, bytes, 0, incoming.getLength()); return new BytesWritable(bytes); }
From source file:com.ebay.nest.io.sede.SerDeUtils.java
License:Apache License
static void buildJSONString(StringBuilder sb, Object o, ObjectInspector oi, String nullStr) { switch (oi.getCategory()) { case PRIMITIVE: { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; if (o == null) { sb.append(nullStr);/*from w ww.j a v a 2 s . co m*/ } else { switch (poi.getPrimitiveCategory()) { case BOOLEAN: { boolean b = ((BooleanObjectInspector) poi).get(o); sb.append(b ? "true" : "false"); break; } case BYTE: { sb.append(((ByteObjectInspector) poi).get(o)); break; } case SHORT: { sb.append(((ShortObjectInspector) poi).get(o)); break; } case INT: { sb.append(((IntObjectInspector) poi).get(o)); break; } case LONG: { sb.append(((LongObjectInspector) poi).get(o)); break; } case FLOAT: { sb.append(((FloatObjectInspector) poi).get(o)); break; } case DOUBLE: { sb.append(((DoubleObjectInspector) poi).get(o)); break; } case STRING: { sb.append('"'); sb.append(escapeString(((StringObjectInspector) poi).getPrimitiveJavaObject(o))); sb.append('"'); break; } case VARCHAR: { sb.append('"'); sb.append( escapeString(((HiveVarcharObjectInspector) poi).getPrimitiveJavaObject(o).toString())); sb.append('"'); break; } case DATE: { sb.append('"'); sb.append(((DateObjectInspector) poi).getPrimitiveWritableObject(o)); sb.append('"'); break; } case TIMESTAMP: { sb.append('"'); sb.append(((TimestampObjectInspector) poi).getPrimitiveWritableObject(o)); sb.append('"'); break; } case BINARY: { BytesWritable bw = ((BinaryObjectInspector) oi).getPrimitiveWritableObject(o); Text txt = new Text(); txt.set(bw.getBytes(), 0, bw.getLength()); sb.append(txt.toString()); break; } case DECIMAL: { sb.append(((HiveDecimalObjectInspector) oi).getPrimitiveJavaObject(o)); break; } default: throw new RuntimeException("Unknown primitive type: " + poi.getPrimitiveCategory()); } } break; } case LIST: { ListObjectInspector loi = (ListObjectInspector) oi; ObjectInspector listElementObjectInspector = loi.getListElementObjectInspector(); List<?> olist = loi.getList(o); if (olist == null) { sb.append(nullStr); } else { sb.append(LBRACKET); for (int i = 0; i < olist.size(); i++) { if (i > 0) { sb.append(COMMA); } buildJSONString(sb, olist.get(i), listElementObjectInspector, JSON_NULL); } sb.append(RBRACKET); } break; } case MAP: { MapObjectInspector moi = (MapObjectInspector) oi; ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector(); ObjectInspector mapValueObjectInspector = moi.getMapValueObjectInspector(); Map<?, ?> omap = moi.getMap(o); if (omap == null) { sb.append(nullStr); } else { sb.append(LBRACE); boolean first = true; for (Object entry : omap.entrySet()) { if (first) { first = false; } else { sb.append(COMMA); } Map.Entry<?, ?> e = (Map.Entry<?, ?>) entry; buildJSONString(sb, e.getKey(), mapKeyObjectInspector, JSON_NULL); sb.append(COLON); buildJSONString(sb, e.getValue(), mapValueObjectInspector, JSON_NULL); } sb.append(RBRACE); } break; } case STRUCT: { StructObjectInspector soi = (StructObjectInspector) oi; List<? extends StructField> structFields = soi.getAllStructFieldRefs(); if (o == null) { sb.append(nullStr); } else { sb.append(LBRACE); for (int i = 0; i < structFields.size(); i++) { if (i > 0) { sb.append(COMMA); } sb.append(QUOTE); sb.append(structFields.get(i).getFieldName()); sb.append(QUOTE); sb.append(COLON); buildJSONString(sb, soi.getStructFieldData(o, structFields.get(i)), structFields.get(i).getFieldObjectInspector(), JSON_NULL); } sb.append(RBRACE); } break; } case UNION: { UnionObjectInspector uoi = (UnionObjectInspector) oi; if (o == null) { sb.append(nullStr); } else { sb.append(LBRACE); sb.append(uoi.getTag(o)); sb.append(COLON); buildJSONString(sb, uoi.getField(o), uoi.getObjectInspectors().get(uoi.getTag(o)), JSON_NULL); sb.append(RBRACE); } break; } default: throw new RuntimeException("Unknown type in ObjectInspector!"); } }
From source file:com.endgame.binarypig.util.StreamUtils.java
License:Apache License
public static void writeToFile(BytesWritable value, File binaryFile) throws IOException { FileOutputStream fileOut = new FileOutputStream(binaryFile); fileOut.write(value.getBytes(), 0, value.getLength()); fileOut.close();/* w w w. j a v a2s .c om*/ }
From source file:com.facebook.hive.orc.lazy.OrcLazyBinary.java
License:Open Source License
public OrcLazyBinary(OrcLazyBinary copy) { super(copy);//from w w w .j av a 2s . c om if (copy.previous != null) { BytesWritable copyPrevious = (BytesWritable) copy.previous; byte[] bytes = new byte[copyPrevious.getLength()]; System.arraycopy(copyPrevious.getBytes(), 0, bytes, 0, copyPrevious.getLength()); previous = new BytesWritable(bytes); } }
From source file:com.facebook.presto.hive.DwrfHiveRecordCursor.java
License:Apache License
private void parseStringColumn(int column) { // don't include column number in message because it causes boxing which is expensive here checkArgument(!isPartitionColumn[column], "Column is a partition key"); loaded[column] = true;/*from w w w . ja v a2 s .co m*/ nulls[column] = false; OrcLazyObject lazyObject = getRawValue(column); if (lazyObject == null) { nulls[column] = true; return; } Object value = materializeValue(lazyObject); if (value == null) { nulls[column] = true; return; } HiveType type = hiveTypes[column]; if (type.getCategory() == Category.MAP || type.getCategory() == Category.LIST || type.getCategory() == Category.STRUCT) { slices[column] = Slices .wrappedBuffer(getJsonBytes(sessionTimeZone, lazyObject, fieldInspectors[column])); } else if (type.equals(HIVE_STRING)) { Text text = checkWritable(value, Text.class); slices[column] = Slices.copyOf(Slices.wrappedBuffer(text.getBytes()), 0, text.getLength()); } else if (type.equals(HIVE_BINARY)) { BytesWritable bytesWritable = checkWritable(value, BytesWritable.class); slices[column] = Slices.copyOf(Slices.wrappedBuffer(bytesWritable.getBytes()), 0, bytesWritable.getLength()); } else { throw new RuntimeException(String.format("%s is not a valid STRING type", type)); } }