List of usage examples for org.apache.hadoop.io BytesWritable getLength
@Override public int getLength()
From source file:com.facebook.presto.hive.orc.DwrfHiveRecordCursor.java
License:Apache License
private void parseStringColumn(int column) { // don't include column number in message because it causes boxing which is expensive here checkArgument(!isPartitionColumn[column], "Column is a partition key"); loaded[column] = true;/*from w ww .j a v a2s . c o m*/ nulls[column] = false; OrcLazyObject lazyObject = getRawValue(column); if (lazyObject == null) { nulls[column] = true; return; } Object value = materializeValue(lazyObject); if (value == null) { nulls[column] = true; return; } HiveType type = hiveTypes[column]; if (type.equals(HIVE_STRING)) { Text text = checkWritable(value, Text.class); slices[column] = Slices.copyOf(Slices.wrappedBuffer(text.getBytes()), 0, text.getLength()); } else if (type.equals(HIVE_BINARY)) { BytesWritable bytesWritable = checkWritable(value, BytesWritable.class); slices[column] = Slices.copyOf(Slices.wrappedBuffer(bytesWritable.getBytes()), 0, bytesWritable.getLength()); } else { throw new RuntimeException(String.format("%s is not a valid STRING type", type)); } }
From source file:com.facebook.presto.hive.orc.OrcHiveRecordCursor.java
License:Apache License
private void parseStringColumn(int column) { // don't include column number in message because it causes boxing which is expensive here checkArgument(!isPartitionColumn[column], "Column is a partition key"); loaded[column] = true;/*from ww w. j a v a 2 s.c o m*/ nulls[column] = false; Object object = getFieldValue(row, hiveColumnIndexes[column]); if (object == null) { nulls[column] = true; return; } HiveType type = hiveTypes[column]; if (type.equals(HIVE_STRING)) { Text text = Types.checkType(object, Text.class, "materialized string value"); slices[column] = Slices.copyOf(Slices.wrappedBuffer(text.getBytes()), 0, text.getLength()); } else if (type.equals(HIVE_BINARY)) { BytesWritable bytesWritable = Types.checkType(object, BytesWritable.class, "materialized binary value"); slices[column] = Slices.copyOf(Slices.wrappedBuffer(bytesWritable.getBytes()), 0, bytesWritable.getLength()); } else { throw new RuntimeException(String.format("%s is not a valid STRING type", type)); } }
From source file:com.facebook.presto.hive.OrcHiveRecordCursor.java
License:Apache License
private void parseStringColumn(int column) { // don't include column number in message because it causes boxing which is expensive here checkArgument(!isPartitionColumn[column], "Column is a partition key"); loaded[column] = true;/* ww w.j a v a 2 s . co m*/ nulls[column] = false; Object object = getFieldValue(row, hiveColumnIndexes[column]); if (object == null) { nulls[column] = true; return; } HiveType type = hiveTypes[column]; if (type.getCategory() == Category.MAP || type.getCategory() == Category.LIST || type.getCategory() == Category.STRUCT) { slices[column] = Slices.wrappedBuffer(getJsonBytes(sessionTimeZone, object, fieldInspectors[column])); } else if (type.equals(HIVE_STRING)) { Text text = Types.checkType(object, Text.class, "materialized string value"); slices[column] = Slices.copyOf(Slices.wrappedBuffer(text.getBytes()), 0, text.getLength()); } else if (type.equals(HIVE_BINARY)) { BytesWritable bytesWritable = Types.checkType(object, BytesWritable.class, "materialized binary value"); slices[column] = Slices.copyOf(Slices.wrappedBuffer(bytesWritable.getBytes()), 0, bytesWritable.getLength()); } else { throw new RuntimeException(String.format("%s is not a valid STRING type", type)); } }
From source file:com.facebook.presto.hive.util.SerDeUtils.java
License:Apache License
private static String getPrimitiveAsString(DateTimeZone sessionTimeZone, Object object, PrimitiveObjectInspector inspector) { switch (inspector.getPrimitiveCategory()) { case BOOLEAN: case BYTE:/*from w w w.j a v a 2 s. c om*/ case SHORT: case INT: case LONG: case FLOAT: case DOUBLE: case STRING: return String.valueOf(inspector.getPrimitiveJavaObject(object)); case TIMESTAMP: return String.valueOf(formatTimestamp(sessionTimeZone, object, (TimestampObjectInspector) inspector)); case BINARY: // Using same Base64 encoder which Jackson uses in JsonGenerator.writeBinary(). BytesWritable writable = ((BinaryObjectInspector) inspector).getPrimitiveWritableObject(object); return Base64Variants.getDefaultVariant() .encode(Arrays.copyOf(writable.getBytes(), writable.getLength())); default: throw new RuntimeException("Unknown primitive type: " + inspector.getPrimitiveCategory()); } }
From source file:com.gotometrics.orderly.FixedByteArrayRowKey.java
License:Apache License
@Override public Object deserialize(ImmutableBytesWritable w) throws IOException { BytesWritable bw = (BytesWritable) super.deserialize(w); if (bw == null) { return null; } else {//from w w w . j a va2s .c o m final byte[] result = new byte[bw.getLength()]; System.arraycopy(bw.getBytes(), 0, result, 0, bw.getLength()); return result; } }
From source file:com.gotometrics.orderly.FixedBytesWritableRowKey.java
License:Apache License
@Override public void serialize(Object o, ImmutableBytesWritable w) throws IOException { byte[] bytesToWriteIn = w.get(); int writeOffset = w.getOffset(); final BytesWritable bytesWritableToWrite = (BytesWritable) o; final int srcLen = bytesWritableToWrite.getLength(); final byte[] bytesToWrite = bytesWritableToWrite.getBytes(); if (srcLen != length) throw new IllegalArgumentException( "can only serialize byte arrays of length " + length + ", not " + srcLen); // apply the sort order mask final byte[] maskedBytesToWrite = maskAll(bytesToWrite, order, 0, srcLen); Bytes.putBytes(bytesToWriteIn, writeOffset, maskedBytesToWrite, 0, srcLen); RowKeyUtils.seek(w, srcLen);/*from w w w.j a v a2 s .c om*/ }
From source file:com.gotometrics.orderly.VariableLengthBytesWritableRowKey.java
License:Apache License
@Override public int getSerializedLength(Object o) throws IOException { if (o == null) return terminate() ? fixedPrefixLength + 1 : fixedPrefixLength; final BytesWritable input = (BytesWritable) o; return fixedPrefixLength + getSerializedLength( toStringRepresentation(input.getBytes(), fixedPrefixLength, input.getLength() - fixedPrefixLength)); }
From source file:com.gotometrics.orderly.VariableLengthBytesWritableRowKey.java
License:Apache License
@Override public void serialize(Object o, ImmutableBytesWritable bytesWritable) throws IOException { byte[] bytesToWriteIn = bytesWritable.get(); int offset = bytesWritable.getOffset(); if (o == null) { if (fixedPrefixLength > 0) throw new IllegalStateException("excepted at least " + fixedPrefixLength + " bytes to write"); else if (terminate()) { // write one (masked) null byte bytesToWriteIn[offset] = mask(NULL); RowKeyUtils.seek(bytesWritable, 1); }//from w w w . j ava 2s . c om } else { final BytesWritable input = (BytesWritable) o; if (fixedPrefixLength > input.getLength()) throw new IllegalStateException("excepted at least " + fixedPrefixLength + " bytes to write"); else { encodeFixedPrefix(input.getBytes(), bytesWritable); encodedCustomizedReversedPackedBcd(toStringRepresentation(input.getBytes(), fixedPrefixLength, input.getLength() - fixedPrefixLength), bytesWritable); } } }
From source file:com.inmobi.messaging.consumer.databus.mapred.TestDatabusBytesWritableInputFormat.java
License:Apache License
/** * read the the given input split./*from w w w. j a va 2 s . c o m*/ * @return List : List of read messages */ private List<Message> readSplit(DatabusBytesWritableInputFormat format, InputSplit split, JobConf job, Reporter reporter) throws IOException { List<Message> result = new ArrayList<Message>(); RecordReader<LongWritable, BytesWritable> reader = format.getRecordReader(split, job, reporter); LongWritable key = ((DatabusBytesWritableRecordReader) reader).createKey(); BytesWritable value = ((DatabusBytesWritableRecordReader) reader).createValue(); while (((DatabusBytesWritableRecordReader) reader).next(key, value)) { byte[] data = new byte[value.getLength()]; System.arraycopy(value.getBytes(), 0, data, 0, value.getLength()); result.add(new Message(ByteBuffer.wrap(data))); value = (BytesWritable) ((DatabusBytesWritableRecordReader) reader).createValue(); } reader.close(); return result; }
From source file:com.jfolson.hive.serde.RBaseSerDe.java
License:Apache License
@Override public Object deserialize(Writable blob) throws SerDeException { BytesWritable data = (BytesWritable) blob; inBarrStr.reset(data.getBytes(), 0, data.getLength()); try {/*from ww w. ja v a2 s. c o m*/ //Read the key if (!keepAsBytes && unwrapKeys) { // It's wrapped as a list TWICE, because R will try to c() things once deserialized RType rtype = tbIn.getInput().readType(); if (rtype == null) { throw new RuntimeException("End of stream"); } if (rtype != RType.VECTOR) { throw new RuntimeException("Error: expecting vector<vector<key>>, instead of " + rtype.name()); } int vectorLength = tbIn.getInput().readVectorHeader(); if (vectorLength != 1) { throw new RuntimeException("Hive cannot support multiple keys"); } rtype = tbIn.getInput().readType(); if (rtype == null) { throw new RuntimeException("Unexpected end of stream"); } if (rtype != RType.VECTOR) { throw new RuntimeException("Error: expecting vector<key>, instead of " + rtype.name()); } vectorLength = tbIn.getInput().readVectorHeader(); if (vectorLength != numKeys) { throw new RuntimeException( "Error: expecting " + numKeys + " values, but list only has: " + vectorLength); } } for (int i = 0; i < numKeys; i++) { //LOG.info("Deserializing column: "+i); row.set(i, deserializeField(tbIn, columnTypes.get(i), row.get(i))); } //Read the value if (!keepAsBytes && unwrapValues) { // It's wrapped as a list TWICE, because R will try to c() things once deserialized RType rtype = tbIn.getInput().readType(); if (rtype == null) { throw new RuntimeException("End of stream"); } if (rtype != RType.VECTOR) { throw new RuntimeException( "Error: expecting vector<vector<value>>, instead of " + rtype.name()); } int vectorLength = tbIn.getInput().readVectorHeader(); if (vectorLength != 1) { throw new RuntimeException("Hive cannot support multiple values"); } rtype = tbIn.getInput().readType(); if (rtype == null) { throw new RuntimeException("Unexpected end of stream"); } if (rtype != RType.VECTOR) { throw new RuntimeException("Error: expecting vector<key>, instead of " + rtype.name()); } vectorLength = tbIn.getInput().readVectorHeader(); if (vectorLength != (numColumns - numKeys)) { throw new RuntimeException("Error: expecting " + (numColumns - numKeys) + " values, but list only has: " + vectorLength); } } for (int i = numKeys; i < numColumns; i++) { //LOG.info("Deserializing column: "+i); row.set(i, deserializeField(tbIn, columnTypes.get(i), row.get(i))); } // The next byte should be the marker // R doesn't want this //assert tbIn.readTypeCode() == Type.ENDOFRECORD; } catch (IOException e) { throw new SerDeException(e); } return row; }