List of usage examples for org.apache.hadoop.io BooleanWritable BooleanWritable
public BooleanWritable()
From source file:co.nubetech.hiho.merge.HihoValue.java
License:Apache License
@Override public void readFields(DataInput in) throws IOException { isOld = new BooleanWritable(); isOld.readFields(in);//from ww w . jav a 2 s . co m valClass = new Text(); valClass.readFields(in); try { val = (V) Class.forName(valClass.toString()).newInstance(); } catch (Exception e) { e.printStackTrace(); } val.readFields(in); }
From source file:com.cloudera.recordservice.mr.RecordServiceRecord.java
License:Apache License
/** * Returns the corresponding Writable object for this column type. *///from w w w .j a v a 2 s .c o m public Writable getWritableInstance(com.cloudera.recordservice.core.Schema.Type type) { switch (type) { case BOOLEAN: return new BooleanWritable(); case TINYINT: return new ByteWritable(); case SMALLINT: return new ShortWritable(); case INT: return new IntWritable(); case BIGINT: return new LongWritable(); case FLOAT: return new FloatWritable(); case DOUBLE: return new DoubleWritable(); case VARCHAR: case CHAR: case STRING: return new Text(); case TIMESTAMP_NANOS: return new TimestampNanosWritable(); case DECIMAL: return new DecimalWritable(); default: throw new UnsupportedOperationException("Unexpected type: " + toString()); } }
From source file:com.datasalt.utils.commons.TestExpiringSemaphore.java
License:Apache License
@Test public void testParallelism() throws InterruptedException { final ExpiringSemaphore semaphore = new ExpiringSemaphore(1, 2000); final BooleanWritable flag = new BooleanWritable(); Thread[] threads = new Thread[10]; for (int i = 0; i < 10; i++) { threads[i] = new Thread() { @Override/*www . ja va 2s . co m*/ public void run() { try { for (int i = 0; i < 200; i++) { semaphore.acquire(); if (flag.get()) { throw new RuntimeException("Semaphore is not behaving as expected"); } flag.set(true); flag.set(false); semaphore.release(); } } catch (InterruptedException e) { throw new RuntimeException(e); } } }; } for (int i = 0; i < 10; i++) { threads[i].start(); } for (int i = 0; i < 10; i++) { threads[i].join(); } }
From source file:com.ebay.nest.io.sede.binarysortable.BinarySortableSerDe.java
License:Apache License
static Object deserialize(InputByteBuffer buffer, TypeInfo type, boolean invert, Object reuse) throws IOException { // Is this field a null? byte isNull = buffer.read(invert); if (isNull == 0) { return null; }//from www . j a v a 2s. c o m assert (isNull == 1); switch (type.getCategory()) { case PRIMITIVE: { PrimitiveTypeInfo ptype = (PrimitiveTypeInfo) type; switch (ptype.getPrimitiveCategory()) { case VOID: { return null; } case BOOLEAN: { BooleanWritable r = reuse == null ? new BooleanWritable() : (BooleanWritable) reuse; byte b = buffer.read(invert); assert (b == 1 || b == 2); r.set(b == 2); return r; } case BYTE: { ByteWritable r = reuse == null ? new ByteWritable() : (ByteWritable) reuse; r.set((byte) (buffer.read(invert) ^ 0x80)); return r; } case SHORT: { ShortWritable r = reuse == null ? new ShortWritable() : (ShortWritable) reuse; int v = buffer.read(invert) ^ 0x80; v = (v << 8) + (buffer.read(invert) & 0xff); r.set((short) v); return r; } case INT: { IntWritable r = reuse == null ? new IntWritable() : (IntWritable) reuse; r.set(deserializeInt(buffer, invert)); return r; } case LONG: { LongWritable r = reuse == null ? new LongWritable() : (LongWritable) reuse; long v = buffer.read(invert) ^ 0x80; for (int i = 0; i < 7; i++) { v = (v << 8) + (buffer.read(invert) & 0xff); } r.set(v); return r; } case FLOAT: { FloatWritable r = reuse == null ? new FloatWritable() : (FloatWritable) reuse; int v = 0; for (int i = 0; i < 4; i++) { v = (v << 8) + (buffer.read(invert) & 0xff); } if ((v & (1 << 31)) == 0) { // negative number, flip all bits v = ~v; } else { // positive number, flip the first bit v = v ^ (1 << 31); } r.set(Float.intBitsToFloat(v)); return r; } case DOUBLE: { DoubleWritable r = reuse == null ? new DoubleWritable() : (DoubleWritable) reuse; long v = 0; for (int i = 0; i < 8; i++) { v = (v << 8) + (buffer.read(invert) & 0xff); } if ((v & (1L << 63)) == 0) { // negative number, flip all bits v = ~v; } else { // positive number, flip the first bit v = v ^ (1L << 63); } r.set(Double.longBitsToDouble(v)); return r; } case STRING: { Text r = reuse == null ? new Text() : (Text) reuse; return deserializeText(buffer, invert, r); } case VARCHAR: { HiveVarcharWritable r = reuse == null ? new HiveVarcharWritable() : (HiveVarcharWritable) reuse; // Use HiveVarchar's internal Text member to read the value. deserializeText(buffer, invert, r.getTextValue()); // If we cache helper data for deserialization we could avoid having // to call getVarcharMaxLength() on every deserialize call. r.enforceMaxLength(getVarcharMaxLength(type)); return r; } case BINARY: { BytesWritable bw = new BytesWritable(); // Get the actual length first int start = buffer.tell(); int length = 0; do { byte b = buffer.read(invert); if (b == 0) { // end of string break; } if (b == 1) { // the last char is an escape char. read the actual char buffer.read(invert); } length++; } while (true); if (length == buffer.tell() - start) { // No escaping happened, so we are already done. bw.set(buffer.getData(), start, length); } else { // Escaping happened, we need to copy byte-by-byte. // 1. Set the length first. bw.set(buffer.getData(), start, length); // 2. Reset the pointer. buffer.seek(start); // 3. Copy the data. byte[] rdata = bw.getBytes(); for (int i = 0; i < length; i++) { byte b = buffer.read(invert); if (b == 1) { // The last char is an escape char, read the actual char. // The serialization format escape \0 to \1, and \1 to \2, // to make sure the string is null-terminated. b = (byte) (buffer.read(invert) - 1); } rdata[i] = b; } // 4. Read the null terminator. byte b = buffer.read(invert); assert (b == 0); } return bw; } case DATE: { DateWritable d = reuse == null ? new DateWritable() : (DateWritable) reuse; d.set(deserializeInt(buffer, invert)); return d; } case TIMESTAMP: TimestampWritable t = (reuse == null ? new TimestampWritable() : (TimestampWritable) reuse); byte[] bytes = new byte[TimestampWritable.BINARY_SORTABLE_LENGTH]; for (int i = 0; i < bytes.length; i++) { bytes[i] = buffer.read(invert); } t.setBinarySortable(bytes, 0); return t; case DECIMAL: { // See serialization of decimal for explanation (below) HiveDecimalWritable bdw = (reuse == null ? new HiveDecimalWritable() : (HiveDecimalWritable) reuse); int b = buffer.read(invert) - 1; assert (b == 1 || b == -1 || b == 0); boolean positive = b != -1; int factor = buffer.read(invert) ^ 0x80; for (int i = 0; i < 3; i++) { factor = (factor << 8) + (buffer.read(invert) & 0xff); } if (!positive) { factor = -factor; } int start = buffer.tell(); int length = 0; do { b = buffer.read(positive ? invert : !invert); assert (b != 1); if (b == 0) { // end of digits break; } length++; } while (true); if (decimalBuffer == null || decimalBuffer.length < length) { decimalBuffer = new byte[length]; } buffer.seek(start); for (int i = 0; i < length; ++i) { decimalBuffer[i] = buffer.read(positive ? invert : !invert); } // read the null byte again buffer.read(positive ? invert : !invert); String digits = new String(decimalBuffer, 0, length, decimalCharSet); BigInteger bi = new BigInteger(digits); HiveDecimal bd = new HiveDecimal(bi).scaleByPowerOfTen(factor - length); if (!positive) { bd = bd.negate(); } bdw.set(bd); return bdw; } default: { throw new RuntimeException("Unrecognized type: " + ptype.getPrimitiveCategory()); } } } case LIST: { ListTypeInfo ltype = (ListTypeInfo) type; TypeInfo etype = ltype.getListElementTypeInfo(); // Create the list if needed ArrayList<Object> r = reuse == null ? new ArrayList<Object>() : (ArrayList<Object>) reuse; // Read the list int size = 0; while (true) { int more = buffer.read(invert); if (more == 0) { // \0 to terminate break; } // \1 followed by each element assert (more == 1); if (size == r.size()) { r.add(null); } r.set(size, deserialize(buffer, etype, invert, r.get(size))); size++; } // Remove additional elements if the list is reused while (r.size() > size) { r.remove(r.size() - 1); } return r; } case MAP: { MapTypeInfo mtype = (MapTypeInfo) type; TypeInfo ktype = mtype.getMapKeyTypeInfo(); TypeInfo vtype = mtype.getMapValueTypeInfo(); // Create the map if needed Map<Object, Object> r; if (reuse == null) { r = new HashMap<Object, Object>(); } else { r = (HashMap<Object, Object>) reuse; r.clear(); } while (true) { int more = buffer.read(invert); if (more == 0) { // \0 to terminate break; } // \1 followed by each key and then each value assert (more == 1); Object k = deserialize(buffer, ktype, invert, null); Object v = deserialize(buffer, vtype, invert, null); r.put(k, v); } return r; } case STRUCT: { StructTypeInfo stype = (StructTypeInfo) type; List<TypeInfo> fieldTypes = stype.getAllStructFieldTypeInfos(); int size = fieldTypes.size(); // Create the struct if needed ArrayList<Object> r = reuse == null ? new ArrayList<Object>(size) : (ArrayList<Object>) reuse; assert (r.size() <= size); // Set the size of the struct while (r.size() < size) { r.add(null); } // Read one field by one field for (int eid = 0; eid < size; eid++) { r.set(eid, deserialize(buffer, fieldTypes.get(eid), invert, r.get(eid))); } return r; } case UNION: { UnionTypeInfo utype = (UnionTypeInfo) type; StandardUnion r = reuse == null ? new StandardUnion() : (StandardUnion) reuse; // Read the tag byte tag = buffer.read(invert); r.setTag(tag); r.setObject(deserialize(buffer, utype.getAllUnionObjectTypeInfos().get(tag), invert, null)); return r; } default: { throw new RuntimeException("Unrecognized type: " + type.getCategory()); } } }
From source file:com.ebay.nest.io.sede.lazy.LazyBoolean.java
License:Apache License
public LazyBoolean(LazyBooleanObjectInspector oi) { super(oi); data = new BooleanWritable(); }
From source file:com.ebay.nest.io.sede.lazybinary.LazyBinaryBoolean.java
License:Apache License
public LazyBinaryBoolean(WritableBooleanObjectInspector oi) { super(oi); data = new BooleanWritable(); }
From source file:com.facebook.hive.orc.lazy.LazyBooleanTreeReader.java
License:Open Source License
BooleanWritable createWritable(Object previous, boolean v) throws IOException { BooleanWritable result = null;/* ww w.j a v a 2s . c om*/ if (previous == null) { result = new BooleanWritable(); } else { result = (BooleanWritable) previous; } result.set(v); return result; }
From source file:com.jfolson.hive.serde.RBaseSerDe.java
License:Apache License
protected Object deserializeField(RTypedBytesWritableInput in, TypeInfo type, Object reuse) throws IOException { RType rtype = in.readTypeCode();//from ww w. j a v a2s.c o m if (rtype == null) { throw new RuntimeException("End of stream"); } // read the type Class<? extends Writable> writableType = RType.getWritableType(rtype); if (writableType == null) { LOG.info("Warning: null Writable type for rtype: " + rtype); } if (writableType != null && writableType.isAssignableFrom(NullWritable.class)) { // indicates that the recorded value is null return null; } //LOG.info("RType should be instantiated as: "+writableType.getSimpleName()); switch (type.getCategory()) { case PRIMITIVE: { PrimitiveTypeInfo ptype = (PrimitiveTypeInfo) type; switch (ptype.getPrimitiveCategory()) { case VOID: { return null; } case BINARY: { TypedBytesWritable r = reuse == null ? new TypedBytesWritable() : (TypedBytesWritable) reuse; byte[] bytes = in.getInput().readRaw(rtype.code); // rewrite the type code r.set(bytes, 0, bytes.length); return r; } case BOOLEAN: { //TODO Fix this hack: if (rtype != RType.BOOL) { in.readNull(); return null; } BooleanWritable r = reuse == null ? new BooleanWritable() : (BooleanWritable) reuse; return in.readBoolean(r); } /*case BYTE: { ByteWritable r = reuse == null ? new ByteWritable() : (ByteWritable) reuse; r = in.readByte(r); return r; }*/ /*case SHORT: { ShortWritable r = reuse == null ? new ShortWritable() : (ShortWritable) reuse; r = in.readShort(r); return r; }*/ case INT: { if (rtype != RType.INT) { in.readNull(); return null; } IntWritable r = reuse == null ? null : (IntWritable) reuse; return in.readInt(r); } /*case LONG: { LongWritable r = reuse == null ? new LongWritable() : (LongWritable) reuse; r = in.readLong(r); return r; }*/ /*case FLOAT: { FloatWritable r = reuse == null ? new FloatWritable() : (FloatWritable) reuse; r = in.readFloat(r); return r; }*/ case DOUBLE: { if (rtype != RType.DOUBLE) { in.readNull(); return null; } DoubleWritable r = reuse == null ? null : (DoubleWritable) reuse; return in.readDouble(r); } case STRING: { // TODO fix this hack if (rtype != RType.STRING) { in.readNull(); return null; } Text r = reuse == null ? null : (Text) reuse; return in.readText(r); } default: { throw new RuntimeException("Unrecognized type: " + ptype.getPrimitiveCategory()); } } } // Currently, deserialization of complex types is not supported case LIST: { if (rtype != RType.VECTOR) { in.readNull(); return null; } ObjectInspector elemOI = ((ListObjectInspector) TypeInfoUtils .getStandardWritableObjectInspectorFromTypeInfo(type)).getListElementObjectInspector(); PrimitiveObjectInspector elemPOI = (PrimitiveObjectInspector) elemOI; Class<? extends Writable> elemClass = (Class<? extends Writable>) elemPOI.getPrimitiveWritableClass(); ArrayWritable l = reuse == null ? new ArrayWritable(elemClass) : new ArrayWritable(elemClass, (Writable[]) reuse); in.readVector(l); return l.get(); } case MAP: case STRUCT: default: { throw new RuntimeException("Unsupported category: " + type.getCategory()); } } }
From source file:com.jfolson.hive.serde.RBaseSerDe.java
License:Apache License
protected void serializeField(Object o, ObjectInspector oi, Object reuse) throws IOException { //LOG.info("Serializing hive type: "+oi.getTypeName()); //LOG.info("Serializing category: "+oi.getCategory().toString()); if (o == null) { tbOut.writeNull();/*w ww. j a va2 s.c o m*/ return; } switch (oi.getCategory()) { case PRIMITIVE: { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; //LOG.info("Serializing primitive: "+poi.getPrimitiveCategory().toString()); switch (poi.getPrimitiveCategory()) { case VOID: { return; } case BINARY: { BinaryObjectInspector boi = (BinaryObjectInspector) poi; TypedBytesWritable bytes = reuse == null ? new TypedBytesWritable() : (TypedBytesWritable) reuse; BytesWritable bytesWrite = boi.getPrimitiveWritableObject(o); if (bytesWrite != null) { bytes.set(bytesWrite); if (!RType.isValid(bytes)) { LOG.error("Invalid typedbytes detected with type: " + RType.getType(bytes).code); bytes.setValue(new Buffer(bytesWrite.getBytes(), 0, bytesWrite.getLength())); } //LOG.info("Writing binary primitive with class: "+bytes.getClass().getName()); tbOut.write(bytes); } return; } case BOOLEAN: { BooleanObjectInspector boi = (BooleanObjectInspector) poi; BooleanWritable r = reuse == null ? new BooleanWritable() : (BooleanWritable) reuse; r.set(boi.get(o)); tbOut.write(r); return; } case BYTE: { ByteObjectInspector boi = (ByteObjectInspector) poi; ByteWritable r = reuse == null ? new ByteWritable() : (ByteWritable) reuse; r.set(boi.get(o)); tbOut.write(r); return; } case SHORT: { ShortObjectInspector spoi = (ShortObjectInspector) poi; ShortWritable r = reuse == null ? new ShortWritable() : (ShortWritable) reuse; r.set(spoi.get(o)); tbOut.write(r); return; } case INT: { IntObjectInspector ioi = (IntObjectInspector) poi; IntWritable r = reuse == null ? new IntWritable() : (IntWritable) reuse; r.set(ioi.get(o)); tbOut.write(r); return; } case LONG: { LongObjectInspector loi = (LongObjectInspector) poi; LongWritable r = reuse == null ? new LongWritable() : (LongWritable) reuse; r.set(loi.get(o)); tbOut.write(r); return; } case FLOAT: { FloatObjectInspector foi = (FloatObjectInspector) poi; FloatWritable r = reuse == null ? new FloatWritable() : (FloatWritable) reuse; r.set(foi.get(o)); tbOut.write(r); return; } case DOUBLE: DoubleObjectInspector doi = (DoubleObjectInspector) poi; DoubleWritable r = reuse == null ? new DoubleWritable() : (DoubleWritable) reuse; r.set(doi.get(o)); tbOut.write(r); return; case STRING: { StringObjectInspector soi = (StringObjectInspector) poi; Text t = soi.getPrimitiveWritableObject(o); tbOut.write(t); return; } default: { throw new RuntimeException("Unrecognized type: " + poi.getPrimitiveCategory()); } } } case LIST: { ListObjectInspector loi = (ListObjectInspector) oi; ObjectInspector elemOI = loi.getListElementObjectInspector(); List l = loi.getList(o); // Don't use array (typecode: 144) until everything supports NA values in typedbytes if (false) {//(elemOI.getCategory()==ObjectInspector.Category.PRIMITIVE){ tbOut.writeArray(l, (PrimitiveObjectInspector) elemOI); } else { tbOut.writeVector(l, (PrimitiveObjectInspector) elemOI); } return; } case MAP: case STRUCT: { // For complex object, serialize to JSON format String s = SerDeUtils.getJSONString(o, oi); Text t = reuse == null ? new Text() : (Text) reuse; // convert to Text and write it t.set(s); tbOut.write(t); return; } default: { throw new RuntimeException("Unrecognized type: " + oi.getCategory()); } } }
From source file:com.jfolson.hive.serde.RTypedBytesSerDe.java
License:Apache License
Object deserializeField(RTypedBytesWritableInput in, TypeInfo type, Object reuse) throws IOException { RType rtype = in.readTypeCode();//from w w w .j ava2 s . c om if (rtype == null) { throw new RuntimeException("End of stream"); } // read the type Class<? extends Writable> writableType = RType.getWritableType(rtype); if (writableType == null) { LOG.info("Warning: null Writable type for rtype: " + rtype); } if (writableType != null && writableType.isAssignableFrom(NullWritable.class)) { // indicates that the recorded value is null return null; } //LOG.info("RType should be instantiated as: "+writableType.getSimpleName()); switch (type.getCategory()) { case PRIMITIVE: { PrimitiveTypeInfo ptype = (PrimitiveTypeInfo) type; switch (ptype.getPrimitiveCategory()) { case VOID: { return null; } case BINARY: { TypedBytesWritable r = reuse == null ? new TypedBytesWritable() : (TypedBytesWritable) reuse; byte[] bytes = in.getInput().readRaw(rtype.code); // rewrite the type code r.set(bytes, 0, bytes.length); return r; } case BOOLEAN: { //TODO Fix this hack: if (rtype != RType.BOOL) { in.readNull(); return null; } BooleanWritable r = reuse == null ? new BooleanWritable() : (BooleanWritable) reuse; return in.readBoolean(r); } /*case BYTE: { ByteWritable r = reuse == null ? new ByteWritable() : (ByteWritable) reuse; r = in.readByte(r); return r; }*/ /*case SHORT: { ShortWritable r = reuse == null ? new ShortWritable() : (ShortWritable) reuse; r = in.readShort(r); return r; }*/ case INT: { if (rtype != RType.INT) { in.readNull(); return null; } IntWritable r = reuse == null ? null : (IntWritable) reuse; return in.readInt(r); } /*case LONG: { LongWritable r = reuse == null ? new LongWritable() : (LongWritable) reuse; r = in.readLong(r); return r; }*/ /*case FLOAT: { FloatWritable r = reuse == null ? new FloatWritable() : (FloatWritable) reuse; r = in.readFloat(r); return r; }*/ case DOUBLE: { if (rtype != RType.DOUBLE) { in.readNull(); return null; } DoubleWritable r = reuse == null ? null : (DoubleWritable) reuse; return in.readDouble(r); } case STRING: { // TODO fix this hack if (rtype != RType.STRING) { in.readNull(); return null; } Text r = reuse == null ? null : (Text) reuse; return in.readText(r); } default: { throw new RuntimeException("Unrecognized type: " + ptype.getPrimitiveCategory()); } } } // Currently, deserialization of complex types is not supported case LIST: { if (rtype != RType.VECTOR) { in.readNull(); return null; } ObjectInspector elemOI = ((ListObjectInspector) TypeInfoUtils .getStandardWritableObjectInspectorFromTypeInfo(type)).getListElementObjectInspector(); PrimitiveObjectInspector elemPOI = (PrimitiveObjectInspector) elemOI; Class<? extends Writable> elemClass = (Class<? extends Writable>) elemPOI.getPrimitiveWritableClass(); ArrayWritable l = reuse == null ? new ArrayWritable(elemClass) : new ArrayWritable(elemClass, (Writable[]) reuse); in.readVector(l); return l.get(); } case MAP: case STRUCT: default: { throw new RuntimeException("Unsupported category: " + type.getCategory()); } } }