List of usage examples for org.apache.hadoop.io BytesWritable getLength
@Override public int getLength()
From source file:com.snowplowanalytics.hive.serde.CfLogDeserializer.java
License:Open Source License
/** * Deserialize an object out of a Writable blob. In most cases, the return * value of this function will be constant since the function will reuse the * returned object. If the client wants to keep a copy of the object, the * client needs to clone the returned value by calling * ObjectInspectorUtils.getStandardObject(). * //from w ww .jav a2s. com */ @Override public Object deserialize(Writable field) throws SerDeException { String row = null; if (field instanceof BytesWritable) { BytesWritable b = (BytesWritable) field; try { row = Text.decode(b.getBytes(), 0, b.getLength()); } catch (CharacterCodingException e) { throw new SerDeException(e); } } else if (field instanceof Text) { row = field.toString(); } try { // Construct and return the S3LogStruct from the row data cachedStruct.parse(row); return cachedStruct; } catch (ClassCastException e) { throw new SerDeException(this.getClass().getName() + " expects Text or BytesWritable", e); } catch (Exception e) { throw new SerDeException(e); } }
From source file:com.snowplowanalytics.snowplow.hadoop.hive.SnowPlowEventDeserializer.java
License:Open Source License
/** * Deserialize an object out of a Writable blob. In most cases, the return * value of this function will be constant since the function will reuse the * returned object. If the client wants to keep a copy of the object, the * client needs to clone the returned value by calling * ObjectInspectorUtils.getStandardObject(). * //from w w w. ja v a 2s . c o m * @param blob The Writable object containing a serialized object * @return A Java object representing the contents in the blob. * @throws SerDeException For any exception during initialization */ @Override public Object deserialize(Writable field) throws SerDeException { String row = null; if (field instanceof BytesWritable) { BytesWritable b = (BytesWritable) field; try { row = Text.decode(b.getBytes(), 0, b.getLength()); } catch (CharacterCodingException e) { throw new SerDeException(e); } } else if (field instanceof Text) { row = field.toString(); } try { // Update in place the S3LogStruct with the row data if (cachedStruct.updateByParsing(row)) return cachedStruct; else return null; } catch (ClassCastException e) { throw new SerDeException(this.getClass().getName() + " expects Text or BytesWritable", e); } catch (Exception e) { if (this.continueOnUnexpectedError) { LOG.error("Could not parse row: \"" + row + "\"", e); return null; } else throw new SerDeException(e); } }
From source file:com.yahoo.sketches.hive.theta.ExcludeSketchUDF.java
License:Apache License
/** * Main logic called by hive if sketchSize is also passed in. Computes the * hash in first sketch excluding the hash in second sketch of two sketches of * same or different column./*from w w w .j av a 2 s . c o m*/ * * @param firstSketchBytes * first sketch to be included. * @param secondSketchBytes * second sketch to be excluded. * @param hashSeed * Only required if input sketches were constructed using an update seed that was not the default. * @return resulting sketch of exclusion. */ public BytesWritable evaluate(final BytesWritable firstSketchBytes, final BytesWritable secondSketchBytes, final long hashSeed) { Sketch firstSketch = null; if (firstSketchBytes != null && firstSketchBytes.getLength() > 0) { firstSketch = Sketch.wrap(new NativeMemory(firstSketchBytes.getBytes()), hashSeed); } Sketch secondSketch = null; if (secondSketchBytes != null && secondSketchBytes.getLength() > 0) { secondSketch = Sketch.wrap(new NativeMemory(secondSketchBytes.getBytes()), hashSeed); } final AnotB anotb = SetOperation.builder().setSeed(hashSeed).buildANotB(); anotb.update(firstSketch, secondSketch); final byte[] excludeSketchBytes = anotb.getResult().toByteArray(); final BytesWritable result = new BytesWritable(); result.set(excludeSketchBytes, 0, excludeSketchBytes.length); return result; }
From source file:com.yahoo.sketches.hive.theta.IntersectSketchUDF.java
License:Apache License
/** * Main logic called by hive if sketchSize is also passed in. Computes the * intersection of two sketches of same or different column. * // www. ja v a 2 s . c o m * @param firstSketchBytes * first sketch to be intersected. * @param secondSketchBytes * second sketch to be intersected. * @param hashSeed * Only required if input sketches were constructed using an update seed that was not the default. * @return resulting sketch of intersection. */ public BytesWritable evaluate(final BytesWritable firstSketchBytes, final BytesWritable secondSketchBytes, final long hashSeed) { Sketch firstSketch = null; if (firstSketchBytes != null && firstSketchBytes.getLength() > 0) { firstSketch = Sketch.wrap(new NativeMemory(firstSketchBytes.getBytes()), hashSeed); } Sketch secondSketch = null; if (secondSketchBytes != null && secondSketchBytes.getLength() > 0) { secondSketch = Sketch.wrap(new NativeMemory(secondSketchBytes.getBytes()), hashSeed); } final Intersection intersect = SetOperation.builder().setSeed(hashSeed).buildIntersection(); intersect.update(firstSketch); intersect.update(secondSketch); return new BytesWritable(intersect.getResult().toByteArray()); }
From source file:com.yahoo.sketches.hive.theta.UnionSketchUDF.java
License:Apache License
/** * Main logic called by hive if sketchSize is also passed in. Union two * sketches of same or different column. * /* w ww .ja va 2s .com*/ * @param firstSketch * first sketch to be unioned. * @param secondSketch * second sketch to be unioned. * @param sketchSize * final output unioned sketch size. * This must be a power of 2 and larger than 16. * @param seed using the seed is not recommended unless you really know why you need it. * @return resulting sketch of union. */ public BytesWritable evaluate(final BytesWritable firstSketch, final BytesWritable secondSketch, final int sketchSize, final long seed) { final Union union = SetOperation.builder().setSeed(seed).buildUnion(sketchSize); if ((firstSketch != null) && (firstSketch.getLength() >= EMPTY_SKETCH_SIZE_BYTES)) { union.update(new NativeMemory(firstSketch.getBytes())); } if ((secondSketch != null) && (secondSketch.getLength() >= EMPTY_SKETCH_SIZE_BYTES)) { union.update(new NativeMemory(secondSketch.getBytes())); } return new BytesWritable(union.getResult().toByteArray()); }
From source file:crunch.MaxTemperature.java
License:Apache License
@Test
public void test() throws IOException {
// vv BytesWritableTest
BytesWritable b = new BytesWritable(new byte[] { 3, 5 });
byte[] bytes = serialize(b);
assertThat(StringUtils.byteToHexString(bytes), is("000000020305"));
// ^^ BytesWritableTest
// vv BytesWritableTest-Capacity
b.setCapacity(11);/*from www. j a va 2 s .c o m*/
assertThat(b.getLength(), is(2));
assertThat(b.getBytes().length, is(11));
// ^^ BytesWritableTest-Capacity
}
From source file:edu.arizona.cs.hadoop.fs.irods.output.HirodsSequenceFileAsBinaryOutputFormat.java
License:Apache License
@Override public RecordWriter<BytesWritable, BytesWritable> getRecordWriter(TaskAttemptContext context) throws IOException { final SequenceFile.Writer out = getSequenceWriter(context, getSequenceFileOutputKeyClass(context), getSequenceFileOutputValueClass(context)); return new RecordWriter<BytesWritable, BytesWritable>() { private WritableValueBytes wvaluebytes = new WritableValueBytes(); @Override/* w ww .ja v a 2 s . co m*/ public void write(BytesWritable bkey, BytesWritable bvalue) throws IOException { wvaluebytes.reset(bvalue); out.appendRaw(bkey.getBytes(), 0, bkey.getLength(), wvaluebytes); wvaluebytes.reset(null); } @Override public void close(TaskAttemptContext context) throws IOException { out.close(); } }; }
From source file:edu.stolaf.cs.wmrserver.streaming.PipeMapRed.java
License:Apache License
/** * Write a value to the output stream using UTF-8 encoding * @param value output value// ww w .java2 s. c o m * @throws IOException */ void write(Object value) throws IOException { byte[] bval; int valSize; if (value instanceof BytesWritable) { BytesWritable val = (BytesWritable) value; bval = val.getBytes(); valSize = val.getLength(); } else if (value instanceof Text) { Text val = (Text) value; bval = val.getBytes(); valSize = val.getLength(); } else { String sval = value.toString(); bval = sval.getBytes("UTF-8"); valSize = bval.length; } clientOut_.write(bval, 0, valSize); }
From source file:hivemall.tools.compress.InflateUDF.java
License:Apache License
@Override public Text evaluate(DeferredObject[] arguments) throws HiveException { if (codec == null) { this.codec = new DeflateCodec(false, true); }//from ww w . j av a 2 s.com Object arg0 = arguments[0].get(); if (arg0 == null) { return null; } BytesWritable b = binaryOI.getPrimitiveWritableObject(arg0); byte[] compressed = b.getBytes(); final int len = b.getLength(); final byte[] decompressed; try { decompressed = codec.decompress(compressed, 0, len); } catch (IOException e) { throw new HiveException("Failed to decompressed. Compressed data format is illegal.", e); } compressed = null; if (result == null) { result = new Text(decompressed); } else { result.set(decompressed, 0, decompressed.length); } return result; }
From source file:hivemall.tools.text.Base91UDF.java
License:Apache License
@Override public Text evaluate(DeferredObject[] arguments) throws HiveException { if (outputBuf == null) { this.outputBuf = new FastByteArrayOutputStream(4096); } else {//ww w .j av a2s .co m outputBuf.reset(); } Object arg0 = arguments[0].get(); if (arg0 == null) { return null; } BytesWritable input = binaryOI.getPrimitiveWritableObject(arg0); final byte[] inputBytes = input.getBytes(); final int len = input.getLength(); try { Base91.encode(inputBytes, 0, len, outputBuf); } catch (IOException e) { throw new HiveException(e); } if (result == null) { byte[] outputBytes = outputBuf.toByteArray(); this.result = new Text(outputBytes); } else { byte[] outputBytes = outputBuf.getInternalArray(); int outputSize = outputBuf.size(); result.set(outputBytes, 0, outputSize); } return result; }