Example usage for org.apache.hadoop.io BytesWritable getBytes

Introduction

In this page you can find the example usage for org.apache.hadoop.io BytesWritable getBytes.

Prototype

@Override
public byte[] getBytes()

Source Link

Document

Get the data backing the BytesWritable.

Usage

From source file:com.snowplowanalytics.hive.serde.CfLogDeserializer.java

License:Open Source License

/**
 * Deserialize an object out of a Writable blob. In most cases, the return
 * value of this function will be constant since the function will reuse the
 * returned object. If the client wants to keep a copy of the object, the
 * client needs to clone the returned value by calling
 * ObjectInspectorUtils.getStandardObject().
 * //www  . ja v  a 2  s.  c o  m
 */
@Override
public Object deserialize(Writable field) throws SerDeException {
    String row = null;
    if (field instanceof BytesWritable) {
        BytesWritable b = (BytesWritable) field;
        try {
            row = Text.decode(b.getBytes(), 0, b.getLength());
        } catch (CharacterCodingException e) {
            throw new SerDeException(e);
        }
    } else if (field instanceof Text) {
        row = field.toString();
    }
    try {
        // Construct and return the S3LogStruct from the row data
        cachedStruct.parse(row);
        return cachedStruct;
    } catch (ClassCastException e) {
        throw new SerDeException(this.getClass().getName() + " expects Text or BytesWritable", e);
    } catch (Exception e) {
        throw new SerDeException(e);
    }
}

From source file:com.snowplowanalytics.snowplow.hadoop.hive.SnowPlowEventDeserializer.java

License:Open Source License

/**
 * Deserialize an object out of a Writable blob. In most cases, the return
 * value of this function will be constant since the function will reuse the
 * returned object. If the client wants to keep a copy of the object, the
 * client needs to clone the returned value by calling
 * ObjectInspectorUtils.getStandardObject().
 * /*from   w  w w . jav a  2  s . c o  m*/
 * @param blob The Writable object containing a serialized object
 * @return A Java object representing the contents in the blob.
 * @throws SerDeException For any exception during initialization
 */
@Override
public Object deserialize(Writable field) throws SerDeException {
    String row = null;
    if (field instanceof BytesWritable) {
        BytesWritable b = (BytesWritable) field;
        try {
            row = Text.decode(b.getBytes(), 0, b.getLength());
        } catch (CharacterCodingException e) {
            throw new SerDeException(e);
        }
    } else if (field instanceof Text) {
        row = field.toString();
    }
    try {
        // Update in place the S3LogStruct with the row data
        if (cachedStruct.updateByParsing(row))
            return cachedStruct;
        else
            return null;
    } catch (ClassCastException e) {
        throw new SerDeException(this.getClass().getName() + " expects Text or BytesWritable", e);
    } catch (Exception e) {
        if (this.continueOnUnexpectedError) {
            LOG.error("Could not parse row: \"" + row + "\"", e);
            return null;
        } else
            throw new SerDeException(e);
    }
}

From source file:com.yahoo.sketches.hive.frequencies.DataToStringsSketchUDAFTest.java

License:Apache License

@Test
public void iterateTerminatePartial() throws Exception {
    ObjectInspector[] inspectors = new ObjectInspector[] { stringInspector, intInspector };
    GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false);
    GenericUDAFEvaluator eval = new DataToStringsSketchUDAF().getEvaluator(info);
    ObjectInspector resultInspector = eval.init(Mode.PARTIAL1, inspectors);
    checkResultInspector(resultInspector);

    @SuppressWarnings("unchecked")
    ItemsState<String> state = (ItemsState<String>) eval.getNewAggregationBuffer();
    eval.iterate(state, new Object[] { new org.apache.hadoop.io.Text("a"), new IntWritable(256) });
    eval.iterate(state, new Object[] { new org.apache.hadoop.io.Text("b"), new IntWritable(256) });

    BytesWritable bytes = (BytesWritable) eval.terminatePartial(state);
    ItemsSketch<String> resultSketch = ItemsSketch.getInstance(new NativeMemory(bytes.getBytes()), serDe);
    Assert.assertEquals(resultSketch.getStreamLength(), 2);
    Assert.assertEquals(resultSketch.getNumActiveItems(), 2);
    Assert.assertEquals(resultSketch.getEstimate("a"), 1);
    Assert.assertEquals(resultSketch.getEstimate("b"), 1);
    eval.close();//from  w  w  w . ja  va 2  s . c  o  m
}

From source file:com.yahoo.sketches.hive.frequencies.DataToStringsSketchUDAFTest.java

License:Apache License

@Test
public void mergeTerminateEmptyState() throws Exception {
    ObjectInspector[] inspectors = new ObjectInspector[] { stringInspector, intInspector };
    GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false);
    GenericUDAFEvaluator eval = new DataToStringsSketchUDAF().getEvaluator(info);
    ObjectInspector resultInspector = eval.init(Mode.PARTIAL2, new ObjectInspector[] { binaryInspector });
    checkResultInspector(resultInspector);

    @SuppressWarnings("unchecked")
    ItemsState<String> state = (ItemsState<String>) eval.getNewAggregationBuffer();

    ItemsSketch<String> sketch1 = new ItemsSketch<String>(256);
    sketch1.update("a");
    eval.merge(state, new BytesWritable(sketch1.toByteArray(serDe)));

    ItemsSketch<String> sketch2 = new ItemsSketch<String>(256);
    sketch2.update("b");
    eval.merge(state, new BytesWritable(sketch2.toByteArray(serDe)));

    BytesWritable bytes = (BytesWritable) eval.terminate(state);
    ItemsSketch<String> resultSketch = ItemsSketch.getInstance(new NativeMemory(bytes.getBytes()), serDe);
    Assert.assertEquals(resultSketch.getStreamLength(), 2);
    Assert.assertEquals(resultSketch.getNumActiveItems(), 2);
    Assert.assertEquals(resultSketch.getEstimate("a"), 1);
    Assert.assertEquals(resultSketch.getEstimate("b"), 1);
    eval.close();//from  w w  w .  j  av  a2 s. c  o  m
}

From source file:com.yahoo.sketches.hive.frequencies.DataToStringsSketchUDAFTest.java

License:Apache License

@Test
public void mergeTerminate() throws Exception {
    ObjectInspector[] inspectors = new ObjectInspector[] { stringInspector, intInspector };
    GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false);
    GenericUDAFEvaluator eval = new DataToStringsSketchUDAF().getEvaluator(info);
    ObjectInspector resultInspector = eval.init(Mode.PARTIAL2, new ObjectInspector[] { binaryInspector });
    checkResultInspector(resultInspector);

    @SuppressWarnings("unchecked")
    ItemsState<String> state = (ItemsState<String>) eval.getNewAggregationBuffer();
    state.init(256);/*from w w  w  .j a  v a2s . com*/
    state.update("a");

    ItemsSketch<String> sketch = new ItemsSketch<String>(256);
    sketch.update("b");

    eval.merge(state, new BytesWritable(sketch.toByteArray(serDe)));

    BytesWritable bytes = (BytesWritable) eval.terminate(state);
    ItemsSketch<String> resultSketch = ItemsSketch.getInstance(new NativeMemory(bytes.getBytes()), serDe);
    Assert.assertEquals(resultSketch.getStreamLength(), 2);
    Assert.assertEquals(resultSketch.getNumActiveItems(), 2);
    Assert.assertEquals(resultSketch.getEstimate("a"), 1);
    Assert.assertEquals(resultSketch.getEstimate("b"), 1);
    eval.close();
}

From source file:com.yahoo.sketches.hive.frequencies.GetFrequentItemsFromStringsSketchUDTF.java

License:Apache License

@Override
public void process(final Object[] data) throws HiveException {
    if (data == null || data[0] == null)
        return;//from w w w.j  a  v  a 2  s  .co m
    final BytesWritable serializedSketch = (BytesWritable) inputObjectInspector
            .getPrimitiveWritableObject(data[0]);
    final ItemsSketch<String> sketch = ItemsSketch.getInstance(new NativeMemory(serializedSketch.getBytes()),
            new ArrayOfStringsSerDe());
    ErrorType errorType = ErrorType.NO_FALSE_POSITIVES;
    if (data.length > 1) {
        errorType = ErrorType.valueOf((String) errorTypeObjectInspector.getPrimitiveJavaObject(data[1]));
    }
    final ItemsSketch.Row<String>[] result = sketch.getFrequentItems(errorType);
    for (int i = 0; i < result.length; i++) {
        forward(new Object[] { result[i].getItem(), result[i].getEstimate(), result[i].getLowerBound(),
                result[i].getUpperBound() });
    }
}

From source file:com.yahoo.sketches.hive.frequencies.ItemsEvaluator.java

License:Apache License

@SuppressWarnings("deprecation")
@Override/*from   w  w w .  j  a  va  2  s .  c om*/
public void merge(final AggregationBuffer buf, Object data) throws HiveException {
    if (data == null)
        return;
    @SuppressWarnings("unchecked")
    final ItemsState<T> state = (ItemsState<T>) buf;
    final BytesWritable serializedSketch = (BytesWritable) inputObjectInspector
            .getPrimitiveWritableObject(data);
    state.update(serializedSketch.getBytes());
}

From source file:com.yahoo.sketches.hive.frequencies.UnionStringsSketchUDAFTest.java

License:Apache License

@Test
public void iterateTerminatePartial() throws Exception {
    ObjectInspector[] inspectors = new ObjectInspector[] { binaryInspector };
    GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false);
    GenericUDAFEvaluator eval = new UnionStringsSketchUDAF().getEvaluator(info);
    ObjectInspector resultInspector = eval.init(Mode.PARTIAL1, inspectors);
    checkResultInspector(resultInspector);

    @SuppressWarnings("unchecked")
    ItemsState<String> state = (ItemsState<String>) eval.getNewAggregationBuffer();
    state.init(256);/*from   w ww  .j a v  a 2s.co m*/
    state.update("a");

    ItemsSketch<String> sketch = new ItemsSketch<String>(256);
    sketch.update("b");

    eval.iterate(state, new Object[] { new BytesWritable(sketch.toByteArray(serDe)) });

    BytesWritable bytes = (BytesWritable) eval.terminatePartial(state);
    ItemsSketch<String> resultSketch = ItemsSketch.getInstance(new NativeMemory(bytes.getBytes()), serDe);
    Assert.assertEquals(resultSketch.getStreamLength(), 2);
    Assert.assertEquals(resultSketch.getNumActiveItems(), 2);
    Assert.assertEquals(resultSketch.getEstimate("a"), 1);
    Assert.assertEquals(resultSketch.getEstimate("b"), 1);
    eval.close();
}

From source file:com.yahoo.sketches.hive.frequencies.UnionStringsSketchUDAFTest.java

License:Apache License

@Test
public void mergeTerminate() throws Exception {
    ObjectInspector[] inspectors = new ObjectInspector[] { binaryInspector };
    GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false);
    GenericUDAFEvaluator eval = new UnionStringsSketchUDAF().getEvaluator(info);
    ObjectInspector resultInspector = eval.init(Mode.PARTIAL2, inspectors);
    checkResultInspector(resultInspector);

    @SuppressWarnings("unchecked")
    ItemsState<String> state = (ItemsState<String>) eval.getNewAggregationBuffer();
    state.init(256);/*  www. ja  va2 s.c  o m*/
    state.update("a");

    ItemsSketch<String> sketch = new ItemsSketch<String>(256);
    sketch.update("b");

    eval.merge(state, new BytesWritable(sketch.toByteArray(serDe)));

    BytesWritable bytes = (BytesWritable) eval.terminate(state);
    ItemsSketch<String> resultSketch = ItemsSketch.getInstance(new NativeMemory(bytes.getBytes()), serDe);
    Assert.assertEquals(resultSketch.getStreamLength(), 2);
    Assert.assertEquals(resultSketch.getNumActiveItems(), 2);
    Assert.assertEquals(resultSketch.getEstimate("a"), 1);
    Assert.assertEquals(resultSketch.getEstimate("b"), 1);
    eval.close();
}

From source file:com.yahoo.sketches.hive.quantiles.DoublesEvaluator.java

License:Apache License

@SuppressWarnings("deprecation")
@Override//w ww . j  a va  2s .c o  m
public void merge(final AggregationBuffer buf, Object data) throws HiveException {
    if (data == null)
        return;
    final DoublesUnionState state = (DoublesUnionState) buf;
    final BytesWritable serializedSketch = (BytesWritable) inputObjectInspector
            .getPrimitiveWritableObject(data);
    state.update(serializedSketch.getBytes());
}