Example usage for org.apache.hadoop.io BytesWritable getBytes

Introduction

In this page you can find the example usage for org.apache.hadoop.io BytesWritable getBytes.

Prototype

@Override
public byte[] getBytes() 

Source Link

Document

Get the data backing the BytesWritable.

Usage

From source file:com.jointhegrid.hivecasudfs.UDFCasMapJoin.java

License:Apache License

@Override
public Object evaluate(DeferredObject[] arguments) throws HiveException {

    clusterName = ((StringObjectInspector) argumentOI[0]).getPrimitiveJavaObject(arguments[0].get());
    hostlist = ((StringObjectInspector) argumentOI[1]).getPrimitiveJavaObject(arguments[1].get());
    keyspace = ((StringObjectInspector) argumentOI[2]).getPrimitiveJavaObject(arguments[2].get());
    columnFamily = ((StringObjectInspector) argumentOI[3]).getPrimitiveJavaObject(arguments[3].get());
    BytesWritable aRowKey = ((BinaryObjectInspector) argumentOI[4])
            .getPrimitiveWritableObject(arguments[4].get());
    BytesWritable aColumn = ((BinaryObjectInspector) argumentOI[5])
            .getPrimitiveWritableObject(arguments[5].get());

    byte[] rk = new byte[aRowKey.getLength()];
    for (int i = 0; i < rk.length; i++) {
        rk[i] = aRowKey.getBytes()[i];
    }/*from  w w  w .  ja v a 2  s  .c  o  m*/
    byte[] cl = new byte[aColumn.getLength()];
    for (int i = 0; i < cl.length; i++) {
        cl[i] = aColumn.getBytes()[i];
    }
    rowKey = ByteBuffer.wrap(rk);
    //column = ByteBuffer.wrap(aColumn.getBytes());
    //column = ByteBuffer.wrap( "lname".getBytes() );
    System.out.println(cl.length);
    System.out.println(new String(cl));
    column = ByteBuffer.wrap(cl);

    if (cluster == null) {
        cluster = HFactory.getOrCreateCluster(clusterName, hostlist);
        ksp = HFactory.createKeyspace(keyspace, cluster);
        cft = new ThriftColumnFamilyTemplate(ksp, this.columnFamily, ByteBufferSerializer.get(),
                ByteBufferSerializer.get());
    }
    try {
        System.out.println("InUDF----rk" + ByteBufferUtil.string(rowKey));
    } catch (CharacterCodingException ex) {
        Logger.getLogger(UDFCasMapJoin.class.getName()).log(Level.SEVERE, null, ex);
    }
    ColumnFamilyResult<ByteBuffer, ByteBuffer> res = cft.queryColumns(rowKey);
    byte[] columnRes = res.getByteArray(column);
    System.out.println("InUDF----cr" + new String(columnRes));
    ByteArrayRef bar = new ByteArrayRef();
    bar.setData(columnRes);
    return bar;

}

---

From source file:com.kylinolap.job.hadoop.cardinality.ColumnCardinalityReducer.java

License:Apache License

@Override
public void reduce(IntWritable key, Iterable<BytesWritable> values, Context context)
        throws IOException, InterruptedException {
    for (BytesWritable v : values) {
        int skey = key.get();
        ByteBuffer buffer = ByteBuffer.wrap(v.getBytes());
        HyperLogLogPlusCounter hll = new HyperLogLogPlusCounter();
        hll.readRegisters(buffer);//  w w  w  . ja  v a 2 s . c om
        getHllc(skey).merge(hll);
        hll.clear();
    }
}

---

From source file:com.kylinolap.job.tools.ColumnCardinalityMapperTest.java

License:Apache License

@SuppressWarnings({ "unchecked" })
@Test//  w  ww .  ja  v  a2 s.co  m
@Ignore
public void testMapperOn177() throws IOException {
    mapDriver.clearInput();
    File file = new File("src/test/resources/data/test_cal_dt/part-r-00000");
    FileReader reader = new FileReader(file);
    BufferedReader breader = new BufferedReader(reader);
    String s = breader.readLine();
    int i = 0;
    while (s != null) {
        LongWritable inputKey = new LongWritable(i++);
        mapDriver.addInput(inputKey, new Text(s));
        s = breader.readLine();
    }
    // breader.close();
    mapDriver.getConfiguration().set(HiveColumnCardinalityJob.KEY_INPUT_DELIM, "\20");
    List<Pair<IntWritable, BytesWritable>> result = mapDriver.run();
    breader.close();
    assertEquals(9, result.size());

    int key1 = result.get(0).getFirst().get();
    BytesWritable value1 = result.get(0).getSecond();
    byte[] bytes = value1.getBytes();
    HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter();
    hllc.readRegisters(ByteBuffer.wrap(bytes));
    assertTrue(key1 > 0);
    assertEquals(8, hllc.getCountEstimate());
}

---

From source file:com.kylinolap.job.tools.ColumnCardinalityMapperTest.java

License:Apache License

@SuppressWarnings("unchecked")
@Test//w  w w  . j  a  v  a 2s  . c  o  m
public void testMapperOnComma() throws IOException {
    mapDriver.clearInput();
    LongWritable inputKey1 = new LongWritable(1);
    LongWritable inputKey2 = new LongWritable(2);
    LongWritable inputKey3 = new LongWritable(3);
    LongWritable inputKey4 = new LongWritable(4);
    LongWritable inputKey5 = new LongWritable(5);
    LongWritable inputKey6 = new LongWritable(6);
    LongWritable inputKey7 = new LongWritable(7);

    mapDriver.addInput(inputKey1, new Text());
    mapDriver.addInput(inputKey2, new Text(strArr));
    mapDriver.addInput(inputKey3, new Text(strArr));
    mapDriver.addInput(inputKey4, new Text(strArr));
    mapDriver.addInput(inputKey5, new Text(strArr));
    mapDriver.addInput(inputKey6, new Text(strArr));
    mapDriver.addInput(inputKey7, new Text(strArr));

    List<Pair<IntWritable, BytesWritable>> result = mapDriver.run();

    assertEquals(9, result.size());

    int key1 = result.get(0).getFirst().get();
    BytesWritable value1 = result.get(0).getSecond();
    byte[] bytes = value1.getBytes();
    HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter();
    hllc.readRegisters(ByteBuffer.wrap(bytes));
    System.out.println("ab\177ab".length());
    assertTrue(key1 > 0);
    assertEquals(1, hllc.getCountEstimate());
}

---

From source file:com.liveramp.cascading_ext.Bytes.java

License:Apache License

public static byte[] getBytes(BytesWritable bw) {
    if (bw.getCapacity() == bw.getLength()) {
        return bw.getBytes();
    } else {//w w w. j  a v a  2  s. c o m
        return copyBytes(bw);
    }
}

---

From source file:com.liveramp.cascading_ext.Bytes.java

License:Apache License

public static byte[] copyBytes(BytesWritable bw) {
    byte[] ret = new byte[bw.getLength()];
    System.arraycopy(bw.getBytes(), 0, ret, 0, bw.getLength());
    return ret;/*from   w  ww  .  j av a2  s .c o  m*/
}

---

From source file:com.liveramp.hank.hadoop.KeyAndPartitionWritable.java

License:Apache License

public KeyAndPartitionWritable(Partitioner partitioner, int numPartitions, BytesWritable key) {
    this.key = key;
    int partition = partitioner.partition(ByteBuffer.wrap(key.getBytes(), 0, key.getLength()), numPartitions);
    this.partition = new IntWritable(partition);
}

---

From source file:com.m6d.hive.protobuf.LongTest.java

License:Apache License

public void testWriteReadProto() throws Exception {
    Path p = new Path(this.ROOT_DIR, "reallybigfile2");

    SequenceFile.Writer w = SequenceFile.createWriter(this.getFileSystem(), new Configuration(), p,
            BytesWritable.class, BytesWritable.class, SequenceFile.CompressionType.BLOCK);

    long startLoad = System.currentTimeMillis();
    int toLoad = load;
    for (int i = 0; i < toLoad; i++) {
        Person.Builder bbuild = Person.newBuilder();
        Person ed = bbuild.setEmail(randomString()).setName(randomString()).setId(randomInt())
                .setHobby(Hobby.newBuilder().setName(randomString())).build();
        Person bo = bbuild.setEmail(randomString()).setName(randomString()).setId(randomInt())
                .setHobby(Hobby.newBuilder().setName(randomString())).build();

        BytesWritable key = new BytesWritable();
        BytesWritable value = new BytesWritable();
        ByteArrayOutputStream s = new ByteArrayOutputStream();
        ed.writeTo(s);//  w w w  . ja va2 s.  c  om

        ByteArrayOutputStream t = new ByteArrayOutputStream();
        bo.writeTo(t);

        key.set(s.toByteArray(), 0, s.size());
        value.set(t.toByteArray(), 0, t.size());
        w.append(key, value);
    }
    w.close();

    long start = System.currentTimeMillis();
    SequenceFile.Reader r = new SequenceFile.Reader(this.getFileSystem(), p, this.createJobConf());
    BytesWritable readkey = new BytesWritable();
    BytesWritable readval = new BytesWritable();
    while (r.next(readkey, readval)) {
        byte[] c = new byte[readkey.getLength()];
        System.arraycopy(readkey.getBytes(), 0, c, 0, readkey.getLength());
        Person.parseFrom(c);

        byte[] d = new byte[readval.getLength()];
        System.arraycopy(readval.getBytes(), 0, d, 0, readval.getLength());
        Person.parseFrom(d);
    }
    long end = System.currentTimeMillis();

    System.out.println("reading proto took" + (end - start));
    r.close();
}

---

From source file:com.m6d.hive.protobuf.ProtobufDeserializer.java

License:Apache License

@Override
public Object deserialize(Writable field) throws SerDeException {
    //if (!(field instanceof Pair)) {
    //  throw new SerDeException("Writable was not an instance of Pair. It was " + field.getClass());
    //}/*w w w.  j a va 2s .  c  o m*/
    BytesWritable key = null;
    BytesWritable value = null;
    if (field instanceof Pair) {
        Pair p = (Pair) field;
        key = (BytesWritable) p.getKey();
        value = (BytesWritable) p.getValue();
    } else if (field instanceof BytesWritable) {
        parseFrom = null;
        value = (BytesWritable) field;
    }
    Message parsedResult = null;
    Message vparsedResult = null;
    try {
        if (parseFrom != null) {
            byte[] b = new byte[key.getLength()];
            System.arraycopy(key.getBytes(), 0, b, 0, key.getLength());
            parsedResult = (Message) parseFrom.invoke(null, b);
        }
        if (vparseFrom != null) {
            byte[] c = new byte[value.getLength()];
            System.arraycopy(value.getBytes(), 0, c, 0, value.getLength());
            vparsedResult = (Message) vparseFrom.invoke(null, c);
        }
    } catch (IllegalAccessException ex) {
        throw new SerDeException(ex.getMessage(), ex);
    } catch (IllegalArgumentException ex) {
        throw new SerDeException(ex.getMessage(), ex);
    } catch (InvocationTargetException ex) {
        throw new SerDeException(ex.getMessage(), ex);
    }

    row.clear();
    keyRow.clear();
    if (parseFrom != null) {
        try {
            this.matchProtoToRow(parsedResult, keyRow, keyOIs, keyColumnNames);
        } catch (Exception ex) {
            throw new SerDeException(ex);
        }
        row.add(keyRow);
    } else {
        row.add(null);
    }
    valueRow.clear();
    if (vparseFrom != null) {
        try {
            this.matchProtoToRow(vparsedResult, valueRow, valueOIs, valueColumnNames);
        } catch (Exception ex) {
            throw new SerDeException(ex);
        }
        row.add(valueRow);
    } else {
        row.add(null);
    }
    return row;
}

---

From source file:com.m6d.hive.protobuf.TestKVAsSeq.java

License:Apache License

@Ignore
public void testRank() throws Exception {
    Path p = new Path(this.ROOT_DIR, "kv");
    SequenceFile.Writer w = SequenceFile.createWriter(this.getFileSystem(), new Configuration(), p,
            BytesWritable.class, BytesWritable.class);
    Person.Builder bbuild = Person.newBuilder();
    Person ed = bbuild.setEmail("ed@email.com").setName("ed").setId(1).build();

    Assert.assertEquals("ed", ed.getName());

    BytesWritable key = new BytesWritable();
    key.set("ed".getBytes(), 0, 2);
    BytesWritable value = new BytesWritable();
    ByteArrayOutputStream s = new ByteArrayOutputStream();
    ed.writeTo(s);/*from   ww w. ja v a 2s .  c  om*/

    Person g = Person.parseFrom(s.toByteArray());
    int bsize = s.toByteArray().length;
    Assert.assertEquals(g.getName(), ed.getName());

    value.set(s.toByteArray(), 0, s.size());

    w.append(key, value);
    w.close();

    //DataOutputBuffer itkey = new DataOutputBuffer();
    //SequenceFile.Reader r = new SequenceFile.Reader(this.getFileSystem(), p, this.createJobConf());
    //SequenceFile.ValueBytes v = r.createValueBytes();

    //while (r.nextRaw(itkey, v) != -1){
    //  v.writeUncompressedBytes(itkey);
    //  Person other = Person.parseFrom(itkey.getData());
    //  Assert.assertEquals(ed.getName(), other.getName());
    //}

    SequenceFile.Reader r = new SequenceFile.Reader(this.getFileSystem(), p, this.createJobConf());
    BytesWritable itKey = new BytesWritable();
    BytesWritable itValue = new BytesWritable();
    r.next(itKey, itValue);
    System.out.println(itValue);

    Assert.assertEquals(bsize, itValue.getLength());
    Assert.assertEquals(bsize, itValue.getSize());
    //Assert.assertEquals(bsize, itValue.getCapacity());
    for (int i = 0; i < bsize; i++) {
        Assert.assertEquals(itValue.getBytes()[i], s.toByteArray()[i]);
    }
    //Assert.assertEquals(itValue.getBytes(), s.toByteArray());
    byte[] whatIWant = new byte[itValue.getLength()];
    System.arraycopy(itValue.getBytes(), 0, whatIWant, 0, itValue.getLength());
    Person other = Person.parseFrom(whatIWant);
    Assert.assertEquals(ed.getName(), other.getName());
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• »