Example usage for org.apache.hadoop.io BytesWritable set

Introduction

In this page you can find the example usage for org.apache.hadoop.io BytesWritable set.

Prototype

public void set(byte[] newData, int offset, int length) 

Source Link

Document

Set the value to a copy of the given byte range

Usage

From source file:com.m6d.hive.protobuf.LongTest.java

License:Apache License

public void testWriteReadProto() throws Exception {
    Path p = new Path(this.ROOT_DIR, "reallybigfile2");

    SequenceFile.Writer w = SequenceFile.createWriter(this.getFileSystem(), new Configuration(), p,
            BytesWritable.class, BytesWritable.class, SequenceFile.CompressionType.BLOCK);

    long startLoad = System.currentTimeMillis();
    int toLoad = load;
    for (int i = 0; i < toLoad; i++) {
        Person.Builder bbuild = Person.newBuilder();
        Person ed = bbuild.setEmail(randomString()).setName(randomString()).setId(randomInt())
                .setHobby(Hobby.newBuilder().setName(randomString())).build();
        Person bo = bbuild.setEmail(randomString()).setName(randomString()).setId(randomInt())
                .setHobby(Hobby.newBuilder().setName(randomString())).build();

        BytesWritable key = new BytesWritable();
        BytesWritable value = new BytesWritable();
        ByteArrayOutputStream s = new ByteArrayOutputStream();
        ed.writeTo(s);/*from  ww  w  .jav a  2 s.  c  o  m*/

        ByteArrayOutputStream t = new ByteArrayOutputStream();
        bo.writeTo(t);

        key.set(s.toByteArray(), 0, s.size());
        value.set(t.toByteArray(), 0, t.size());
        w.append(key, value);
    }
    w.close();

    long start = System.currentTimeMillis();
    SequenceFile.Reader r = new SequenceFile.Reader(this.getFileSystem(), p, this.createJobConf());
    BytesWritable readkey = new BytesWritable();
    BytesWritable readval = new BytesWritable();
    while (r.next(readkey, readval)) {
        byte[] c = new byte[readkey.getLength()];
        System.arraycopy(readkey.getBytes(), 0, c, 0, readkey.getLength());
        Person.parseFrom(c);

        byte[] d = new byte[readval.getLength()];
        System.arraycopy(readval.getBytes(), 0, d, 0, readval.getLength());
        Person.parseFrom(d);
    }
    long end = System.currentTimeMillis();

    System.out.println("reading proto took" + (end - start));
    r.close();
}

---

From source file:com.m6d.hive.protobuf.LongTest.java

License:Apache License

public void testBigProto() throws Exception {
    Path p = new Path(this.ROOT_DIR, "reallybigfile");

    SequenceFile.Writer w = SequenceFile.createWriter(this.getFileSystem(), new Configuration(), p,
            BytesWritable.class, BytesWritable.class, SequenceFile.CompressionType.BLOCK);

    long startLoad = System.currentTimeMillis();
    int toLoad = load;
    for (int i = 0; i < toLoad; i++) {
        Person.Builder bbuild = Person.newBuilder();
        //  Person ed = bbuild.setEmail("ed@email.com").setName("ed").
        //          setId(i).setHobby(Hobby.newBuilder().setName("java")).build();

        Person ed = bbuild.setEmail(randomString()).setName(randomString()).setId(randomInt())
                .setHobby(Hobby.newBuilder().setName(randomString())).build();

        // Person bo = bbuild.setEmail("bo@email.com").setName("bo").
        //        setId(i).setHobby(Hobby.newBuilder().setName("bball")).build();
        Person bo = bbuild.setEmail(randomString()).setName(randomString()).setId(randomInt())
                .setHobby(Hobby.newBuilder().setName(randomString())).build();

        BytesWritable key = new BytesWritable();
        BytesWritable value = new BytesWritable();
        ByteArrayOutputStream s = new ByteArrayOutputStream();
        ed.writeTo(s);/*w  w  w  . ja va  2  s .c om*/

        ByteArrayOutputStream t = new ByteArrayOutputStream();
        bo.writeTo(t);

        key.set(s.toByteArray(), 0, s.size());
        value.set(t.toByteArray(), 0, t.size());
        w.append(key, value);
    }
    w.close();
    System.out.println("len " + this.getFileSystem().getFileStatus(p).getLen());
    long endLoad = System.currentTimeMillis();
    System.out.println((endLoad - startLoad) + " time taken loading");

    String jarFile;
    jarFile = KVAsVSeqFileBinaryInputFormat.class.getProtectionDomain().getCodeSource().getLocation().getFile();
    client.execute("add jar " + jarFile);
    client.execute("set hive.aux.jars.path=file:///" + jarFile);

    client.execute("create table  bigproto   " + " ROW FORMAT SERDE '" + ProtobufDeserializer.class.getName()
            + "'" + " WITH SERDEPROPERTIES ('KEY_SERIALIZE_CLASS'='" + Ex.Person.class.getName()
            + "','VALUE_SERIALIZE_CLASS'='" + Ex.Person.class.getName() + "'   )" + " STORED AS INPUTFORMAT '"
            + KVAsVSeqFileBinaryInputFormat.class.getName() + "'" + " OUTPUTFORMAT '"
            + SequenceFileOutputFormat.class.getName() + "'");

    client.execute("load data local inpath '" + p.toString() + "' into table bigproto");

    long startQuery = System.currentTimeMillis();
    client.execute("SELECT count(1) FROM bigproto");
    List<String> results = client.fetchAll();
    Assert.assertEquals(toLoad + "", results.get(0));
    long endQuery = System.currentTimeMillis();

    System.out.println((endQuery - startQuery) + " Proto Query time taken");
    client.execute("drop table bigproto");

}

---

From source file:com.m6d.hive.protobuf.TestKVAsSeq.java

License:Apache License

@Ignore
public void testRank() throws Exception {
    Path p = new Path(this.ROOT_DIR, "kv");
    SequenceFile.Writer w = SequenceFile.createWriter(this.getFileSystem(), new Configuration(), p,
            BytesWritable.class, BytesWritable.class);
    Person.Builder bbuild = Person.newBuilder();
    Person ed = bbuild.setEmail("ed@email.com").setName("ed").setId(1).build();

    Assert.assertEquals("ed", ed.getName());

    BytesWritable key = new BytesWritable();
    key.set("ed".getBytes(), 0, 2);
    BytesWritable value = new BytesWritable();
    ByteArrayOutputStream s = new ByteArrayOutputStream();
    ed.writeTo(s);/*from w w  w  . j a v a  2s  . c o  m*/

    Person g = Person.parseFrom(s.toByteArray());
    int bsize = s.toByteArray().length;
    Assert.assertEquals(g.getName(), ed.getName());

    value.set(s.toByteArray(), 0, s.size());

    w.append(key, value);
    w.close();

    //DataOutputBuffer itkey = new DataOutputBuffer();
    //SequenceFile.Reader r = new SequenceFile.Reader(this.getFileSystem(), p, this.createJobConf());
    //SequenceFile.ValueBytes v = r.createValueBytes();

    //while (r.nextRaw(itkey, v) != -1){
    //  v.writeUncompressedBytes(itkey);
    //  Person other = Person.parseFrom(itkey.getData());
    //  Assert.assertEquals(ed.getName(), other.getName());
    //}

    SequenceFile.Reader r = new SequenceFile.Reader(this.getFileSystem(), p, this.createJobConf());
    BytesWritable itKey = new BytesWritable();
    BytesWritable itValue = new BytesWritable();
    r.next(itKey, itValue);
    System.out.println(itValue);

    Assert.assertEquals(bsize, itValue.getLength());
    Assert.assertEquals(bsize, itValue.getSize());
    //Assert.assertEquals(bsize, itValue.getCapacity());
    for (int i = 0; i < bsize; i++) {
        Assert.assertEquals(itValue.getBytes()[i], s.toByteArray()[i]);
    }
    //Assert.assertEquals(itValue.getBytes(), s.toByteArray());
    byte[] whatIWant = new byte[itValue.getLength()];
    System.arraycopy(itValue.getBytes(), 0, whatIWant, 0, itValue.getLength());
    Person other = Person.parseFrom(whatIWant);
    Assert.assertEquals(ed.getName(), other.getName());
}

---

From source file:com.m6d.hive.protobuf.TestProto.java

License:Apache License

public void testAlist() throws Exception {

    AThing.Builder aThingBuild = AThing.newBuilder();
    AThing aThing = aThingBuild.addLuckynumbers(7).addLuckynumbers(4).addToys("car").build();
    AList.Builder aListBuild = AList.newBuilder();
    AList aList = aListBuild.addAge(2).addAge(3).addThings(aThing).build();

    BytesWritable key = new BytesWritable();
    BytesWritable value = new BytesWritable();

    ByteArrayOutputStream s = new ByteArrayOutputStream();
    aList.writeTo(s);/*  w  w w  .j av a 2s  .  c  om*/
    key.set(s.toByteArray(), 0, s.toByteArray().length);
    value.set(s.toByteArray(), 0, s.toByteArray().length);

    Pair p = new Pair();
    p.setKey(key);
    p.setValue(value);

    ProtobufDeserializer de = new ProtobufDeserializer();
    de.keyClass = Ex.AList.class;
    de.valueClass = Ex.AList.class;
    de.parseFrom = de.keyClass.getMethod("parseFrom", de.byteArrayParameters);
    de.vparseFrom = de.valueClass.getMethod("parseFrom", de.byteArrayParameters);
    Object returnObj = de.deserialize(p);
    ArrayList keyVal = (ArrayList) returnObj;
    System.out.println(keyVal);
    List keyObj = (List) keyVal.get(0);
    List valObj = (List) keyVal.get(1);

}

---

From source file:com.m6d.hive.protobuf.TestProto.java

License:Apache License

public void testDeserialize() throws Exception {

    Person.Builder bbuild = Person.newBuilder();
    bbuild.setEmail("ed@email.com").setName("ed").setId(1);
    bbuild.setHobby(Hobby.newBuilder().setName("fishing"));
    Person ed = bbuild.build();//from  w  w  w .ja  v a 2  s  .c  om

    BytesWritable key = new BytesWritable();
    BytesWritable value = new BytesWritable();

    ByteArrayOutputStream s = new ByteArrayOutputStream();
    ed.writeTo(s);
    key.set(s.toByteArray(), 0, s.toByteArray().length);
    value.set(s.toByteArray(), 0, s.toByteArray().length);

    Pair p = new Pair();
    p.setKey(key);
    p.setValue(value);

    ProtobufDeserializer de = new ProtobufDeserializer();
    de.keyClass = Ex.Person.class;
    de.valueClass = Ex.Person.class;
    de.parseFrom = de.keyClass.getMethod("parseFrom", de.byteArrayParameters);
    de.vparseFrom = de.valueClass.getMethod("parseFrom", de.byteArrayParameters);

    Object result = de.deserialize(p);

}

---

From source file:com.m6d.hive.protobuf.TestProto.java

License:Apache License

public void testWithHas() throws Exception {
    Path p = new Path(ROOT_DIR, "nulstuff");
    SequenceFile.Writer w = SequenceFile.createWriter(this.getFileSystem(), new Configuration(), p,
            BytesWritable.class, BytesWritable.class);

    Ex.Person.Builder personBuilder = Ex.Person.newBuilder();
    Ex.AddressBook.Builder addressBuilder = Ex.AddressBook.newBuilder();

    //here we set pete's name but not his optional email

    Hobby.Builder h = Hobby.newBuilder();
    h.setName("tanning");
    Person pete = personBuilder.setId(4).setName("pete").build();
    Person stan = personBuilder.clear().setEmail("stan@prodigy.net").setId(10).setName("stan")
            .setHobby(h.build()).build();
    AddressBook b = addressBuilder.addPerson(pete).addPerson(stan).build();

    BytesWritable key = new BytesWritable();
    BytesWritable value = new BytesWritable();
    ByteArrayOutputStream s = new ByteArrayOutputStream();
    b.writeTo(s);// w w  w.  j  av a  2 s  .c o m
    ByteArrayOutputStream t = new ByteArrayOutputStream();
    b.writeTo(t);
    key.set(s.toByteArray(), 0, s.size());
    value.set(t.toByteArray(), 0, t.size());
    w.append(key, value);

    w.close();

    String jarFile;
    jarFile = KVAsVSeqFileBinaryInputFormat.class.getProtectionDomain().getCodeSource().getLocation().getFile();

    System.out.println("set hive.aux.jars.path=file:///" + jarFile);

    client.execute("add jar " + jarFile);
    client.execute("set hive.aux.jars.path=file:///" + jarFile);

    client.execute("create table  nulstuf   " + " ROW FORMAT SERDE '" + ProtobufDeserializer.class.getName()
            + "'" + " WITH SERDEPROPERTIES ('KEY_SERIALIZE_CLASS'='" + Ex.AddressBook.class.getName()
            + "','VALUE_SERIALIZE_CLASS'='" + Ex.AddressBook.class.getName() + "'   )"
            + " STORED AS INPUTFORMAT '" + KVAsVSeqFileBinaryInputFormat.class.getName() + "'"
            + " OUTPUTFORMAT '" + SequenceFileOutputFormat.class.getName() + "'");

    client.execute("load data local inpath '" + p.toString() + "' into table nulstuf");
    client.execute("SELECT key from nulstuf ");

    List<String> results = client.fetchAll();
    //String expectedResuls = "{\"personcount\":2,\"personlist\":[{\"email\":\"stan@prodigy.net\",\"hobby\":{\"name\":\"\"},\"id\":10,\"name\":\"stan\"},{\"email\":\"stan@prodigy.net\",\"hobby\":{\"name\":\"\"},\"id\":10,\"name\":\"stan\"}]}";

    String expectedResuls = "{\"personcount\":2,\"personlist\":[{\"email\":null,\"hobby\":null,\"id\":4,\"name\":\"pete\"},{\"email\":\"stan@prodigy.net\",\"hobby\":{\"name\":\"tanning\"},\"id\":10,\"name\":\"stan\"}]}";
    //String expectedResuls= "";
    System.out.println(results.get(0));
    Assert.assertEquals(expectedResuls, results.get(0));

    client.execute("Drop table nulstuf");

}

---

From source file:com.m6d.hive.protobuf.TestProto.java

License:Apache License

public void testListWithAddressBook() throws Exception {
    Path p = new Path(ROOT_DIR, "addressbook");
    SequenceFile.Writer w = SequenceFile.createWriter(this.getFileSystem(), new Configuration(), p,
            BytesWritable.class, BytesWritable.class);

    Ex.Person.Builder personBuilder = Ex.Person.newBuilder();
    Ex.AddressBook.Builder addressBuilder = Ex.AddressBook.newBuilder();

    Person pete = personBuilder.setEmail("pete@aol.com").setId(4).setName("pete").build();
    Person stan = personBuilder.clear().setEmail("stan@prodigy.net").setId(10).setName("stan").build();
    AddressBook b = addressBuilder.addPerson(pete).addPerson(stan).build();

    BytesWritable key = new BytesWritable();
    BytesWritable value = new BytesWritable();
    ByteArrayOutputStream s = new ByteArrayOutputStream();
    b.writeTo(s);/* w w  w.j a  v a  2  s  .co m*/
    ByteArrayOutputStream t = new ByteArrayOutputStream();
    b.writeTo(t);
    key.set(s.toByteArray(), 0, s.size());
    value.set(t.toByteArray(), 0, t.size());
    w.append(key, value);

    w.close();

    String jarFile;
    jarFile = KVAsVSeqFileBinaryInputFormat.class.getProtectionDomain().getCodeSource().getLocation().getFile();

    System.out.println("set hive.aux.jars.path=file:///" + jarFile);

    client.execute("add jar " + jarFile);
    client.execute("set hive.aux.jars.path=file:///" + jarFile);

    client.execute("create table  address   " + " ROW FORMAT SERDE '" + ProtobufDeserializer.class.getName()
            + "'" + " WITH SERDEPROPERTIES ('KEY_SERIALIZE_CLASS'='" + Ex.AddressBook.class.getName()
            + "','VALUE_SERIALIZE_CLASS'='" + Ex.AddressBook.class.getName() + "'   )"
            + " STORED AS INPUTFORMAT '" + KVAsVSeqFileBinaryInputFormat.class.getName() + "'"
            + " OUTPUTFORMAT '" + SequenceFileOutputFormat.class.getName() + "'");

    client.execute("load data local inpath '" + p.toString() + "' into table address");
    client.execute("SELECT key from address ");

    List<String> results = client.fetchAll();
    //String expectedResuls = "{\"personcount\":2,\"personlist\":[{\"email\":\"stan@prodigy.net\",\"hobby\":{\"name\":\"\"},\"id\":10,\"name\":\"stan\"},{\"email\":\"stan@prodigy.net\",\"hobby\":{\"name\":\"\"},\"id\":10,\"name\":\"stan\"}]}";

    String expectedResuls = "{\"personcount\":2,\"personlist\":[{\"email\":\"pete@aol.com\",\"hobby\":null,\"id\":4,\"name\":\"pete\"},{\"email\":\"stan@prodigy.net\",\"hobby\":null,\"id\":10,\"name\":\"stan\"}]}";
    //String expectedResuls= "";
    Assert.assertEquals(expectedResuls, results.get(0));

}

---

From source file:com.m6d.hive.protobuf.TestProto.java

License:Apache License

public void testBigTable() throws Exception {
    Path p = new Path(this.ROOT_DIR, "bigfile");
    SequenceFile.Writer w = SequenceFile.createWriter(this.getFileSystem(), new Configuration(), p,
            BytesWritable.class, BytesWritable.class);

    for (int i = 0; i < 100; i++) {
        Person.Builder bbuild = Person.newBuilder();
        Person ed = bbuild.setEmail("ed@email.com").setName("ed").setId(i)
                .setHobby(Hobby.newBuilder().setName("java")).build();

        Person bo = bbuild.setEmail("bo@email.com").setName("bo").setId(i)
                .setHobby(Hobby.newBuilder().setName("bball")).build();

        BytesWritable key = new BytesWritable();
        BytesWritable value = new BytesWritable();
        ByteArrayOutputStream s = new ByteArrayOutputStream();
        ed.writeTo(s);//from ww  w.  j a v  a 2s . c o m

        ByteArrayOutputStream t = new ByteArrayOutputStream();
        bo.writeTo(t);

        key.set(s.toByteArray(), 0, s.size());
        value.set(t.toByteArray(), 0, t.size());
        w.append(key, value);
    }
    w.close();

    String jarFile;
    jarFile = KVAsVSeqFileBinaryInputFormat.class.getProtectionDomain().getCodeSource().getLocation().getFile();

    System.out.println("set hive.aux.jars.path=file:///" + jarFile);

    client.execute("add jar " + jarFile);
    client.execute("set hive.aux.jars.path=file:///" + jarFile);

    client.execute("create table  bigproto   " + " ROW FORMAT SERDE '" + ProtobufDeserializer.class.getName()
            + "'" + " WITH SERDEPROPERTIES ('KEY_SERIALIZE_CLASS'='" + Ex.Person.class.getName()
            + "','VALUE_SERIALIZE_CLASS'='" + Ex.Person.class.getName() + "'   )" + " STORED AS INPUTFORMAT '"
            + KVAsVSeqFileBinaryInputFormat.class.getName() + "'" + " OUTPUTFORMAT '"
            + SequenceFileOutputFormat.class.getName() + "'");

    client.execute("load data local inpath '" + p.toString() + "' into table bigproto");
    client.execute("SELECT key.id, key.name FROM bigproto where key.id< 10");

    List<String> results = client.fetchAll();

    Assert.assertEquals("0\ted", results.get(0));
    Assert.assertEquals("1\ted", results.get(1));
    Assert.assertEquals(10, results.size());
    client.execute("drop table bigproto");

}

---

From source file:com.m6d.hive.protobuf.TestProto.java

License:Apache License

public void testCreateAndReadTable() throws Exception {
    Path p = new Path(this.ROOT_DIR, "protofile");
    SequenceFile.Writer w = SequenceFile.createWriter(this.getFileSystem(), new Configuration(), p,
            BytesWritable.class, BytesWritable.class);

    Person.Builder bbuild = Person.newBuilder();
    Person ed = bbuild.setEmail("ed@email.com").setName("ed").setId(1)
            .setHobby(Hobby.newBuilder().setName("java")).build();

    Person bo = bbuild.setEmail("bo@email.com").setName("bo").setId(1)
            .setHobby(Hobby.newBuilder().setName("bball")).build();

    BytesWritable key = new BytesWritable();
    BytesWritable value = new BytesWritable();
    ByteArrayOutputStream s = new ByteArrayOutputStream();
    ed.writeTo(s);//from   w  ww.  ja  v a2  s.  c om

    ByteArrayOutputStream t = new ByteArrayOutputStream();
    bo.writeTo(t);

    key.set(s.toByteArray(), 0, s.size());
    value.set(t.toByteArray(), 0, t.size());
    w.append(key, value);
    w.close();

    client.execute("create table  protoperson   " + " ROW FORMAT SERDE '" + ProtobufDeserializer.class.getName()
            + "'" + " WITH SERDEPROPERTIES ('KEY_SERIALIZE_CLASS'='" + Ex.Person.class.getName()
            + "','VALUE_SERIALIZE_CLASS'='" + Ex.Person.class.getName() + "'   )" + " STORED AS INPUTFORMAT '"
            + KVAsVSeqFileBinaryInputFormat.class.getName() + "'" + " OUTPUTFORMAT '"
            + SequenceFileOutputFormat.class.getName() + "'");

    client.execute("load data local inpath '" + p.toString() + "' into table protoperson");
    client.execute("SELECT * FROM protoperson");

    List<String> results = client.fetchAll();
    //String expected = "{\"email\":\"ed@email.com\",\"hobby\":{\"name\":\"java\"},\"id\":1,\"name\":\"ed\"}\t";
    String expected = "null\t";
    expected = expected
            + "{\"email\":\"bo@email.com\",\"hobby\":{\"name\":\"bball\"},\"id\":1,\"name\":\"bo\"}";
    Assert.assertEquals(expected, results.get(0));
    client.execute("drop table protoperson");

}

---

From source file:com.m6d.hive.protobuf.TestProto.java

License:Apache License

public void testCreateAndReadTableList() throws Exception {
    Path p = new Path(this.ROOT_DIR, "protolistfile");
    SequenceFile.Writer w = SequenceFile.createWriter(this.getFileSystem(), new Configuration(), p,
            BytesWritable.class, BytesWritable.class);

    AThing.Builder aThingBuild = AThing.newBuilder();
    AThing aThing = aThingBuild.addLuckynumbers(7).addLuckynumbers(4).addToys("car").build();
    AList.Builder aListBuild = AList.newBuilder();
    AList aList = aListBuild.addAge(2).addAge(3).addThings(aThing).build();

    BytesWritable key = new BytesWritable();
    BytesWritable value = new BytesWritable();
    ByteArrayOutputStream s = new ByteArrayOutputStream();
    aList.writeTo(s);/* ww w .  j  a v a 2s  .  c  o  m*/

    ByteArrayOutputStream t = new ByteArrayOutputStream();
    aList.writeTo(t);

    key.set(s.toByteArray(), 0, s.size());
    value.set(t.toByteArray(), 0, t.size());
    w.append(key, value);
    w.close();

    String jarFile;
    jarFile = KVAsVSeqFileBinaryInputFormat.class.getProtectionDomain().getCodeSource().getLocation().getFile();

    System.out.println("set hive.aux.jars.path=file:///" + jarFile);

    client.execute("add jar " + jarFile);
    client.execute("set hive.aux.jars.path=file:///" + jarFile);

    client.execute("create table  tablewithlist   " + " ROW FORMAT SERDE '"
            + ProtobufDeserializer.class.getName() + "'" + " WITH SERDEPROPERTIES ('KEY_SERIALIZE_CLASS'='"
            + Ex.AList.class.getName() + "','VALUE_SERIALIZE_CLASS'='" + Ex.AList.class.getName() + "'   )"
            + " STORED AS INPUTFORMAT '" + KVAsVSeqFileBinaryInputFormat.class.getName() + "'"
            + " OUTPUTFORMAT '" + SequenceFileOutputFormat.class.getName() + "'");

    client.execute("load data local inpath '" + p.toString() + "' into table tablewithlist");
    client.execute("SELECT key FROM tablewithlist");

    List<String> results = client.fetchAll();
    String expected = "";
    expected = "{\"agecount\":2,\"agelist\":[2,3],\"thingscount\":1,\"thingslist\":[{\"luckynumberscount\":2,\"luckynumberslist\":[7,4],\"toyscount\":1,\"toyslist\":[\"car\"]}]}";
    Assert.assertEquals(expected, results.get(0));
    client.execute("drop table tablewithlist");

}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• »