Example usage for org.apache.thrift.transport TMemoryBuffer write

Introduction

In this page you can find the example usage for org.apache.thrift.transport TMemoryBuffer write.

Prototype

@Override
    public void write(byte[] buf, int off, int len) 

Source Link

Usage

From source file:org.apache.hadoop.hive.serde2.TestTCTLSeparatedProtocol.java

License:Apache License

public void testReads() throws Exception {
    TMemoryBuffer trans = new TMemoryBuffer(1024);
    String foo = "Hello";
    String bar = "World!";

    String key = "22";
    String value = "TheValue";
    String key2 = "24";
    String value2 = "TheValueAgain";

    byte columnSeparator[] = { 1 };
    byte elementSeparator[] = { 2 };
    byte kvSeparator[] = { 3 };

    trans.write(foo.getBytes(), 0, foo.getBytes().length);
    trans.write(columnSeparator, 0, 1);/*from   www .j  a v a 2  s .  c o  m*/

    trans.write(columnSeparator, 0, 1);

    trans.write(bar.getBytes(), 0, bar.getBytes().length);
    trans.write(columnSeparator, 0, 1);

    trans.write(key.getBytes(), 0, key.getBytes().length);
    trans.write(kvSeparator, 0, 1);
    trans.write(value.getBytes(), 0, value.getBytes().length);
    trans.write(elementSeparator, 0, 1);

    trans.write(key2.getBytes(), 0, key2.getBytes().length);
    trans.write(kvSeparator, 0, 1);
    trans.write(value2.getBytes(), 0, value2.getBytes().length);

    trans.flush();

    // use 3 as the row buffer size to force lots of re-buffering.
    TCTLSeparatedProtocol prot = new TCTLSeparatedProtocol(trans, 1024);
    prot.initialize(new Configuration(), new Properties());

    prot.readStructBegin();

    prot.readFieldBegin();
    String hello = prot.readString();
    prot.readFieldEnd();

    assertTrue(hello.equals(foo));

    prot.readFieldBegin();
    assertTrue(prot.readString().equals(""));
    prot.readFieldEnd();

    prot.readFieldBegin();
    assertTrue(prot.readString().equals(bar));
    prot.readFieldEnd();

    prot.readFieldBegin();
    TMap mapHeader = prot.readMapBegin();
    assertTrue(mapHeader.size == 2);

    assertTrue(prot.readI32() == 22);
    assertTrue(prot.readString().equals(value));
    assertTrue(prot.readI32() == 24);
    assertTrue(prot.readString().equals(value2));
    prot.readMapEnd();
    prot.readFieldEnd();

    prot.readFieldBegin();
    hello = prot.readString();
    prot.readFieldEnd();
    assertNull(hello);

    prot.readStructEnd();
}

---

From source file:org.apache.hadoop.hive.serde2.TestTCTLSeparatedProtocol.java

License:Apache License

public void testWrites() throws Exception {
    TMemoryBuffer trans = new TMemoryBuffer(1024);
    TCTLSeparatedProtocol prot = new TCTLSeparatedProtocol(trans, 1024);

    prot.writeStructBegin(new TStruct());
    prot.writeFieldBegin(new TField());
    prot.writeI32(100);//from  www .ja  va 2 s . c om
    prot.writeFieldEnd();

    prot.writeFieldBegin(new TField());
    prot.writeListBegin(new TList());
    prot.writeDouble(348.55);
    prot.writeDouble(234.22);
    prot.writeListEnd();
    prot.writeFieldEnd();

    prot.writeFieldBegin(new TField());
    prot.writeString("hello world!");
    prot.writeFieldEnd();

    prot.writeFieldBegin(new TField());
    prot.writeMapBegin(new TMap());
    prot.writeString("key1");
    prot.writeString("val1");
    prot.writeString("key2");
    prot.writeString("val2");
    prot.writeString("key3");
    prot.writeString("val3");
    prot.writeMapEnd();
    prot.writeFieldEnd();

    prot.writeFieldBegin(new TField());
    prot.writeListBegin(new TList());
    prot.writeString("elem1");
    prot.writeString("elem2");
    prot.writeListEnd();
    prot.writeFieldEnd();

    prot.writeFieldBegin(new TField());
    prot.writeString("bye!");
    prot.writeFieldEnd();

    prot.writeStructEnd();
    trans.flush();
    byte[] b = new byte[3 * 1024];
    int len = trans.read(b, 0, b.length);
    String test = new String(b, 0, len);

    String testRef = "100
348.55234.22
hello world!
key1val1key2val2key3val3
elem1elem2
bye!";
    assertTrue(test.equals(testRef));

    trans = new TMemoryBuffer(1023);
    trans.write(b, 0, len);

    //
    // read back!
    //

    prot = new TCTLSeparatedProtocol(trans, 10);
    prot.initialize(new Configuration(), new Properties());

    // 100 is the start
    prot.readStructBegin();
    prot.readFieldBegin();
    assertTrue(prot.readI32() == 100);
    prot.readFieldEnd();

    // let's see if doubles work ok
    prot.readFieldBegin();
    TList l = prot.readListBegin();
    assertTrue(l.size == 2);
    assertTrue(prot.readDouble() == 348.55);
    assertTrue(prot.readDouble() == 234.22);
    prot.readListEnd();
    prot.readFieldEnd();

    // nice message
    prot.readFieldBegin();
    assertTrue(prot.readString().equals("hello world!"));
    prot.readFieldEnd();

    // 3 element map
    prot.readFieldBegin();
    TMap m = prot.readMapBegin();
    assertTrue(m.size == 3);
    assertTrue(prot.readString().equals("key1"));
    assertTrue(prot.readString().equals("val1"));
    assertTrue(prot.readString().equals("key2"));
    assertTrue(prot.readString().equals("val2"));
    assertTrue(prot.readString().equals("key3"));
    assertTrue(prot.readString().equals("val3"));
    prot.readMapEnd();
    prot.readFieldEnd();

    // the 2 element list
    prot.readFieldBegin();
    l = prot.readListBegin();
    assertTrue(l.size == 2);
    assertTrue(prot.readString().equals("elem1"));
    assertTrue(prot.readString().equals("elem2"));
    prot.readListEnd();
    prot.readFieldEnd();

    // final string
    prot.readFieldBegin();
    assertTrue(prot.readString().equals("bye!"));
    prot.readFieldEnd();

    // shouldl return nulls at end
    prot.readFieldBegin();
    assertNull(prot.readString());
    prot.readFieldEnd();

    // shouldl return nulls at end
    prot.readFieldBegin();
    assertNull(prot.readString());
    prot.readFieldEnd();

    prot.readStructEnd();
}

---

From source file:org.apache.hadoop.hive.serde2.TestTCTLSeparatedProtocol.java

License:Apache License

public void testQuotedWrites() throws Exception {
    TMemoryBuffer trans = new TMemoryBuffer(4096);
    TCTLSeparatedProtocol prot = new TCTLSeparatedProtocol(trans, 4096);
    Properties schema = new Properties();
    schema.setProperty(serdeConstants.QUOTE_CHAR, "\"");
    schema.setProperty(serdeConstants.FIELD_DELIM, ",");
    prot.initialize(new Configuration(), schema);

    String testStr = "\"hello, world!\"";

    prot.writeStructBegin(new TStruct());

    prot.writeFieldBegin(new TField());
    prot.writeString(testStr);//from   ww  w .  ja  v  a 2s.  co  m
    prot.writeFieldEnd();

    prot.writeFieldBegin(new TField());
    prot.writeListBegin(new TList());
    prot.writeString("elem1");
    prot.writeString("elem2");
    prot.writeListEnd();
    prot.writeFieldEnd();

    prot.writeStructEnd();
    prot.writeString("\n");

    trans.flush();

    byte b[] = new byte[4096];
    int len = trans.read(b, 0, b.length);

    trans = new TMemoryBuffer(4096);
    trans.write(b, 0, len);
    prot = new TCTLSeparatedProtocol(trans, 1024);
    prot.initialize(new Configuration(), schema);

    prot.readStructBegin();
    prot.readFieldBegin();
    final String firstRead = prot.readString();
    prot.readFieldEnd();

    testStr = testStr.replace("\"", "");

    assertEquals(testStr, firstRead);

    // the 2 element list
    prot.readFieldBegin();
    TList l = prot.readListBegin();
    assertTrue(l.size == 2);
    assertTrue(prot.readString().equals("elem1"));
    assertTrue(prot.readString().equals("elem2"));
    prot.readListEnd();
    prot.readFieldEnd();

    // shouldl return nulls at end
    prot.readFieldBegin();
    assertNull(prot.readString());
    prot.readFieldEnd();

    // shouldl return nulls at end
    prot.readFieldBegin();
    assertNull(prot.readString());
    prot.readFieldEnd();

    prot.readStructEnd();
}

---

From source file:org.apache.hadoop.hive.serde2.TestTCTLSeparatedProtocol.java

License:Apache License

/**
 * Tests a sample apache log format. This is actually better done in general
 * with a more TRegexLike protocol, but for this case, TCTLSeparatedProtocol
 * can do it.//from   ww w.  j  ava 2  s.c om
 */
public void test1ApacheLogFormat() throws Exception {
    final String sample = "127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] \"GET /apache_pb.gif HTTP/1.0\" 200 2326";

    TMemoryBuffer trans = new TMemoryBuffer(4096);
    trans.write(sample.getBytes(), 0, sample.getBytes().length);
    trans.flush();

    TCTLSeparatedProtocol prot = new TCTLSeparatedProtocol(trans, 4096);
    Properties schema = new Properties();

    // this is a hacky way of doing the quotes since it will match any 2 of
    // these, so
    // "[ hello this is something to split [" would be considered to be quoted.
    schema.setProperty(serdeConstants.QUOTE_CHAR, "(\"|\\[|\\])");

    schema.setProperty(serdeConstants.FIELD_DELIM, " ");
    schema.setProperty(serdeConstants.SERIALIZATION_NULL_FORMAT, "-");
    prot.initialize(new Configuration(), schema);

    prot.readStructBegin();

    // ip address
    prot.readFieldBegin();
    final String ip = prot.readString();
    prot.readFieldEnd();

    assertEquals("127.0.0.1", ip);

    // identd
    prot.readFieldBegin();
    final String identd = prot.readString();
    prot.readFieldEnd();

    assertNull(identd);

    // user
    prot.readFieldBegin();
    final String user = prot.readString();
    prot.readFieldEnd();

    assertEquals("frank", user);

    // finishTime
    prot.readFieldBegin();
    final String finishTime = prot.readString();
    prot.readFieldEnd();

    assertEquals("10/Oct/2000:13:55:36 -0700", finishTime);

    // requestLine
    prot.readFieldBegin();
    final String requestLine = prot.readString();
    prot.readFieldEnd();

    assertEquals("GET /apache_pb.gif HTTP/1.0", requestLine);

    // returncode
    prot.readFieldBegin();
    final int returnCode = prot.readI32();
    prot.readFieldEnd();

    assertEquals(200, returnCode);

    // return size
    prot.readFieldBegin();
    final int returnSize = prot.readI32();
    prot.readFieldEnd();

    assertEquals(2326, returnSize);

    prot.readStructEnd();
}

---

From source file:org.apache.hadoop.hive.serde2.TestTCTLSeparatedProtocol.java

License:Apache License

public void testNulls() throws Exception {
    TMemoryBuffer trans = new TMemoryBuffer(1024);
    TCTLSeparatedProtocol prot = new TCTLSeparatedProtocol(trans, 10);
    prot.initialize(new Configuration(), new Properties());

    prot.writeStructBegin(new TStruct());

    prot.writeFieldBegin(new TField());
    prot.writeString(null);/*from w w w  . jav a 2  s .c  om*/
    prot.writeFieldEnd();

    prot.writeFieldBegin(new TField());
    prot.writeString(null);
    prot.writeFieldEnd();

    prot.writeFieldBegin(new TField());
    prot.writeI32(100);
    prot.writeFieldEnd();

    prot.writeFieldBegin(new TField());
    prot.writeString(null);
    prot.writeFieldEnd();

    prot.writeFieldBegin(new TField());
    prot.writeMapBegin(new TMap());
    prot.writeString(null);
    prot.writeString(null);
    prot.writeString("key2");
    prot.writeString(null);
    prot.writeString(null);
    prot.writeString("val3");
    prot.writeMapEnd();
    prot.writeFieldEnd();

    prot.writeStructEnd();

    byte b[] = new byte[3 * 1024];
    int len = trans.read(b, 0, b.length);
    String written = new String(b, 0, len);

    String testRef = "\\N
\\N
100
\\N
\\N\\Nkey2\\N\\Nval3";

    assertTrue(testRef.equals(written));

    trans = new TMemoryBuffer(1023);
    trans.write(b, 0, len);

    prot = new TCTLSeparatedProtocol(trans, 3);
    prot.initialize(new Configuration(), new Properties());

    prot.readStructBegin();

    prot.readFieldBegin();
    String ret = prot.readString();
    prot.readFieldEnd();

    assertNull(ret);

    prot.readFieldBegin();
    ret = prot.readString();
    prot.readFieldEnd();

    assertNull(ret);

    prot.readFieldBegin();
    int ret1 = prot.readI32();
    prot.readFieldEnd();

    assertTrue(ret1 == 100);

    prot.readFieldBegin();
    ret1 = prot.readI32();
    prot.readFieldEnd();

    prot.readFieldBegin();
    TMap map = prot.readMapBegin();

    assertTrue(map.size == 3);

    assertNull(prot.readString());
    assertNull(prot.readString());

    assertTrue(prot.readString().equals("key2"));
    assertNull(prot.readString());

    assertNull(prot.readString());
    assertTrue(prot.readString().equals("val3"));

    prot.readMapEnd();
    prot.readFieldEnd();

    assertTrue(ret1 == 0);
}

---