Example usage for org.apache.hadoop.io Text Text

Introduction

In this page you can find the example usage for org.apache.hadoop.io Text Text.

Prototype

public Text(byte[] utf8)

Source Link

Document

Construct from a byte array.

Usage

From source file:cascading.tuple.hadoop.SpillableTupleHadoopTest.java

License:Open Source License

private void performListTest(int size, int threshold, CompressionCodec codec, int spills) {
    Configuration jobConf = new Configuration();

    jobConf.set("io.serializations",
            TestSerialization.class.getName() + "," + WritableSerialization.class.getName()); // disable/replace WritableSerialization class
    jobConf.set("cascading.serialization.tokens",
            "1000=" + BooleanWritable.class.getName() + ",10001=" + Text.class.getName()); // not using Text, just testing parsing

    HadoopSpillableTupleList list = new HadoopSpillableTupleList(threshold, codec, jobConf);

    for (int i = 0; i < size; i++) {
        String aString = "string number " + i;
        double random = Math.random();

        list.add(new Tuple(i, aString, random, new Text(aString), new TestText(aString),
                new Tuple("inner tuple", new BytesWritable(aString.getBytes()))));
    }/*w  w w .  ja v a  2  s.  com*/

    assertEquals("not equal: list.size();", size, list.size());

    assertEquals("not equal: list.getNumFiles()", spills, list.spillCount());

    int i = -1;
    int count = 0;
    for (Tuple tuple : list) {
        int value = tuple.getInteger(0);
        assertTrue("wrong diff", value - i == 1);
        assertEquals("wrong value", "string number " + count, tuple.getObject(3).toString());
        assertEquals("wrong value", "string number " + count, tuple.getObject(4).toString());
        assertTrue("wrong type", tuple.getObject(5) instanceof Tuple);

        BytesWritable bytesWritable = (BytesWritable) ((Tuple) tuple.getObject(5)).getObject(1);
        byte[] bytes = bytesWritable.getBytes();
        String actual = new String(bytes, 0, bytesWritable.getLength());

        assertEquals("wrong value", "string number " + count, actual);

        i = value;
        count++;
    }

    assertEquals("not equal: list.size();", size, count);

    Iterator<Tuple> iterator = list.iterator();

    assertEquals("not equal: iterator.next().get(1)", "string number 0", iterator.next().getObject(1));
    assertEquals("not equal: iterator.next().get(1)", "string number 1", iterator.next().getObject(1));
}

From source file:cascading.tuple.hadoop.SpillableTupleHadoopTest.java

License:Open Source License

private void performMapTest(int numKeys, int listSize, int mapThreshold, int listThreshold,
        Configuration jobConf) {//from   w w w  .  j  av  a2 s .  c  om
    jobConf.set("io.serializations",
            TestSerialization.class.getName() + "," + WritableSerialization.class.getName()); // disable/replace WritableSerialization class
    jobConf.set("cascading.serialization.tokens",
            "1000=" + BooleanWritable.class.getName() + ",10001=" + Text.class.getName()); // not using Text, just testing parsing

    HadoopFlowProcess flowProcess = new HadoopFlowProcess(jobConf);
    HadoopSpillableTupleMap map = new HadoopSpillableTupleMap(SpillableProps.defaultMapInitialCapacity,
            SpillableProps.defaultMapLoadFactor, mapThreshold, listThreshold, flowProcess);

    Set<Integer> keySet = new HashSet<Integer>();
    Random gen = new Random(1);

    for (int i = 0; i < listSize * numKeys; i++) {
        String aString = "string number " + i;
        double random = Math.random();

        double keys = numKeys / 3.0;
        int key = (int) (gen.nextDouble() * keys + gen.nextDouble() * keys + gen.nextDouble() * keys);

        Tuple tuple = new Tuple(i, aString, random, new Text(aString), new TestText(aString),
                new Tuple("inner tuple", new BytesWritable(aString.getBytes())));

        map.get(new Tuple(key)).add(tuple);

        keySet.add(key);
    }

    // the list test above verifies the contents are being serialized, the Map is just a container of lists.
    assertEquals("not equal: map.size();", keySet.size(), map.size());
}

From source file:cascading.tuple.SpillableTupleTest.java

License:Open Source License

private void performSpillTest(int size, int threshold, CompressionCodec codec) {
    SpillableTupleList list = new SpillableTupleList(threshold, null, codec);

    for (int i = 0; i < size; i++) {
        String aString = "string number " + i;
        double random = Math.random();

        list.add(new Tuple(i, aString, random, new Text(aString)));
    }//w ww  . j a va  2  s  .c o m

    assertEquals("not equal: list.size();", size, list.size());
    assertEquals("not equal: list.getNumFiles()", (int) Math.floor(size / threshold), list.getNumFiles());

    int i = -1;
    int count = 0;
    for (Tuple tuple : list) {
        int value = tuple.getInteger(0);
        assertTrue("wrong diff", value - i == 1);
        assertEquals("wrong value", "string number " + count, tuple.get(3).toString());
        i = value;
        count++;
    }

    assertEquals("not equal: list.size();", size, count);

    Iterator<Tuple> iterator = list.iterator();

    assertEquals("not equal: iterator.next().get(0)", "string number 0", iterator.next().get(1));
}

From source file:cereal.examples.pojo.PojoExample.java

License:Apache License

public static void main(String[] args) throws Exception {
    Person p = new Person();
    p.setFirstName("Bob");
    p.setMiddleName("Joe");
    p.setLastName("Franklin");
    p.setAge(30);//  w w  w . j a v a2  s  .c o  m
    p.setHeight(72);
    p.setWeight(220);

    Registry registry = new RegistryImpl();
    registry.add(new PojoPersonMapping());
    String tableName = "pojo_people";
    ZooKeeperInstance inst = new ZooKeeperInstance("accumulo", "127.0.0.1");
    Connector conn = inst.getConnector("root", new PasswordToken("secret"));
    if (!conn.tableOperations().exists(tableName)) {
        conn.tableOperations().create(tableName);
    }

    System.out.println("Person: " + p);

    try (Store store = new StoreImpl(registry, conn, tableName)) {
        store.write(Collections.singleton(p));
        store.flush();

        Person pCopy = store.read(new Text("Bob_Joe_Franklin"), Person.class);
        System.out.println("Copy: " + pCopy);
    }
}

From source file:cereal.examples.pojo.PojoPersonMapping.java

License:Apache License

@Override
public Text getRowId(Person obj) {
    String firstName = obj.getFirstName(), middleName = obj.getMiddleName(), lastName = obj.getLastName();
    StringBuilder row = new StringBuilder(32);
    if (null != firstName) {
        row.append(firstName);// ww w.j av a  2  s .  c om
    }
    if (null != middleName) {
        if (row.length() > 0) {
            row.append("_");
        }
        row.append(middleName);
    }
    if (null != lastName) {
        if (row.length() > 0) {
            row.append("_");
        }
        row.append(lastName);
    }
    return new Text(row.toString());
}

From source file:cereal.examples.pojo.PojoPersonMapping.java

License:Apache License

Text text(String txt) {
    return new Text(txt);
}

From source file:cereal.examples.protobuf.ProtobufEngineMapping.java

License:Apache License

@Override
public Text getRowId(Engine obj) {
    StringBuilder sb = new StringBuilder(32);
    if (obj.hasCylinders()) {
        sb.append(obj.getCylinders());// w  w  w  .j  a v a2s.  c o m
    }
    if (obj.hasDisplacement()) {
        if (0 < sb.length()) {
            sb.append("_");
        }
        sb.append(obj.getDisplacement());
    }
    if (obj.hasHorsepower()) {
        if (0 < sb.length()) {
            sb.append("_");
        }
        sb.append(obj.getHorsepower());
    }
    if (obj.hasTorque()) {
        if (0 < sb.length()) {
            sb.append("_");
        }
        sb.append(obj.getTorque());
    }
    return new Text(sb.toString());
}

From source file:cereal.examples.protobuf.ProtobufExample.java

License:Apache License

public static void main(String[] args) throws Exception {
    Person p = Person.newBuilder().setFirstName("Bob").setMiddleName("Joe").setLastName("Franklin").setAge(30)
            .setHeight(72).setWeight(220).build();

    Registry registry = new RegistryImpl();
    registry.add(new ProtobufPersonMapping(registry));
    String tableName = "pb_people";
    ZooKeeperInstance inst = new ZooKeeperInstance("accumulo", "127.0.0.1");
    Connector conn = inst.getConnector("root", new PasswordToken("secret"));
    if (!conn.tableOperations().exists(tableName)) {
        conn.tableOperations().create(tableName);
    }/* w  w  w.j  a  v  a  2s.com*/

    System.out.println("Person: [" + TextFormat.shortDebugString(p) + "]");

    try (Store store = new StoreImpl(registry, conn, tableName)) {
        store.write(Collections.singleton(p));
        store.flush();

        Person pCopy = store.read(new Text("Bob_Joe_Franklin"), Person.class);
        System.out.println("Copy: [" + TextFormat.shortDebugString(pCopy) + "]");
    }
}

From source file:cereal.examples.protobuf.ProtobufPersonMapping.java

License:Apache License

@Override
public Text getRowId(Person obj) {
    StringBuilder sb = new StringBuilder(32);
    if (obj.hasFirstName()) {
        sb.append(obj.getFirstName());//  w  w w . j  a  va 2 s  . co m
    }
    if (obj.hasMiddleName()) {
        if (0 < sb.length()) {
            sb.append("_");
        }
        sb.append(obj.getMiddleName());
    }
    if (obj.hasLastName()) {
        if (0 < sb.length()) {
            sb.append("_");
        }
        sb.append(obj.getLastName());
    }
    return new Text(sb.toString());
}

From source file:cereal.examples.protobuf.ProtobufVehicleMapping.java

License:Apache License

@Override
public Text getRowId(Vehicle obj) {
    StringBuilder sb = new StringBuilder(32);
    if (obj.hasMake()) {
        sb.append(obj.getMake());//from w  w w . j  av a  2  s.co m
    }
    if (obj.hasModel()) {
        if (0 < sb.length()) {
            sb.append("_");
        }
        sb.append(obj.getModel());
    }
    if (obj.hasWheels()) {
        if (0 < sb.length()) {
            sb.append("_");
        }
        sb.append(obj.getWheels());
    }
    return new Text(sb.toString());
}