Example usage for org.apache.hadoop.io Text Text

List of usage examples for org.apache.hadoop.io Text Text

Introduction

In this page you can find the example usage for org.apache.hadoop.io Text Text.

Prototype

public Text(byte[] utf8) 

Source Link

Document

Construct from a byte array.

Usage

From source file:cascading.tuple.hadoop.SpillableTupleHadoopTest.java

License:Open Source License

private void performListTest(int size, int threshold, CompressionCodec codec, int spills) {
    Configuration jobConf = new Configuration();

    jobConf.set("io.serializations",
            TestSerialization.class.getName() + "," + WritableSerialization.class.getName()); // disable/replace WritableSerialization class
    jobConf.set("cascading.serialization.tokens",
            "1000=" + BooleanWritable.class.getName() + ",10001=" + Text.class.getName()); // not using Text, just testing parsing

    HadoopSpillableTupleList list = new HadoopSpillableTupleList(threshold, codec, jobConf);

    for (int i = 0; i < size; i++) {
        String aString = "string number " + i;
        double random = Math.random();

        list.add(new Tuple(i, aString, random, new Text(aString), new TestText(aString),
                new Tuple("inner tuple", new BytesWritable(aString.getBytes()))));
    }/*w  w w .  ja v a  2  s.  com*/

    assertEquals("not equal: list.size();", size, list.size());

    assertEquals("not equal: list.getNumFiles()", spills, list.spillCount());

    int i = -1;
    int count = 0;
    for (Tuple tuple : list) {
        int value = tuple.getInteger(0);
        assertTrue("wrong diff", value - i == 1);
        assertEquals("wrong value", "string number " + count, tuple.getObject(3).toString());
        assertEquals("wrong value", "string number " + count, tuple.getObject(4).toString());
        assertTrue("wrong type", tuple.getObject(5) instanceof Tuple);

        BytesWritable bytesWritable = (BytesWritable) ((Tuple) tuple.getObject(5)).getObject(1);
        byte[] bytes = bytesWritable.getBytes();
        String actual = new String(bytes, 0, bytesWritable.getLength());

        assertEquals("wrong value", "string number " + count, actual);

        i = value;
        count++;
    }

    assertEquals("not equal: list.size();", size, count);

    Iterator<Tuple> iterator = list.iterator();

    assertEquals("not equal: iterator.next().get(1)", "string number 0", iterator.next().getObject(1));
    assertEquals("not equal: iterator.next().get(1)", "string number 1", iterator.next().getObject(1));
}

From source file:cascading.tuple.hadoop.SpillableTupleHadoopTest.java

License:Open Source License

private void performMapTest(int numKeys, int listSize, int mapThreshold, int listThreshold,
        Configuration jobConf) {//from   w w w  .  j  av  a2 s .  c  om
    jobConf.set("io.serializations",
            TestSerialization.class.getName() + "," + WritableSerialization.class.getName()); // disable/replace WritableSerialization class
    jobConf.set("cascading.serialization.tokens",
            "1000=" + BooleanWritable.class.getName() + ",10001=" + Text.class.getName()); // not using Text, just testing parsing

    HadoopFlowProcess flowProcess = new HadoopFlowProcess(jobConf);
    HadoopSpillableTupleMap map = new HadoopSpillableTupleMap(SpillableProps.defaultMapInitialCapacity,
            SpillableProps.defaultMapLoadFactor, mapThreshold, listThreshold, flowProcess);

    Set<Integer> keySet = new HashSet<Integer>();
    Random gen = new Random(1);

    for (int i = 0; i < listSize * numKeys; i++) {
        String aString = "string number " + i;
        double random = Math.random();

        double keys = numKeys / 3.0;
        int key = (int) (gen.nextDouble() * keys + gen.nextDouble() * keys + gen.nextDouble() * keys);

        Tuple tuple = new Tuple(i, aString, random, new Text(aString), new TestText(aString),
                new Tuple("inner tuple", new BytesWritable(aString.getBytes())));

        map.get(new Tuple(key)).add(tuple);

        keySet.add(key);
    }

    // the list test above verifies the contents are being serialized, the Map is just a container of lists.
    assertEquals("not equal: map.size();", keySet.size(), map.size());
}

From source file:cascading.tuple.SpillableTupleTest.java

License:Open Source License

private void performSpillTest(int size, int threshold, CompressionCodec codec) {
    SpillableTupleList list = new SpillableTupleList(threshold, null, codec);

    for (int i = 0; i < size; i++) {
        String aString = "string number " + i;
        double random = Math.random();

        list.add(new Tuple(i, aString, random, new Text(aString)));
    }//w ww  . j a va  2  s  .c o m

    assertEquals("not equal: list.size();", size, list.size());
    assertEquals("not equal: list.getNumFiles()", (int) Math.floor(size / threshold), list.getNumFiles());

    int i = -1;
    int count = 0;
    for (Tuple tuple : list) {
        int value = tuple.getInteger(0);
        assertTrue("wrong diff", value - i == 1);
        assertEquals("wrong value", "string number " + count, tuple.get(3).toString());
        i = value;
        count++;
    }

    assertEquals("not equal: list.size();", size, count);

    Iterator<Tuple> iterator = list.iterator();

    assertEquals("not equal: iterator.next().get(0)", "string number 0", iterator.next().get(1));
}

From source file:cereal.examples.pojo.PojoExample.java

License:Apache License

public static void main(String[] args) throws Exception {
    Person p = new Person();
    p.setFirstName("Bob");
    p.setMiddleName("Joe");
    p.setLastName("Franklin");
    p.setAge(30);//  w w  w . j a v a2  s  .c o  m
    p.setHeight(72);
    p.setWeight(220);

    Registry registry = new RegistryImpl();
    registry.add(new PojoPersonMapping());
    String tableName = "pojo_people";
    ZooKeeperInstance inst = new ZooKeeperInstance("accumulo", "127.0.0.1");
    Connector conn = inst.getConnector("root", new PasswordToken("secret"));
    if (!conn.tableOperations().exists(tableName)) {
        conn.tableOperations().create(tableName);
    }

    System.out.println("Person: " + p);

    try (Store store = new StoreImpl(registry, conn, tableName)) {
        store.write(Collections.singleton(p));
        store.flush();

        Person pCopy = store.read(new Text("Bob_Joe_Franklin"), Person.class);
        System.out.println("Copy: " + pCopy);
    }
}

From source file:cereal.examples.pojo.PojoPersonMapping.java

License:Apache License

@Override
public Text getRowId(Person obj) {
    String firstName = obj.getFirstName(), middleName = obj.getMiddleName(), lastName = obj.getLastName();
    StringBuilder row = new StringBuilder(32);
    if (null != firstName) {
        row.append(firstName);// ww w.j av a  2  s .  c om
    }
    if (null != middleName) {
        if (row.length() > 0) {
            row.append("_");
        }
        row.append(middleName);
    }
    if (null != lastName) {
        if (row.length() > 0) {
            row.append("_");
        }
        row.append(lastName);
    }
    return new Text(row.toString());
}

From source file:cereal.examples.pojo.PojoPersonMapping.java

License:Apache License

Text text(String txt) {
    return new Text(txt);
}

From source file:cereal.examples.protobuf.ProtobufEngineMapping.java

License:Apache License

@Override
public Text getRowId(Engine obj) {
    StringBuilder sb = new StringBuilder(32);
    if (obj.hasCylinders()) {
        sb.append(obj.getCylinders());// w  w  w  .j  a v a2s.  c o m
    }
    if (obj.hasDisplacement()) {
        if (0 < sb.length()) {
            sb.append("_");
        }
        sb.append(obj.getDisplacement());
    }
    if (obj.hasHorsepower()) {
        if (0 < sb.length()) {
            sb.append("_");
        }
        sb.append(obj.getHorsepower());
    }
    if (obj.hasTorque()) {
        if (0 < sb.length()) {
            sb.append("_");
        }
        sb.append(obj.getTorque());
    }
    return new Text(sb.toString());
}

From source file:cereal.examples.protobuf.ProtobufExample.java

License:Apache License

public static void main(String[] args) throws Exception {
    Person p = Person.newBuilder().setFirstName("Bob").setMiddleName("Joe").setLastName("Franklin").setAge(30)
            .setHeight(72).setWeight(220).build();

    Registry registry = new RegistryImpl();
    registry.add(new ProtobufPersonMapping(registry));
    String tableName = "pb_people";
    ZooKeeperInstance inst = new ZooKeeperInstance("accumulo", "127.0.0.1");
    Connector conn = inst.getConnector("root", new PasswordToken("secret"));
    if (!conn.tableOperations().exists(tableName)) {
        conn.tableOperations().create(tableName);
    }/* w  w  w.j  a  v  a  2s.com*/

    System.out.println("Person: [" + TextFormat.shortDebugString(p) + "]");

    try (Store store = new StoreImpl(registry, conn, tableName)) {
        store.write(Collections.singleton(p));
        store.flush();

        Person pCopy = store.read(new Text("Bob_Joe_Franklin"), Person.class);
        System.out.println("Copy: [" + TextFormat.shortDebugString(pCopy) + "]");
    }
}

From source file:cereal.examples.protobuf.ProtobufPersonMapping.java

License:Apache License

@Override
public Text getRowId(Person obj) {
    StringBuilder sb = new StringBuilder(32);
    if (obj.hasFirstName()) {
        sb.append(obj.getFirstName());//  w  w w . j  a  va 2 s  . co m
    }
    if (obj.hasMiddleName()) {
        if (0 < sb.length()) {
            sb.append("_");
        }
        sb.append(obj.getMiddleName());
    }
    if (obj.hasLastName()) {
        if (0 < sb.length()) {
            sb.append("_");
        }
        sb.append(obj.getLastName());
    }
    return new Text(sb.toString());
}

From source file:cereal.examples.protobuf.ProtobufVehicleMapping.java

License:Apache License

@Override
public Text getRowId(Vehicle obj) {
    StringBuilder sb = new StringBuilder(32);
    if (obj.hasMake()) {
        sb.append(obj.getMake());//from w  w w . j  av a  2  s.co m
    }
    if (obj.hasModel()) {
        if (0 < sb.length()) {
            sb.append("_");
        }
        sb.append(obj.getModel());
    }
    if (obj.hasWheels()) {
        if (0 < sb.length()) {
            sb.append("_");
        }
        sb.append(obj.getWheels());
    }
    return new Text(sb.toString());
}