Example usage for org.apache.hadoop.io ArrayWritable ArrayWritable

Introduction

In this page you can find the example usage for org.apache.hadoop.io ArrayWritable ArrayWritable.

Prototype

public ArrayWritable(String[] strings) 

Source Link

Usage

From source file:co.nubetech.hiho.dedup.TestHashUtility.java

License:Apache License

@Test
public void testMD5HashForArrayWritableKey() throws IOException {
    ArrayWritable key = new ArrayWritable(new String[] { "abc123" });
    MD5Hash md5HashKey1 = HashUtility.getMD5Hash(key);
    MD5Hash md5HashKey2 = HashUtility.getMD5Hash(key);
    assertEquals(md5HashKey1, md5HashKey2);
}

---

From source file:com.csiro.hadoop.WritableTest.java

public static void main(String[] args) {
    System.out.println("*** Primitive Writable ***");

    BooleanWritable bool1 = new BooleanWritable(true);
    ByteWritable byte1 = new ByteWritable((byte) 3);
    System.out.printf("Boolean:%s Byte:%d\n", bool1, byte1.get());

    IntWritable int1 = new IntWritable(5);
    IntWritable int2 = new IntWritable(17);
    System.out.printf("I1:%d I2:%d\n", int1.get(), int2.get());

    int1.set(int2.get());
    System.out.printf("I1:%d I2:%d\n", int1.get(), int2.get());

    Integer int3 = new Integer(23);
    int1.set(int3);
    System.out.printf("I1:%d I2:%d\n", int1.get(), int2.get());

    System.out.println("*** Array Writable ***");

    ArrayWritable a = new ArrayWritable(IntWritable.class);
    a.set(new IntWritable[] { new IntWritable(1), new IntWritable(3), new IntWritable(5) });

    IntWritable[] values = (IntWritable[]) a.get();
    for (IntWritable i : values) {
        System.out.println(i);//from w  w  w . jav  a 2 s .  c  om
    }

    IntArrayWritable ia = new IntArrayWritable();
    ia.set(new IntWritable[] { new IntWritable(1), new IntWritable(3), new IntWritable(5) });

    IntWritable[] ivalues = (IntWritable[]) ia.get();

    ia.set((new LongWritable[] { new LongWritable(10001) }));

    System.out.println("*** Map Writables ***");

    MapWritable m = new MapWritable();
    IntWritable key1 = new IntWritable(5);
    NullWritable value1 = NullWritable.get();

    m.put(key1, value1);
    System.out.println(m.containsKey(key1));
    System.out.println(m.get(key1));
    m.put(new LongWritable(100000000), key1);
    Set<Writable> keys = m.keySet();

    for (Writable k : keys)
        System.out.println(k.getClass());

}

---

From source file:com.ikanow.aleph2.search_service.elasticsearch.utils.TestJsonNodeWritableUtils.java

License:Apache License

@SuppressWarnings("deprecation")
@Test/*from   www  .j a  v a2 s . c  o m*/
public void test_mapWritableWrapper() {
    final ObjectMapper mapper = BeanTemplateUtils.configureMapper(Optional.empty());

    final MapWritable m1 = new MapWritable();

    m1.put(new Text("test1"), new BooleanWritable(true));

    final MapWritable m2 = new MapWritable();
    m2.put(new Text("nested"), m1);
    m2.put(new Text("test2"), new Text("test2"));

    final ArrayWritable a1 = new ArrayWritable(IntWritable.class);
    a1.set(new Writable[] { new IntWritable(4), new IntWritable(5) });

    final ArrayWritable a2 = new ArrayWritable(MapWritable.class);
    a2.set(new Writable[] { m1, m1 });

    m2.put(new Text("array"), a2);
    m1.put(new Text("array"), a1);

    final JsonNode j2 = JsonNodeWritableUtils.from(m2);

    assertEquals(3, j2.size());

    // Check j's contents
    assertEquals(Stream.of("nested", "test2", "array").sorted().collect(Collectors.toList()),
            Optionals.streamOf(j2.fieldNames(), false).sorted().collect(Collectors.toList()));
    assertEquals("test2", j2.get("test2").asText());

    final JsonNode j1 = j2.get("nested");
    assertEquals(2, j1.size());
    final JsonNode j1b = JsonNodeWritableUtils.from(m1);
    assertTrue("{\"test1\":true,\"array\":[4,5]}".equals(j1b.toString())
            || "{\"array\":[4,5],\"test1\":true}".equals(j1b.toString())); //(tests entrySet)
    final ArrayNode an = mapper.createArrayNode();
    an.add(mapper.convertValue(4, JsonNode.class));
    an.add(mapper.convertValue(5, JsonNode.class));
    assertEquals(Arrays.asList(mapper.convertValue(true, JsonNode.class), an),
            Optionals.streamOf(((ObjectNode) j1).elements(), false).collect(Collectors.toList()));

    // OK, now test adding:

    assertEquals(2, j1.size());

    final ObjectNode o1 = (ObjectNode) j1;
    o1.put("added", "added_this");

    final ObjectNodeWrapper o1c = (ObjectNodeWrapper) o1;
    assertFalse(o1c.containsKey("not_present"));
    assertTrue(o1c.containsKey("added"));
    assertTrue(o1c.containsKey("test1"));

    assertEquals(Stream.of("test1", "array", "added").sorted().collect(Collectors.toList()),
            Optionals.streamOf(j1.fieldNames(), false).sorted().collect(Collectors.toList()));
    assertEquals(
            Arrays.asList(mapper.convertValue(true, JsonNode.class), an,
                    mapper.convertValue("added_this", JsonNode.class)),
            Optionals.streamOf(((ObjectNode) j1).elements(), false).collect(Collectors.toList()));
    assertTrue(j1.toString().contains("added_this"));
    assertTrue(j1.toString().contains("4,5"));

    assertEquals(mapper.convertValue("added_this", JsonNode.class), j1.get("added"));

    assertEquals(3, j1.size());

    // OK now test removing:

    assertEquals(null, o1.remove("not_present"));
    assertEquals(mapper.convertValue(true, JsonNode.class), o1.remove("test1"));
    assertEquals(2, o1.size());
    ObjectNode o1b = o1.remove(Arrays.asList("added", "array"));
    assertEquals(0, o1.size());
    assertEquals(0, o1b.size());

    o1.putAll(JsonNodeWritableUtils.from(m1)); // will be minus one object
    assertEquals(2, o1.size());
    assertTrue(o1c.containsValue(mapper.convertValue(true, JsonNode.class)));
    assertFalse(o1c.containsValue("banana"));

    final ObjectNodeWrapper o2 = (ObjectNodeWrapper) JsonNodeWritableUtils.from(m2);
    assertFalse(o2.isEmpty());
    assertTrue(o2.containsKey("array"));
    assertFalse(o2.containsValue("array"));
    assertTrue(o2.containsValue(mapper.convertValue("test2", JsonNode.class)));
    assertEquals(TextNode.class, o2.remove("test2").getClass());
    assertEquals(2, o2.size());
    o2.removeAll();
    assertEquals(0, o2.size());
}

---

From source file:com.jfolson.hive.serde.RBaseSerDe.java

License:Apache License

protected Object deserializeField(RTypedBytesWritableInput in, TypeInfo type, Object reuse) throws IOException {

    RType rtype = in.readTypeCode();/* w  w w. ja  v a  2s . co  m*/
    if (rtype == null) {
        throw new RuntimeException("End of stream");
    }

    // read the type
    Class<? extends Writable> writableType = RType.getWritableType(rtype);
    if (writableType == null) {
        LOG.info("Warning: null Writable type for rtype: " + rtype);
    }
    if (writableType != null && writableType.isAssignableFrom(NullWritable.class)) {
        // indicates that the recorded value is null
        return null;
    }
    //LOG.info("RType should be instantiated as: "+writableType.getSimpleName());

    switch (type.getCategory()) {
    case PRIMITIVE: {
        PrimitiveTypeInfo ptype = (PrimitiveTypeInfo) type;
        switch (ptype.getPrimitiveCategory()) {

        case VOID: {
            return null;
        }

        case BINARY: {
            TypedBytesWritable r = reuse == null ? new TypedBytesWritable() : (TypedBytesWritable) reuse;
            byte[] bytes = in.getInput().readRaw(rtype.code);
            // rewrite the type code
            r.set(bytes, 0, bytes.length);
            return r;
        }

        case BOOLEAN: {
            //TODO Fix this hack:
            if (rtype != RType.BOOL) {
                in.readNull();
                return null;
            }
            BooleanWritable r = reuse == null ? new BooleanWritable() : (BooleanWritable) reuse;
            return in.readBoolean(r);
        }
        /*case BYTE: {
          ByteWritable r = reuse == null ? new ByteWritable()
              : (ByteWritable) reuse;
          r = in.readByte(r);
          return r;
        }*/
        /*case SHORT: {
          ShortWritable r = reuse == null ? new ShortWritable()
              : (ShortWritable) reuse;
          r = in.readShort(r);
          return r;
        }*/
        case INT: {
            if (rtype != RType.INT) {
                in.readNull();
                return null;
            }
            IntWritable r = reuse == null ? null : (IntWritable) reuse;
            return in.readInt(r);
        }
        /*case LONG: {
          LongWritable r = reuse == null ? new LongWritable()
              : (LongWritable) reuse;
          r = in.readLong(r);
          return r;
        }*/
        /*case FLOAT: {
          FloatWritable r = reuse == null ? new FloatWritable()
              : (FloatWritable) reuse;
          r = in.readFloat(r);
          return r;
        }*/
        case DOUBLE: {
            if (rtype != RType.DOUBLE) {
                in.readNull();
                return null;
            }
            DoubleWritable r = reuse == null ? null : (DoubleWritable) reuse;
            return in.readDouble(r);
        }
        case STRING: {
            // TODO fix this hack
            if (rtype != RType.STRING) {
                in.readNull();
                return null;
            }
            Text r = reuse == null ? null : (Text) reuse;
            return in.readText(r);
        }
        default: {
            throw new RuntimeException("Unrecognized type: " + ptype.getPrimitiveCategory());
        }
        }
    }
    // Currently, deserialization of complex types is not supported
    case LIST: {
        if (rtype != RType.VECTOR) {
            in.readNull();
            return null;
        }
        ObjectInspector elemOI = ((ListObjectInspector) TypeInfoUtils
                .getStandardWritableObjectInspectorFromTypeInfo(type)).getListElementObjectInspector();

        PrimitiveObjectInspector elemPOI = (PrimitiveObjectInspector) elemOI;

        Class<? extends Writable> elemClass = (Class<? extends Writable>) elemPOI.getPrimitiveWritableClass();
        ArrayWritable l = reuse == null ? new ArrayWritable(elemClass)
                : new ArrayWritable(elemClass, (Writable[]) reuse);
        in.readVector(l);
        return l.get();
    }
    case MAP:
    case STRUCT:
    default: {
        throw new RuntimeException("Unsupported category: " + type.getCategory());
    }
    }
}

---

From source file:com.jfolson.hive.serde.RTypedBytesSerDe.java

License:Apache License

Object deserializeField(RTypedBytesWritableInput in, TypeInfo type, Object reuse) throws IOException {

    RType rtype = in.readTypeCode();//  w  ww . j av a2s. c  o m
    if (rtype == null) {
        throw new RuntimeException("End of stream");
    }

    // read the type
    Class<? extends Writable> writableType = RType.getWritableType(rtype);
    if (writableType == null) {
        LOG.info("Warning: null Writable type for rtype: " + rtype);
    }
    if (writableType != null && writableType.isAssignableFrom(NullWritable.class)) {
        // indicates that the recorded value is null
        return null;
    }
    //LOG.info("RType should be instantiated as: "+writableType.getSimpleName());

    switch (type.getCategory()) {
    case PRIMITIVE: {
        PrimitiveTypeInfo ptype = (PrimitiveTypeInfo) type;
        switch (ptype.getPrimitiveCategory()) {

        case VOID: {
            return null;
        }

        case BINARY: {
            TypedBytesWritable r = reuse == null ? new TypedBytesWritable() : (TypedBytesWritable) reuse;
            byte[] bytes = in.getInput().readRaw(rtype.code);
            // rewrite the type code
            r.set(bytes, 0, bytes.length);
            return r;
        }

        case BOOLEAN: {
            //TODO Fix this hack:
            if (rtype != RType.BOOL) {
                in.readNull();
                return null;
            }
            BooleanWritable r = reuse == null ? new BooleanWritable() : (BooleanWritable) reuse;
            return in.readBoolean(r);
        }
        /*case BYTE: {
          ByteWritable r = reuse == null ? new ByteWritable()
              : (ByteWritable) reuse;
          r = in.readByte(r);
          return r;
        }*/
        /*case SHORT: {
          ShortWritable r = reuse == null ? new ShortWritable()
              : (ShortWritable) reuse;
          r = in.readShort(r);
          return r;
        }*/
        case INT: {
            if (rtype != RType.INT) {
                in.readNull();
                return null;
            }
            IntWritable r = reuse == null ? null : (IntWritable) reuse;
            return in.readInt(r);
        }
        /*case LONG: {
          LongWritable r = reuse == null ? new LongWritable()
              : (LongWritable) reuse;
          r = in.readLong(r);
          return r;
        }*/
        /*case FLOAT: {
          FloatWritable r = reuse == null ? new FloatWritable()
              : (FloatWritable) reuse;
          r = in.readFloat(r);
          return r;
        }*/
        case DOUBLE: {
            if (rtype != RType.DOUBLE) {
                in.readNull();
                return null;
            }
            DoubleWritable r = reuse == null ? null : (DoubleWritable) reuse;
            return in.readDouble(r);
        }
        case STRING: {
            // TODO fix this hack
            if (rtype != RType.STRING) {
                in.readNull();
                return null;
            }
            Text r = reuse == null ? null : (Text) reuse;
            return in.readText(r);
        }
        default: {
            throw new RuntimeException("Unrecognized type: " + ptype.getPrimitiveCategory());
        }
        }
    }
    // Currently, deserialization of complex types is not supported
    case LIST: {
        if (rtype != RType.VECTOR) {
            in.readNull();
            return null;
        }
        ObjectInspector elemOI = ((ListObjectInspector) TypeInfoUtils
                .getStandardWritableObjectInspectorFromTypeInfo(type)).getListElementObjectInspector();

        PrimitiveObjectInspector elemPOI = (PrimitiveObjectInspector) elemOI;

        Class<? extends Writable> elemClass = (Class<? extends Writable>) elemPOI.getPrimitiveWritableClass();
        ArrayWritable l = reuse == null ? new ArrayWritable(elemClass)
                : new ArrayWritable(elemClass, (Writable[]) reuse);
        in.readVector(l);
        return l.get();
    }
    case MAP:
    case STRUCT:
    default: {
        throw new RuntimeException("Unsupported category: " + type.getCategory());
    }
    }
}

---

From source file:com.jfolson.hive.serde.RTypedBytesWritableInput.java

License:Apache License

public ArrayWritable readVector(ArrayWritable aw) throws IOException {
    if (aw == null) {
        aw = new ArrayWritable(Writable.class);
    } /* else if (!aw.getValueClass().equals(clazz)) {
           throw new RuntimeException("value class has to be "+clazz.getCanonicalName());
         }*//*from  ww w.j  ava 2  s  .c o m*/
    int length = in.readVectorHeader();
    Writable[] writables = new Writable[length];
    for (int i = 0; i < length; i++) {
        Writable w = null;
        if (aw.getValueClass() == TypedBytesWritable.class) {
            w = readRaw();
        } else {
            w = this.read();
            if (w instanceof NullWritable) {
                w = null;
            }
        }
        writables[i] = w;
    }

    aw.set(writables);
    return aw;
}

---

From source file:com.mozilla.hadoop.hbase.mapreduce.MultiScanTableMapReduceUtil.java

License:Apache License

/**
 * Converts base64 scans string back into a Scan array
 * @param base64//from   w w  w .  j a va  2s.co m
 * @return
 * @throws IOException
 */
public static Scan[] convertStringToScanArray(final String base64) throws IOException {
    final DataInputStream dis = new DataInputStream(new ByteArrayInputStream(Base64.decode(base64)));

    ArrayWritable aw = new ArrayWritable(Scan.class);
    aw.readFields(dis);

    Writable[] writables = aw.get();
    Scan[] scans = new Scan[writables.length];
    for (int i = 0; i < writables.length; i++) {
        scans[i] = (Scan) writables[i];
    }

    return scans;
}

---

From source file:com.ricemap.spateDB.mapred.ShapeArrayRecordReader.java

License:Apache License

@Override
public ArrayWritable createValue() {
    return new ArrayWritable(shapeClass);
}

---

From source file:crunch.MaxTemperature.java

License:Apache License

@Test
    public void test() throws IOException {
        // vv ArrayWritableTest
        ArrayWritable writable = new ArrayWritable(Text.class);
        // ^^ ArrayWritableTest
        writable.set(new Text[] { new Text("cat"), new Text("dog") });

        TextArrayWritable dest = new TextArrayWritable();
        WritableUtils.cloneInto(dest, writable);
        assertThat(dest.get().length, is(2));
        // TODO: fix cast, also use single assert
        assertThat((Text) dest.get()[0], is(new Text("cat")));
        assertThat((Text) dest.get()[1], is(new Text("dog")));

        Text[] copy = (Text[]) dest.toArray();
        assertThat(copy[0], is(new Text("cat")));
        assertThat(copy[1], is(new Text("dog")));
    }//  www.  j a  va2  s . c  o m

---

From source file:edu.ub.ahstfg.indexer.mapred.IndexRecordReader.java

License:Open Source License

@Override
public ArrayWritable createValue() {
    return new ArrayWritable(DocumentDescriptor.class);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• »