Example usage for org.apache.hadoop.io ArrayWritable get

Introduction

In this page you can find the example usage for org.apache.hadoop.io ArrayWritable get.

Prototype

public Writable[] get() 

Source Link

Usage

From source file:com.csiro.hadoop.WritableTest.java

public static void main(String[] args) {
    System.out.println("*** Primitive Writable ***");

    BooleanWritable bool1 = new BooleanWritable(true);
    ByteWritable byte1 = new ByteWritable((byte) 3);
    System.out.printf("Boolean:%s Byte:%d\n", bool1, byte1.get());

    IntWritable int1 = new IntWritable(5);
    IntWritable int2 = new IntWritable(17);
    System.out.printf("I1:%d I2:%d\n", int1.get(), int2.get());

    int1.set(int2.get());
    System.out.printf("I1:%d I2:%d\n", int1.get(), int2.get());

    Integer int3 = new Integer(23);
    int1.set(int3);
    System.out.printf("I1:%d I2:%d\n", int1.get(), int2.get());

    System.out.println("*** Array Writable ***");

    ArrayWritable a = new ArrayWritable(IntWritable.class);
    a.set(new IntWritable[] { new IntWritable(1), new IntWritable(3), new IntWritable(5) });

    IntWritable[] values = (IntWritable[]) a.get();
    for (IntWritable i : values) {
        System.out.println(i);//from  w  ww  . jav a 2 s  . c o m
    }

    IntArrayWritable ia = new IntArrayWritable();
    ia.set(new IntWritable[] { new IntWritable(1), new IntWritable(3), new IntWritable(5) });

    IntWritable[] ivalues = (IntWritable[]) ia.get();

    ia.set((new LongWritable[] { new LongWritable(10001) }));

    System.out.println("*** Map Writables ***");

    MapWritable m = new MapWritable();
    IntWritable key1 = new IntWritable(5);
    NullWritable value1 = NullWritable.get();

    m.put(key1, value1);
    System.out.println(m.containsKey(key1));
    System.out.println(m.get(key1));
    m.put(new LongWritable(100000000), key1);
    Set<Writable> keys = m.keySet();

    for (Writable k : keys)
        System.out.println(k.getClass());

}

---

From source file:com.jfolson.hive.serde.RBaseSerDe.java

License:Apache License

protected Object deserializeField(RTypedBytesWritableInput in, TypeInfo type, Object reuse) throws IOException {

    RType rtype = in.readTypeCode();/*from   ww w.java  2s.c o  m*/
    if (rtype == null) {
        throw new RuntimeException("End of stream");
    }

    // read the type
    Class<? extends Writable> writableType = RType.getWritableType(rtype);
    if (writableType == null) {
        LOG.info("Warning: null Writable type for rtype: " + rtype);
    }
    if (writableType != null && writableType.isAssignableFrom(NullWritable.class)) {
        // indicates that the recorded value is null
        return null;
    }
    //LOG.info("RType should be instantiated as: "+writableType.getSimpleName());

    switch (type.getCategory()) {
    case PRIMITIVE: {
        PrimitiveTypeInfo ptype = (PrimitiveTypeInfo) type;
        switch (ptype.getPrimitiveCategory()) {

        case VOID: {
            return null;
        }

        case BINARY: {
            TypedBytesWritable r = reuse == null ? new TypedBytesWritable() : (TypedBytesWritable) reuse;
            byte[] bytes = in.getInput().readRaw(rtype.code);
            // rewrite the type code
            r.set(bytes, 0, bytes.length);
            return r;
        }

        case BOOLEAN: {
            //TODO Fix this hack:
            if (rtype != RType.BOOL) {
                in.readNull();
                return null;
            }
            BooleanWritable r = reuse == null ? new BooleanWritable() : (BooleanWritable) reuse;
            return in.readBoolean(r);
        }
        /*case BYTE: {
          ByteWritable r = reuse == null ? new ByteWritable()
              : (ByteWritable) reuse;
          r = in.readByte(r);
          return r;
        }*/
        /*case SHORT: {
          ShortWritable r = reuse == null ? new ShortWritable()
              : (ShortWritable) reuse;
          r = in.readShort(r);
          return r;
        }*/
        case INT: {
            if (rtype != RType.INT) {
                in.readNull();
                return null;
            }
            IntWritable r = reuse == null ? null : (IntWritable) reuse;
            return in.readInt(r);
        }
        /*case LONG: {
          LongWritable r = reuse == null ? new LongWritable()
              : (LongWritable) reuse;
          r = in.readLong(r);
          return r;
        }*/
        /*case FLOAT: {
          FloatWritable r = reuse == null ? new FloatWritable()
              : (FloatWritable) reuse;
          r = in.readFloat(r);
          return r;
        }*/
        case DOUBLE: {
            if (rtype != RType.DOUBLE) {
                in.readNull();
                return null;
            }
            DoubleWritable r = reuse == null ? null : (DoubleWritable) reuse;
            return in.readDouble(r);
        }
        case STRING: {
            // TODO fix this hack
            if (rtype != RType.STRING) {
                in.readNull();
                return null;
            }
            Text r = reuse == null ? null : (Text) reuse;
            return in.readText(r);
        }
        default: {
            throw new RuntimeException("Unrecognized type: " + ptype.getPrimitiveCategory());
        }
        }
    }
    // Currently, deserialization of complex types is not supported
    case LIST: {
        if (rtype != RType.VECTOR) {
            in.readNull();
            return null;
        }
        ObjectInspector elemOI = ((ListObjectInspector) TypeInfoUtils
                .getStandardWritableObjectInspectorFromTypeInfo(type)).getListElementObjectInspector();

        PrimitiveObjectInspector elemPOI = (PrimitiveObjectInspector) elemOI;

        Class<? extends Writable> elemClass = (Class<? extends Writable>) elemPOI.getPrimitiveWritableClass();
        ArrayWritable l = reuse == null ? new ArrayWritable(elemClass)
                : new ArrayWritable(elemClass, (Writable[]) reuse);
        in.readVector(l);
        return l.get();
    }
    case MAP:
    case STRUCT:
    default: {
        throw new RuntimeException("Unsupported category: " + type.getCategory());
    }
    }
}

---

From source file:com.jfolson.hive.serde.RTypedBytesSerDe.java

License:Apache License

Object deserializeField(RTypedBytesWritableInput in, TypeInfo type, Object reuse) throws IOException {

    RType rtype = in.readTypeCode();//w w  w  . ja  va2s. com
    if (rtype == null) {
        throw new RuntimeException("End of stream");
    }

    // read the type
    Class<? extends Writable> writableType = RType.getWritableType(rtype);
    if (writableType == null) {
        LOG.info("Warning: null Writable type for rtype: " + rtype);
    }
    if (writableType != null && writableType.isAssignableFrom(NullWritable.class)) {
        // indicates that the recorded value is null
        return null;
    }
    //LOG.info("RType should be instantiated as: "+writableType.getSimpleName());

    switch (type.getCategory()) {
    case PRIMITIVE: {
        PrimitiveTypeInfo ptype = (PrimitiveTypeInfo) type;
        switch (ptype.getPrimitiveCategory()) {

        case VOID: {
            return null;
        }

        case BINARY: {
            TypedBytesWritable r = reuse == null ? new TypedBytesWritable() : (TypedBytesWritable) reuse;
            byte[] bytes = in.getInput().readRaw(rtype.code);
            // rewrite the type code
            r.set(bytes, 0, bytes.length);
            return r;
        }

        case BOOLEAN: {
            //TODO Fix this hack:
            if (rtype != RType.BOOL) {
                in.readNull();
                return null;
            }
            BooleanWritable r = reuse == null ? new BooleanWritable() : (BooleanWritable) reuse;
            return in.readBoolean(r);
        }
        /*case BYTE: {
          ByteWritable r = reuse == null ? new ByteWritable()
              : (ByteWritable) reuse;
          r = in.readByte(r);
          return r;
        }*/
        /*case SHORT: {
          ShortWritable r = reuse == null ? new ShortWritable()
              : (ShortWritable) reuse;
          r = in.readShort(r);
          return r;
        }*/
        case INT: {
            if (rtype != RType.INT) {
                in.readNull();
                return null;
            }
            IntWritable r = reuse == null ? null : (IntWritable) reuse;
            return in.readInt(r);
        }
        /*case LONG: {
          LongWritable r = reuse == null ? new LongWritable()
              : (LongWritable) reuse;
          r = in.readLong(r);
          return r;
        }*/
        /*case FLOAT: {
          FloatWritable r = reuse == null ? new FloatWritable()
              : (FloatWritable) reuse;
          r = in.readFloat(r);
          return r;
        }*/
        case DOUBLE: {
            if (rtype != RType.DOUBLE) {
                in.readNull();
                return null;
            }
            DoubleWritable r = reuse == null ? null : (DoubleWritable) reuse;
            return in.readDouble(r);
        }
        case STRING: {
            // TODO fix this hack
            if (rtype != RType.STRING) {
                in.readNull();
                return null;
            }
            Text r = reuse == null ? null : (Text) reuse;
            return in.readText(r);
        }
        default: {
            throw new RuntimeException("Unrecognized type: " + ptype.getPrimitiveCategory());
        }
        }
    }
    // Currently, deserialization of complex types is not supported
    case LIST: {
        if (rtype != RType.VECTOR) {
            in.readNull();
            return null;
        }
        ObjectInspector elemOI = ((ListObjectInspector) TypeInfoUtils
                .getStandardWritableObjectInspectorFromTypeInfo(type)).getListElementObjectInspector();

        PrimitiveObjectInspector elemPOI = (PrimitiveObjectInspector) elemOI;

        Class<? extends Writable> elemClass = (Class<? extends Writable>) elemPOI.getPrimitiveWritableClass();
        ArrayWritable l = reuse == null ? new ArrayWritable(elemClass)
                : new ArrayWritable(elemClass, (Writable[]) reuse);
        in.readVector(l);
        return l.get();
    }
    case MAP:
    case STRUCT:
    default: {
        throw new RuntimeException("Unsupported category: " + type.getCategory());
    }
    }
}

---

From source file:com.jfolson.hive.serde.RTypedBytesWritableOutput.java

License:Apache License

public void writeVector(ArrayWritable aw) throws IOException {
    Writable[] writables = aw.get();
    out.writeVectorHeader(writables.length);
    for (Writable writable : writables) {
        write(writable);/*from  w w w. j  a va2s .co  m*/
    }
}

---

From source file:com.jfolson.hive.serde.RTypedBytesWritableOutput.java

License:Apache License

public void writeArray(ArrayWritable o, RType type) throws IOException {

    int typecode = type.code;
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    RTypedBytesWritableOutput rtbo = RTypedBytesWritableOutput.get(new DataOutputStream(baos));

    for (Writable w : o.get()) {
        rtbo.writeRaw(w, type);/* ww w . ja  v  a2s.  co m*/
    }
    out.writeArray(typecode, baos.toByteArray());
}

---

From source file:com.mozilla.hadoop.hbase.mapreduce.MultiScanTableMapReduceUtil.java

License:Apache License

/**
 * Converts base64 scans string back into a Scan array
 * @param base64/*from  w w  w.j  a  v a 2s.co  m*/
 * @return
 * @throws IOException
 */
public static Scan[] convertStringToScanArray(final String base64) throws IOException {
    final DataInputStream dis = new DataInputStream(new ByteArrayInputStream(Base64.decode(base64)));

    ArrayWritable aw = new ArrayWritable(Scan.class);
    aw.readFields(dis);

    Writable[] writables = aw.get();
    Scan[] scans = new Scan[writables.length];
    for (int i = 0; i < writables.length; i++) {
        scans[i] = (Scan) writables[i];
    }

    return scans;
}

---

From source file:com.stratio.deep.es.utils.UtilES.java

License:Apache License

private static <T> Object subDocumentListCase(Type type, ArrayWritable arrayWritable)
        throws IllegalAccessException, InstantiationException, InvocationTargetException,
        NoSuchMethodException {// w  w  w.ja  v  a  2s  .  com
    ParameterizedType listType = (ParameterizedType) type;

    Class<?> listClass = (Class<?>) listType.getActualTypeArguments()[0];

    List list = new ArrayList();
    Writable[] writetable = arrayWritable.get();

    for (int i = 0; i < writetable.length; i++) {
        list.add(getObjectFromJson(listClass, (LinkedMapWritable) writetable[i]));
    }

    return list;
}

---

From source file:com.uber.hoodie.common.HoodieMergeOnReadTestUtils.java

License:Apache License

public static List<GenericRecord> getRecordsUsingInputFormat(List<String> inputPaths, String basePath)
        throws IOException {
    JobConf jobConf = new JobConf();
    Schema schema = HoodieAvroUtils.addMetadataFields(Schema.parse(TRIP_EXAMPLE_SCHEMA));
    HoodieRealtimeInputFormat inputFormat = new HoodieRealtimeInputFormat();
    setPropsForInputFormat(inputFormat, jobConf, schema, basePath);
    return inputPaths.stream().map(path -> {
        setInputPath(jobConf, path);//from  w w w  . jav a  2 s .com
        List<GenericRecord> records = new ArrayList<>();
        try {
            List<InputSplit> splits = Arrays.asList(inputFormat.getSplits(jobConf, 1));
            RecordReader recordReader = inputFormat.getRecordReader(splits.get(0), jobConf, null);
            Void key = (Void) recordReader.createKey();
            ArrayWritable writable = (ArrayWritable) recordReader.createValue();
            while (recordReader.next(key, writable)) {
                GenericRecordBuilder newRecord = new GenericRecordBuilder(schema);
                // writable returns an array with [field1, field2, _hoodie_commit_time,
                // _hoodie_commit_seqno]
                Writable[] values = writable.get();
                schema.getFields().forEach(field -> {
                    newRecord.set(field, values[2]);
                });
                records.add(newRecord.build());
            }
        } catch (IOException ie) {
            ie.printStackTrace();
        }
        return records;
    }).reduce((a, b) -> {
        a.addAll(b);
        return a;
    }).get();
}

---

From source file:com.uber.hoodie.hadoop.HoodieInputFormatTest.java

License:Apache License

private void ensureRecordsInCommit(String msg, String commit, int expectedNumberOfRecordsInCommit,
        int totalExpected) throws IOException {
    int actualCount = 0;
    int totalCount = 0;
    InputSplit[] splits = inputFormat.getSplits(jobConf, 1);
    for (InputSplit split : splits) {
        RecordReader<NullWritable, ArrayWritable> recordReader = inputFormat.getRecordReader(split, jobConf,
                null);/*from  w  w w .  ja  v a  2 s .  com*/
        NullWritable key = recordReader.createKey();
        ArrayWritable writable = recordReader.createValue();

        while (recordReader.next(key, writable)) {
            // writable returns an array with [field1, field2, _hoodie_commit_time,
            // _hoodie_commit_seqno]
            // Take the commit time and compare with the one we are interested in
            if (commit.equals((writable.get()[2]).toString())) {
                actualCount++;
            }
            totalCount++;
        }
    }
    assertEquals(msg, expectedNumberOfRecordsInCommit, actualCount);
    assertEquals(msg, totalExpected, totalCount);
}

---

From source file:com.uber.hoodie.hadoop.realtime.AbstractRealtimeRecordReader.java

License:Apache License

/**
 * Prints a JSON representation of the ArrayWritable for easier debuggability
 *//* w  w  w.  ja v a2 s. c  om*/
protected static String arrayWritableToString(ArrayWritable writable) {
    if (writable == null) {
        return "null";
    }
    StringBuilder builder = new StringBuilder();
    Writable[] values = writable.get();
    builder.append("\"values_" + Math.random() + "_" + values.length + "\": {");
    int i = 0;
    for (Writable w : values) {
        if (w instanceof ArrayWritable) {
            builder.append(arrayWritableToString((ArrayWritable) w)).append(",");
        } else {
            builder.append("\"value" + i + "\":" + "\"" + w + "\"").append(",");
            if (w == null) {
                builder.append("\"type" + i + "\":" + "\"unknown\"").append(",");
            } else {
                builder.append("\"type" + i + "\":" + "\"" + w.getClass().getSimpleName() + "\"").append(",");
            }
        }
        i++;
    }
    builder.deleteCharAt(builder.length() - 1);
    builder.append("}");
    return builder.toString();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• »