Example usage for org.apache.hadoop.io WritableUtils readVInt

Introduction

In this page you can find the example usage for org.apache.hadoop.io WritableUtils readVInt.

Prototype

public static int readVInt(DataInput stream) throws IOException

Source Link

Document

Reads a zero-compressed encoded integer from input stream and returns it.

Usage

From source file:org.apache.crunch.types.writable.UnionWritable.java

License:Apache License

@Override
public void readFields(DataInput in) throws IOException {
    this.index = WritableUtils.readVInt(in);
    if (value == null) {
        value = new BytesWritable();
    }/* w w  w.j  a  v  a 2  s . c  o m*/
    value.readFields(in);
}

From source file:org.apache.druid.indexer.InputRowSerde.java

License:Apache License

private static byte[] readBytes(DataInput in) throws IOException {
    int size = WritableUtils.readVInt(in);
    byte[] result = new byte[size];
    in.readFully(result, 0, size);/*w w  w . jav a  2s  . co  m*/
    return result;
}

From source file:org.apache.druid.indexer.InputRowSerde.java

License:Apache License

private static List<String> readStringArray(DataInput in) throws IOException {
    int count = WritableUtils.readVInt(in);
    if (count == 0) {
        return null;
    }//from  ww  w  .j a  v  a 2  s.c  o  m
    List<String> values = Lists.newArrayListWithCapacity(count);
    for (int i = 0; i < count; i++) {
        values.add(readString(in));
    }
    return values;
}

From source file:org.apache.druid.indexer.InputRowSerde.java

License:Apache License

public static final InputRow fromBytes(final Map<String, IndexSerdeTypeHelper> typeHelperMap, byte[] data,
        AggregatorFactory[] aggs) {/*from   www.  jav a  2 s  . co m*/
    try {
        ByteArrayDataInput in = ByteStreams.newDataInput(data);

        //Read timestamp
        long timestamp = in.readLong();

        Map<String, Object> event = new HashMap<>();

        //Read dimensions
        List<String> dimensions = new ArrayList<>();
        int dimNum = WritableUtils.readVInt(in);
        for (int i = 0; i < dimNum; i++) {
            String dimension = readString(in);
            dimensions.add(dimension);

            IndexSerdeTypeHelper typeHelper = typeHelperMap.get(dimension);
            if (typeHelper == null) {
                typeHelper = STRING_HELPER;
            }
            Object dimValues = typeHelper.deserialize(in);
            if (dimValues == null) {
                continue;
            }

            if (typeHelper.getType() == ValueType.STRING) {
                List<String> dimensionValues = (List<String>) dimValues;
                if (dimensionValues.size() == 1) {
                    event.put(dimension, dimensionValues.get(0));
                } else {
                    event.put(dimension, dimensionValues);
                }
            } else {
                event.put(dimension, dimValues);
            }
        }

        //Read metrics
        int metricSize = WritableUtils.readVInt(in);
        for (int i = 0; i < metricSize; i++) {
            String metric = readString(in);
            String type = getType(metric, aggs, i);
            byte metricNullability = in.readByte();
            if (metricNullability == NullHandling.IS_NULL_BYTE) {
                // metric value is null.
                continue;
            }
            if ("float".equals(type)) {
                event.put(metric, in.readFloat());
            } else if ("long".equals(type)) {
                event.put(metric, WritableUtils.readVLong(in));
            } else if ("double".equals(type)) {
                event.put(metric, in.readDouble());
            } else {
                ComplexMetricSerde serde = getComplexMetricSerde(type);
                byte[] value = readBytes(in);
                event.put(metric, serde.fromBytes(value, 0, value.length));
            }
        }

        return new MapBasedInputRow(timestamp, dimensions, event);
    } catch (IOException ex) {
        throw new RuntimeException(ex);
    }
}

From source file:org.apache.fluo.accumulo.util.ByteArrayUtil.java

License:Apache License

public static final List<Bytes> split(byte[] b) {

    ArrayList<Bytes> ret = new ArrayList<>();

    try (InputStream in = new ByteArrayInputStream(b)) {
        DataInputStream dis = new DataInputStream(in);

        BytesBuilder builder = Bytes.builder(b.length);

        while (true) {
            int len = WritableUtils.readVInt(dis);
            builder.append(dis, len);//www. j  av  a  2 s  . c  o m
            ret.add(builder.toBytes());
            builder.setLength(0);
        }
    } catch (EOFException ee) {
        // at end of file
    } catch (IOException e) {
        throw new RuntimeException(e);
    }

    return ret;
}

From source file:org.apache.fluo.core.impl.Environment.java

License:Apache License

private static Map<Column, ObserverConfiguration> readObservers(DataInputStream dis) throws IOException {

    HashMap<Column, ObserverConfiguration> omap = new HashMap<>();

    int num = WritableUtils.readVInt(dis);
    for (int i = 0; i < num; i++) {
        Column col = ColumnUtil.readColumn(dis);
        String clazz = dis.readUTF();
        Map<String, String> params = new HashMap<>();
        int numParams = WritableUtils.readVInt(dis);
        for (int j = 0; j < numParams; j++) {
            String k = dis.readUTF();
            String v = dis.readUTF();
            params.put(k, v);//from w w  w .j  a v a 2s .  c  o m
        }

        ObserverConfiguration observerConfig = new ObserverConfiguration(clazz);
        observerConfig.setParameters(params);

        omap.put(col, observerConfig);
    }

    return omap;
}

From source file:org.apache.fluo.core.observer.v1.ObserverStoreV1.java

License:Apache License

private static Map<Column, org.apache.fluo.api.config.ObserverSpecification> readObservers(DataInputStream dis)
        throws IOException {

    ImmutableMap.Builder<Column, org.apache.fluo.api.config.ObserverSpecification> omapBuilder = new ImmutableMap.Builder<>();

    int num = WritableUtils.readVInt(dis);
    for (int i = 0; i < num; i++) {
        Column col = ColumnUtil.readColumn(dis);
        String clazz = dis.readUTF();
        Map<String, String> params = new HashMap<>();
        int numParams = WritableUtils.readVInt(dis);
        for (int j = 0; j < numParams; j++) {
            String k = dis.readUTF();
            String v = dis.readUTF();
            params.put(k, v);/*from  w  ww.j a  v a2s  .  c o  m*/
        }

        org.apache.fluo.api.config.ObserverSpecification ospec = new org.apache.fluo.api.config.ObserverSpecification(
                clazz, params);
        omapBuilder.put(col, ospec);
    }
    return omapBuilder.build();
}

From source file:org.apache.gora.filter.MapFieldValueFilter.java

License:Apache License

@Override
public void readFields(DataInput in) throws IOException {
    fieldName = Text.readString(in);
    mapKey = new Utf8(Text.readString(in));
    filterOp = WritableUtils.readEnum(in, FilterOp.class);
    operands.clear();//  w  w  w  .  ja va2s .  co m
    int operandsSize = WritableUtils.readVInt(in);
    for (int i = 0; i < operandsSize; i++) {
        Object operand = ObjectWritable.readObject(in, conf);
        if (operand instanceof String) {
            operand = new Utf8((String) operand);
        }
        operands.add(operand);
    }
    filterIfMissing = in.readBoolean();
}

From source file:org.apache.gora.filter.SingleFieldValueFilter.java

License:Apache License

@Override
public void readFields(DataInput in) throws IOException {
    fieldName = Text.readString(in);
    filterOp = WritableUtils.readEnum(in, FilterOp.class);
    operands.clear();/*from  w  w w.  jav  a 2  s .c o  m*/
    int operandsSize = WritableUtils.readVInt(in);
    for (int i = 0; i < operandsSize; i++) {
        Object operand = ObjectWritable.readObject(in, conf);
        if (operand instanceof String) {
            operand = new Utf8((String) operand);
        }
        operands.add(operand);
    }
    filterIfMissing = in.readBoolean();
}

From source file:org.apache.gora.util.IOUtils.java

License:Apache License

/** Deserializes the object in the given datainput using
 * available Hadoop serializations./*from w w w.  j a v a  2s .  co m*/
 * @throws IOException */
public static <T> T deserialize(Configuration conf, DataInput in, T obj, Class<T> objClass) throws IOException {
    SerializationFactory serializationFactory = new SerializationFactory(getOrCreateConf(conf));
    Deserializer<T> deserializer = serializationFactory.getDeserializer(objClass);

    int length = WritableUtils.readVInt(in);
    byte[] arr = new byte[length];
    in.readFully(arr);
    List<ByteBuffer> list = new ArrayList<ByteBuffer>();
    list.add(ByteBuffer.wrap(arr));
    ByteBufferInputStream is = new ByteBufferInputStream(list);

    try {
        deserializer.open(is);
        T newObj = deserializer.deserialize(obj);
        return newObj;

    } finally {
        if (deserializer != null)
            deserializer.close();
        if (is != null)
            is.close();
    }
}