Example usage for org.apache.hadoop.io WritableUtils writeVInt

Introduction

In this page you can find the example usage for org.apache.hadoop.io WritableUtils writeVInt.

Prototype

public static void writeVInt(DataOutput stream, int i) throws IOException 

Source Link

Document

Serializes an integer to a binary stream with zero-compressed encoding.

Usage

From source file:org.apache.druid.indexer.InputRowSerde.java

License:Apache License

public static final SerializeResult toBytes(final Map<String, IndexSerdeTypeHelper> typeHelperMap,
        final InputRow row, AggregatorFactory[] aggs) {
    try {// www .  j  a va2 s. com
        List<String> parseExceptionMessages = new ArrayList<>();
        ByteArrayDataOutput out = ByteStreams.newDataOutput();

        //write timestamp
        out.writeLong(row.getTimestampFromEpoch());

        //writing all dimensions
        List<String> dimList = row.getDimensions();

        WritableUtils.writeVInt(out, dimList.size());
        for (String dim : dimList) {
            IndexSerdeTypeHelper typeHelper = typeHelperMap.get(dim);
            if (typeHelper == null) {
                typeHelper = STRING_HELPER;
            }
            writeString(dim, out);

            try {
                typeHelper.serialize(out, row.getRaw(dim));
            } catch (ParseException pe) {
                parseExceptionMessages.add(pe.getMessage());
            }
        }

        //writing all metrics
        Supplier<InputRow> supplier = () -> row;
        WritableUtils.writeVInt(out, aggs.length);
        for (AggregatorFactory aggFactory : aggs) {
            String k = aggFactory.getName();
            writeString(k, out);

            try (Aggregator agg = aggFactory.factorize(IncrementalIndex
                    .makeColumnSelectorFactory(VirtualColumns.EMPTY, aggFactory, supplier, true))) {
                try {
                    agg.aggregate();
                } catch (ParseException e) {
                    // "aggregate" can throw ParseExceptions if a selector expects something but gets something else.
                    log.debug(e, "Encountered parse error, skipping aggregator[%s].", k);
                    parseExceptionMessages.add(e.getMessage());
                }

                String t = aggFactory.getTypeName();
                if (agg.isNull()) {
                    out.writeByte(NullHandling.IS_NULL_BYTE);
                } else {
                    out.writeByte(NullHandling.IS_NOT_NULL_BYTE);
                    if ("float".equals(t)) {
                        out.writeFloat(agg.getFloat());
                    } else if ("long".equals(t)) {
                        WritableUtils.writeVLong(out, agg.getLong());
                    } else if ("double".equals(t)) {
                        out.writeDouble(agg.getDouble());
                    } else {
                        //its a complex metric
                        Object val = agg.get();
                        ComplexMetricSerde serde = getComplexMetricSerde(t);
                        writeBytes(serde.toBytes(val), out);
                    }
                }
            }
        }

        return new SerializeResult(out.toByteArray(), parseExceptionMessages);
    } catch (IOException ex) {
        throw new RuntimeException(ex);
    }
}

---

From source file:org.apache.druid.indexer.InputRowSerde.java

License:Apache License

private static void writeBytes(@Nullable byte[] value, ByteArrayDataOutput out) throws IOException {
    int length = value == null ? -1 : value.length;
    WritableUtils.writeVInt(out, length);
    if (value != null) {
        out.write(value, 0, value.length);
    }/*from  w  ww  . java2s . c o m*/
}

---

From source file:org.apache.druid.indexer.InputRowSerde.java

License:Apache License

private static void writeStringArray(List<String> values, ByteArrayDataOutput out) throws IOException {
    if (values == null || values.size() == 0) {
        WritableUtils.writeVInt(out, 0);
        return;//from ww  w. jav  a2 s .c  o m
    }
    WritableUtils.writeVInt(out, values.size());
    for (String value : values) {
        writeString(value, out);
    }
}

---

From source file:org.apache.fluo.core.client.Operations.java

License:Apache License

private static void serializeObservers(DataOutputStream dos, Map<Column, ObserverConfiguration> colObservers)
        throws IOException {
    // TODO use a human readable serialized format like json

    Set<Entry<Column, ObserverConfiguration>> es = colObservers.entrySet();

    WritableUtils.writeVInt(dos, colObservers.size());

    for (Entry<Column, ObserverConfiguration> entry : es) {
        ColumnUtil.writeColumn(entry.getKey(), dos);
        dos.writeUTF(entry.getValue().getClassName());
        Map<String, String> params = entry.getValue().getParameters();
        WritableUtils.writeVInt(dos, params.size());
        for (Entry<String, String> pentry : entry.getValue().getParameters().entrySet()) {
            dos.writeUTF(pentry.getKey());
            dos.writeUTF(pentry.getValue());
        }/*  w  w  w . jav a 2s.  c  o  m*/
    }
}

---

From source file:org.apache.fluo.core.observer.v1.ObserverStoreV1.java

License:Apache License

private static void serializeObservers(DataOutputStream dos,
        Map<Column, org.apache.fluo.api.config.ObserverSpecification> colObservers) throws IOException {
    // TODO use a human readable serialized format like json

    Set<Entry<Column, org.apache.fluo.api.config.ObserverSpecification>> es = colObservers.entrySet();

    WritableUtils.writeVInt(dos, colObservers.size());

    for (Entry<Column, org.apache.fluo.api.config.ObserverSpecification> entry : es) {
        ColumnUtil.writeColumn(entry.getKey(), dos);
        dos.writeUTF(entry.getValue().getClassName());
        Map<String, String> params = entry.getValue().getConfiguration().toMap();
        WritableUtils.writeVInt(dos, params.size());
        for (Entry<String, String> pentry : params.entrySet()) {
            dos.writeUTF(pentry.getKey());
            dos.writeUTF(pentry.getValue());
        }/*from  w  ww.  ja  v a 2s . c  o  m*/
    }
}

---

From source file:org.apache.gora.filter.MapFieldValueFilter.java

License:Apache License

@Override
public void write(DataOutput out) throws IOException {
    Text.writeString(out, fieldName);
    Text.writeString(out, mapKey.toString());
    WritableUtils.writeEnum(out, filterOp);
    WritableUtils.writeVInt(out, operands.size());
    for (int i = 0; i < operands.size(); i++) {
        Object operand = operands.get(i);
        if (operand instanceof String) {
            throw new IllegalStateException("Use Utf8 instead of String for operands");
        }//from  ww w.ja  va 2 s . com
        if (operand instanceof Utf8) {
            operand = operand.toString();
        }
        if (operand instanceof Boolean) {
            ObjectWritable.writeObject(out, operand, Boolean.TYPE, conf);
        } else if (operand instanceof Character) {
            ObjectWritable.writeObject(out, operand, Character.TYPE, conf);
        } else if (operand instanceof Byte) {
            ObjectWritable.writeObject(out, operand, Byte.TYPE, conf);
        } else if (operand instanceof Short) {
            ObjectWritable.writeObject(out, operand, Short.TYPE, conf);
        } else if (operand instanceof Integer) {
            ObjectWritable.writeObject(out, operand, Integer.TYPE, conf);
        } else if (operand instanceof Long) {
            ObjectWritable.writeObject(out, operand, Long.TYPE, conf);
        } else if (operand instanceof Float) {
            ObjectWritable.writeObject(out, operand, Float.TYPE, conf);
        } else if (operand instanceof Double) {
            ObjectWritable.writeObject(out, operand, Double.TYPE, conf);
        } else if (operand instanceof Void) {
            ObjectWritable.writeObject(out, operand, Void.TYPE, conf);
        } else {
            ObjectWritable.writeObject(out, operand, operand.getClass(), conf);
        }
    }
    out.writeBoolean(filterIfMissing);
}

---

From source file:org.apache.gora.filter.SingleFieldValueFilter.java

License:Apache License

@Override
public void write(DataOutput out) throws IOException {
    Text.writeString(out, fieldName);
    WritableUtils.writeEnum(out, filterOp);
    WritableUtils.writeVInt(out, operands.size());
    for (int i = 0; i < operands.size(); i++) {
        Object operand = operands.get(i);
        if (operand instanceof String) {
            throw new IllegalStateException("Use Utf8 instead of String for operands");
        }//w w  w .j  a  v  a  2 s. c  om
        if (operand instanceof Utf8) {
            operand = operand.toString();
        }
        if (operand instanceof Boolean) {
            ObjectWritable.writeObject(out, operand, Boolean.TYPE, conf);
        } else if (operand instanceof Character) {
            ObjectWritable.writeObject(out, operand, Character.TYPE, conf);
        } else if (operand instanceof Byte) {
            ObjectWritable.writeObject(out, operand, Byte.TYPE, conf);
        } else if (operand instanceof Short) {
            ObjectWritable.writeObject(out, operand, Short.TYPE, conf);
        } else if (operand instanceof Integer) {
            ObjectWritable.writeObject(out, operand, Integer.TYPE, conf);
        } else if (operand instanceof Long) {
            ObjectWritable.writeObject(out, operand, Long.TYPE, conf);
        } else if (operand instanceof Float) {
            ObjectWritable.writeObject(out, operand, Float.TYPE, conf);
        } else if (operand instanceof Double) {
            ObjectWritable.writeObject(out, operand, Double.TYPE, conf);
        } else if (operand instanceof Void) {
            ObjectWritable.writeObject(out, operand, Void.TYPE, conf);
        } else {
            ObjectWritable.writeObject(out, operand, operand.getClass(), conf);
        }
    }
    out.writeBoolean(filterIfMissing);
}

---

From source file:org.apache.gora.util.IOUtils.java

License:Apache License

/** Serializes the object to the given dataoutput using
 * available Hadoop serializations/*w  w  w .ja  va  2s .  c  o m*/
 * @throws IOException */
public static <T> void serialize(Configuration conf, DataOutput out, T obj, Class<T> objClass)
        throws IOException {

    SerializationFactory serializationFactory = new SerializationFactory(getOrCreateConf(conf));
    Serializer<T> serializer = serializationFactory.getSerializer(objClass);

    ByteBufferOutputStream os = new ByteBufferOutputStream();
    try {
        serializer.open(os);
        serializer.serialize(obj);

        int length = 0;
        List<ByteBuffer> buffers = os.getBufferList();
        for (ByteBuffer buffer : buffers) {
            length += buffer.limit() - buffer.arrayOffset();
        }

        WritableUtils.writeVInt(out, length);
        for (ByteBuffer buffer : buffers) {
            byte[] arr = buffer.array();
            out.write(arr, buffer.arrayOffset(), buffer.limit());
        }

    } finally {
        if (serializer != null)
            serializer.close();
        if (os != null)
            os.close();
    }
}

---

From source file:org.apache.gora.util.IOUtils.java

License:Apache License

/**
 * Writes a boolean[] to the output./*from  ww  w.  java 2s .  c  o m*/
 */
public static void writeBoolArray(DataOutput out, boolean[] boolArray) throws IOException {

    WritableUtils.writeVInt(out, boolArray.length);

    byte b = 0;
    int i = 0;
    for (i = 0; i < boolArray.length; i++) {
        if (i % 8 == 0 && i != 0) {
            out.writeByte(b);
            b = 0;
        }
        b >>= 1;
        if (boolArray[i])
            b |= 0x80;
        else
            b &= 0x7F;
    }
    if (i % 8 != 0) {
        for (int j = 0; j < 8 - (i % 8); j++) { //shift for the remaining byte
            b >>= 1;
            b &= 0x7F;
        }
    }

    out.writeByte(b);
}

---

From source file:org.apache.gora.util.IOUtils.java

License:Apache License

/**
 * Writes the String array to the given DataOutput.
 * @param out the data output to write to
 * @param arr the array to write// w w  w.j  a v  a  2 s .  c o m
 * @see #readStringArray(DataInput)
 */
public static void writeStringArray(DataOutput out, String[] arr) throws IOException {
    WritableUtils.writeVInt(out, arr.length);
    for (String str : arr) {
        Text.writeString(out, str);
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• »