Example usage for org.apache.hadoop.io Writable write

Introduction

In this page you can find the example usage for org.apache.hadoop.io Writable write.

Prototype

void write(DataOutput out) throws IOException;

Source Link

Document

Serialize the fields of this object to out.

Usage

From source file:gaffer.utils.WritableToStringConverter.java

License:Apache License

public static String serialiseToString(Writable writable) throws IOException {
    // Serialise to a byte array
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream out = new DataOutputStream(baos);
    Text.writeString(out, writable.getClass().getName());
    writable.write(out);

    // Convert the byte array to a base64 string
    return Base64.encodeBase64String(baos.toByteArray());
}

From source file:gobblin.util.HadoopUtils.java

License:Apache License

/**
 * Serialize a {@link Writable} object into a string.
 *
 * @param writable the {@link Writable} object to be serialized
 * @return a string serialized from the {@link Writable} object
 * @throws IOException if there's something wrong with the serialization
 *//*from www .  j a v  a  2s.c o m*/
public static String serializeToString(Writable writable) throws IOException {
    try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
            DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream)) {
        writable.write(dataOutputStream);
        return BaseEncoding.base64().encode(byteArrayOutputStream.toByteArray());
    }
}

From source file:io.aos.hdfs.WritableTestBase.java

License:Apache License

public static byte[] serialize(Writable writable) throws IOException {
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    DataOutputStream dataOut = new DataOutputStream(out);
    writable.write(dataOut);
    dataOut.close();//from  w w w  .  j av a2 s.  c  o  m
    return out.toByteArray();
}

From source file:it.crs4.pydoop.mapreduce.pipes.BinaryProtocol.java

License:Apache License

/**
 * Write the given object to the stream. If it is a Text or BytesWritable,
 * write it directly. Otherwise, write it to a buffer and then write the
 * length and data to the stream.//from  w w  w  .  j a v  a2s .  co  m
 * @param obj the object to write
 * @throws IOException
 */
private void writeObject(Writable obj) throws IOException {
    // For Text and BytesWritable, encode them directly, so that they end up
    // in C++ as the natural translations.
    if (obj instanceof Text) {
        Text t = (Text) obj;
        int len = t.getLength();
        WritableUtils.writeVInt(stream, len);
        stream.write(t.getBytes(), 0, len);
    } else if (obj instanceof BytesWritable) {
        BytesWritable b = (BytesWritable) obj;
        int len = b.getLength();
        WritableUtils.writeVInt(stream, len);
        stream.write(b.getBytes(), 0, len);
    } else if (obj == null) {
        // write a zero length string
        WritableUtils.writeVInt(stream, 0);
    } else {
        buffer.reset();
        obj.write(buffer);
        int length = buffer.getLength();
        WritableUtils.writeVInt(stream, length);
        stream.write(buffer.getData(), 0, length);
    }
}

From source file:it.crs4.pydoop.mapreduce.pipes.CommonStub.java

License:Apache License

protected void writeObject(Writable obj, DataOutputStream stream) throws IOException {
    // For Text and BytesWritable, encode them directly, so that they end up
    // in C++ as the natural translations.
    System.err.println("obj: " + obj);

    DataOutputBuffer buffer = new DataOutputBuffer();
    if (obj instanceof Text) {
        Text t = (Text) obj;
        int len = t.getLength();
        WritableUtils.writeVLong(stream, len);
        stream.flush();/*from  ww w  .  j  av a2  s  .c  o  m*/

        stream.write(t.getBytes(), 0, len);
        stream.flush();
        System.err.println("len: " + len);

    } else if (obj instanceof BytesWritable) {
        BytesWritable b = (BytesWritable) obj;
        int len = b.getLength();
        WritableUtils.writeVLong(stream, len);
        stream.write(b.getBytes(), 0, len);
        System.err.println("len: " + len);
    } else {
        buffer.reset();
        obj.write(buffer);
        int length = buffer.getLength();
        WritableUtils.writeVInt(stream, length);
        stream.write(buffer.getData(), 0, length);
        System.err.println("len: " + length);
    }
    stream.flush();

}

From source file:it.crs4.pydoop.pipes.BinaryProtocol.java

License:Apache License

/**
 * Write the given object to the stream. If it is a Text or BytesWritable,
 * write it directly. Otherwise, write it to a buffer and then write the
 * length and data to the stream./*from ww  w  . java 2 s .c  o  m*/
 * @param obj the object to write
 * @throws IOException
 */
private void writeObject(Writable obj) throws IOException {
    // For Text and BytesWritable, encode them directly, so that they end up
    // in C++ as the natural translations.
    if (obj instanceof Text) {
        Text t = (Text) obj;
        int len = t.getLength();
        WritableUtils.writeVInt(stream, len);
        stream.write(t.getBytes(), 0, len);
    } else if (obj instanceof BytesWritable) {
        BytesWritable b = (BytesWritable) obj;
        int len = b.getLength();
        WritableUtils.writeVInt(stream, len);
        stream.write(b.getBytes(), 0, len);
    } else {
        buffer.reset();
        obj.write(buffer);
        int length = buffer.getLength();
        WritableUtils.writeVInt(stream, length);
        stream.write(buffer.getData(), 0, length);
    }
}

From source file:it.crs4.seal.common.TestContext.java

License:Open Source License

/**
 * Duplicate Writable object by by serializing and then unserializing it.
 *//*  w ww . java2 s. com*/
private Object duplicateWritable(Object oldItem) throws java.io.IOException {
    if (oldItem == null)
        return null;

    Writable w = (Writable) oldItem;
    // duplicate the key by serializing and then unserializing it
    ByteArrayOutputStream obytes = new ByteArrayOutputStream();
    DataOutputStream ostream = new DataOutputStream(obytes);

    w.write(ostream);
    ostream.close();

    Object newItem;
    try {
        newItem = oldItem.getClass().newInstance();
    } catch (Exception e) {
        throw new RuntimeException("error instantiating duplicate key or value.  Message: " + e.getMessage());
    }

    ByteArrayInputStream ibytes = new ByteArrayInputStream(obytes.toByteArray());
    DataInputStream istream = new DataInputStream(ibytes);
    ((Writable) newItem).readFields(istream);
    return newItem;
}

From source file:ml.shifu.guagua.mapreduce.GuaguaInputSplitTest.java

License:Apache License

public byte[] objectToBytes(Writable result) {
    ByteArrayOutputStream out = null;
    DataOutputStream dataOut = null;
    try {//w w  w .  j av a2 s.c o  m
        out = new ByteArrayOutputStream();
        dataOut = new DataOutputStream(out);
        result.write(dataOut);
    } catch (IOException e) {
        throw new GuaguaRuntimeException(e);
    } finally {
        if (dataOut != null) {
            try {
                dataOut.close();
            } catch (IOException e) {
                throw new GuaguaRuntimeException(e);
            }
        }
    }
    return out.toByteArray();
}

From source file:net.sf.katta.lib.lucene.Hit.java

License:Apache License

@Override
public void write(final DataOutput out) throws IOException {
    out.writeFloat(_score);/* w w  w .ja va 2 s . c  o m*/
    if (_node != null) {
        out.writeBoolean(true);
        _node.write(out);
    } else {
        out.writeBoolean(false);
    }
    _shard.write(out);
    out.writeInt(_docId);
    if (_sortFields == null) {
        out.writeByte(0);
    } else {
        out.writeByte(_sortFields.length);
        for (Writable writable : _sortFields) {
            writable.write(out);
        }
    }
}

From source file:net.sf.katta.lib.lucene.HitsMapWritable.java

License:Apache License

public void write(final DataOutput out) throws IOException {
    long start = 0;
    if (LOG.isDebugEnabled()) {
        start = System.currentTimeMillis();
    }/*from www. ja v a  2  s .  c o  m*/
    out.writeUTF(_nodeName);
    out.writeInt(_totalHits);
    if (_sortFieldTypes == null) {
        out.writeByte(0);
    } else {
        out.writeByte(_sortFieldTypes.length);
        for (WritableType writableType : _sortFieldTypes) {
            out.writeByte(writableType.ordinal());
        }
    }
    int shardCount = _shards.size();
    out.writeInt(shardCount);
    byte shardIndex = 0;
    Map<String, Byte> shardIndexByShard = new HashMap<String, Byte>(shardCount);
    for (String shard : _shards) {
        out.writeUTF(shard);
        shardIndexByShard.put(shard, shardIndex);
        shardIndex++;
    }
    out.writeInt(_hits.size());
    for (Hit hit : _hits) {
        out.writeByte(shardIndexByShard.get(hit.getShard()));
        out.writeFloat(hit.getScore());
        out.writeInt(hit.getDocId());
        WritableComparable[] sortFields = hit.getSortFields();
        if (sortFields == null) {
            out.writeByte(0);
        } else {
            out.writeByte(sortFields.length);
            for (Writable writable : sortFields) {
                writable.write(out);
            }
        }
    }
    if (LOG.isDebugEnabled()) {
        final long end = System.currentTimeMillis();
        LOG.debug("HitsMap writing took " + (end - start) / 1000.0 + "sec.");
        LOG.debug("HitsMap writing ended at: " + end + " for server " + _nodeName);
    }
}