Example usage for org.apache.hadoop.io WritableUtils writeVInt

List of usage examples for org.apache.hadoop.io WritableUtils writeVInt

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.io.*
  5. WritableUtils
  6. writeVInt

Introduction

In this page you can find the example usage for org.apache.hadoop.io WritableUtils writeVInt.

Prototype

public static void writeVInt(DataOutput stream, int i) throws IOException

Source Link

Document

Serializes an integer to a binary stream with zero-compressed encoding.

Usage

From source file:it.crs4.pydoop.mapreduce.pipes.BinaryProtocol.java

License:Apache License

public void reduceValue(Writable value) throws IOException {
    WritableUtils.writeVInt(stream, MessageType.REDUCE_VALUE.code);
    writeObject(value);
}

From source file:it.crs4.pydoop.mapreduce.pipes.BinaryProtocol.java

License:Apache License

public void endOfInput() throws IOException {
    WritableUtils.writeVInt(stream, MessageType.CLOSE.code);
    LOG.debug("Sent close command");
}

From source file:it.crs4.pydoop.mapreduce.pipes.BinaryProtocol.java

License:Apache License

public void abort() throws IOException {
    WritableUtils.writeVInt(stream, MessageType.ABORT.code);
    LOG.debug("Sent abort command");
}

From source file:it.crs4.pydoop.mapreduce.pipes.BinaryProtocol.java

License:Apache License

/**
 * Write the given object to the stream. If it is a Text or BytesWritable,
 * write it directly. Otherwise, write it to a buffer and then write the
 * length and data to the stream.//from  w  ww. j  a  va2  s  .  c om
 * @param obj the object to write
 * @throws IOException
 */
private void writeObject(Writable obj) throws IOException {
    // For Text and BytesWritable, encode them directly, so that they end up
    // in C++ as the natural translations.
    if (obj instanceof Text) {
        Text t = (Text) obj;
        int len = t.getLength();
        WritableUtils.writeVInt(stream, len);
        stream.write(t.getBytes(), 0, len);
    } else if (obj instanceof BytesWritable) {
        BytesWritable b = (BytesWritable) obj;
        int len = b.getLength();
        WritableUtils.writeVInt(stream, len);
        stream.write(b.getBytes(), 0, len);
    } else if (obj == null) {
        // write a zero length string
        WritableUtils.writeVInt(stream, 0);
    } else {
        buffer.reset();
        obj.write(buffer);
        int length = buffer.getLength();
        WritableUtils.writeVInt(stream, length);
        stream.write(buffer.getData(), 0, length);
    }
}

From source file:it.crs4.pydoop.mapreduce.pipes.CommonStub.java

License:Apache License

protected void writeObject(Writable obj, DataOutputStream stream) throws IOException {
    // For Text and BytesWritable, encode them directly, so that they end up
    // in C++ as the natural translations.
    System.err.println("obj: " + obj);

    DataOutputBuffer buffer = new DataOutputBuffer();
    if (obj instanceof Text) {
        Text t = (Text) obj;
        int len = t.getLength();
        WritableUtils.writeVLong(stream, len);
        stream.flush();/*from   w  ww. ja v  a  2  s  .c om*/

        stream.write(t.getBytes(), 0, len);
        stream.flush();
        System.err.println("len: " + len);

    } else if (obj instanceof BytesWritable) {
        BytesWritable b = (BytesWritable) obj;
        int len = b.getLength();
        WritableUtils.writeVLong(stream, len);
        stream.write(b.getBytes(), 0, len);
        System.err.println("len: " + len);
    } else {
        buffer.reset();
        obj.write(buffer);
        int length = buffer.getLength();
        WritableUtils.writeVInt(stream, length);
        stream.write(buffer.getData(), 0, length);
        System.err.println("len: " + length);
    }
    stream.flush();

}

From source file:it.crs4.pydoop.mapreduce.pipes.CommonStub.java

License:Apache License

protected void initSoket() throws Exception {
    int port = Integer.parseInt(System.getenv("mapreduce.pipes.command.port"));

    System.err.println("port:" + port);

    java.net.InetAddress address = java.net.InetAddress.getLocalHost();

    socket = new Socket(address.getHostName(), port);
    InputStream input = socket.getInputStream();
    OutputStream output = socket.getOutputStream();

    System.err.println("ready to read");
    // try to read
    dataInput = new DataInputStream(input);

    WritableUtils.readVInt(dataInput);//from  ww w .  j  a  v  a2s  .  co  m

    String str = Text.readString(dataInput);

    Text.readString(dataInput);

    dataOut = new DataOutputStream(output);
    WritableUtils.writeVInt(dataOut, 57);
    String s = createDigest("password".getBytes(), str);

    Text.writeString(dataOut, s);

    // start
    WritableUtils.readVInt(dataInput);
    int cuttentAnswer = WritableUtils.readVInt(dataInput);
    System.out.println("CURRENT_PROTOCOL_VERSION:" + cuttentAnswer);

    // get configuration
    // should be MessageType.SET_JOB_CONF.code
    WritableUtils.readVInt(dataInput);

    // array length

    int j = WritableUtils.readVInt(dataInput);
    for (int i = 0; i < j; i++) {
        Text.readString(dataInput);
        i++;
        Text.readString(dataInput);
    }
}

From source file:it.crs4.pydoop.mapreduce.pipes.PipeApplicationRunnableStub.java

License:Apache License

public void binaryProtocolStub() {
    try {/*w w w .j av a  2s . c o  m*/

        initSoket();
        System.out.println("start OK");

        // RUN_MAP.code
        // should be 3

        int answer = WritableUtils.readVInt(dataInput);
        System.out.println("RunMap:" + answer);
        FileSplit split = new FileSplit();
        readObject(split, dataInput);

        WritableUtils.readVInt(dataInput);
        WritableUtils.readVInt(dataInput);
        // end runMap
        // get InputTypes
        WritableUtils.readVInt(dataInput);
        String inText = Text.readString(dataInput);
        System.out.println("Key class:" + inText);
        inText = Text.readString(dataInput);
        System.out.println("Value class:" + inText);

        @SuppressWarnings("unused")
        int inCode = 0;

        // read all data from sender and write to output
        while ((inCode = WritableUtils.readVInt(dataInput)) == 4) {
            FloatWritable key = new FloatWritable();
            NullWritable value = NullWritable.get();
            readObject(key, dataInput);
            System.out.println("value:" + key.get());
            readObject(value, dataInput);
        }

        WritableUtils.writeVInt(dataOut, 54);

        dataOut.flush();
        dataOut.close();

    } catch (Exception x) {
        x.printStackTrace();
    } finally {
        closeSoket();
    }

}

From source file:it.crs4.pydoop.mapreduce.pipes.PipeApplicationStub.java

License:Apache License

public void binaryProtocolStub() {
    try {//from w  w w .  j a v a  2  s.  co m

        System.err.println("binaryProtocolStub()");
        initSoket();
        System.err.println("done with socket()");

        // output code
        WritableUtils.writeVInt(dataOut, 50);
        IntWritable wt = new IntWritable();
        wt.set(123);
        writeObject(wt, dataOut);
        writeObject(new Text("value"), dataOut);

        //  PARTITIONED_OUTPUT
        WritableUtils.writeVInt(dataOut, 51);
        WritableUtils.writeVInt(dataOut, 0);
        writeObject(wt, dataOut);
        writeObject(new Text("value"), dataOut);

        // STATUS
        WritableUtils.writeVInt(dataOut, 52);
        Text.writeString(dataOut, "PROGRESS");
        dataOut.flush();

        // progress
        WritableUtils.writeVInt(dataOut, 53);
        dataOut.writeFloat(0.55f);
        // register counter
        WritableUtils.writeVInt(dataOut, 55);
        // id
        WritableUtils.writeVInt(dataOut, 0);
        Text.writeString(dataOut, "group");
        Text.writeString(dataOut, "name");
        // increment counter
        WritableUtils.writeVInt(dataOut, 56);
        WritableUtils.writeVInt(dataOut, 0);

        WritableUtils.writeVLong(dataOut, 2);

        // map item
        int intValue = WritableUtils.readVInt(dataInput);
        System.out.println("intValue:" + intValue);
        IntWritable iw = new IntWritable();
        readObject(iw, dataInput);
        System.out.println("key:" + iw.get());
        Text txt = new Text();
        readObject(txt, dataInput);
        System.out.println("value:" + txt.toString());

        // done
        // end of session
        WritableUtils.writeVInt(dataOut, 54);

        System.out.println("finish");
        dataOut.flush();
        dataOut.close();

    } catch (Exception x) {
        x.printStackTrace();
    } finally {
        closeSoket();
    }
}

From source file:it.crs4.pydoop.mapreduce.pipes.PipeReducerStub.java

License:Apache License

public void binaryProtocolStub() {
    try {/*w  ww .  jav  a  2s.  com*/

        initSoket();

        //should be 5
        //RUN_REDUCE boolean 
        WritableUtils.readVInt(dataInput);
        WritableUtils.readVInt(dataInput);
        int intValue = WritableUtils.readVInt(dataInput);
        System.out.println("getIsJavaRecordWriter:" + intValue);

        // reduce key
        WritableUtils.readVInt(dataInput);
        // value of reduce key
        BooleanWritable value = new BooleanWritable();
        readObject(value, dataInput);
        System.out.println("reducer key :" + value);
        // reduce value code:

        // reduce values
        while ((intValue = WritableUtils.readVInt(dataInput)) == 7) {
            Text txt = new Text();
            // value
            readObject(txt, dataInput);
            System.out.println("reduce value  :" + txt);
        }

        // done
        WritableUtils.writeVInt(dataOut, 54);

        dataOut.flush();
        dataOut.close();

    } catch (Exception x) {
        x.printStackTrace();
    } finally {
        closeSoket();

    }
}

From source file:it.crs4.pydoop.pipes.BinaryProtocol.java

License:Apache License

public void setJobConf(JobConf job) throws IOException {
    WritableUtils.writeVInt(stream, MessageType.SET_JOB_CONF.code);
    List<String> list = new ArrayList<String>();
    for (Map.Entry<String, String> itm : job) {
        list.add(itm.getKey());//w w  w.ja  v  a 2  s .  c o m
        list.add(itm.getValue());
    }
    WritableUtils.writeVInt(stream, list.size());
    for (String entry : list) {
        Text.writeString(stream, entry);
    }
}