Example usage for org.apache.hadoop.io WritableUtils readVInt

Introduction

In this page you can find the example usage for org.apache.hadoop.io WritableUtils readVInt.

Prototype

public static int readVInt(DataInput stream) throws IOException 

Source Link

Document

Reads a zero-compressed encoded integer from input stream and returns it.

Usage

From source file:io.fluo.api.data.Bytes.java

License:Apache License

/**
 * Splits a bytes object into several bytes objects
 * // w  w  w . j  a  v a 2s.  c o m
 * @param b Original bytes object
 * @return List of bytes objects
 */
public static List<Bytes> split(Bytes b) {
    ByteArrayInputStream bais;
    if (b.isBackedByArray())
        bais = new ByteArrayInputStream(b.getBackingArray(), b.offset(), b.length());
    else
        bais = new ByteArrayInputStream(b.toArray());

    DataInputStream dis = new DataInputStream(bais);

    ArrayList<Bytes> ret = new ArrayList<>();

    try {
        while (true) {
            int len = WritableUtils.readVInt(dis);
            // TODO could get pointers into original byte seq
            byte field[] = new byte[len];
            dis.readFully(field);
            ret.add(wrap(field));
        }
    } catch (EOFException ee) {

    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    return ret;
}

---

From source file:io.fluo.core.impl.Environment.java

License:Apache License

private static Map<Column, ObserverConfiguration> readObservers(DataInputStream dis) throws IOException {

    HashMap<Column, ObserverConfiguration> omap = new HashMap<>();

    int num = WritableUtils.readVInt(dis);
    for (int i = 0; i < num; i++) {
        Column col = new Column();
        col.readFields(dis);//from   w  w  w .jav  a 2s.c o m
        String clazz = dis.readUTF();
        Map<String, String> params = new HashMap<>();
        int numParams = WritableUtils.readVInt(dis);
        for (int j = 0; j < numParams; j++) {
            String k = dis.readUTF();
            String v = dis.readUTF();
            params.put(k, v);
        }

        ObserverConfiguration observerConfig = new ObserverConfiguration(clazz);
        observerConfig.setParameters(params);

        omap.put(col, observerConfig);
    }

    return omap;
}

---

From source file:it.crs4.pydoop.mapreduce.pipes.CommonStub.java

License:Apache License

protected void readObject(Writable obj, DataInputStream inStream) throws IOException {
    int numBytes = WritableUtils.readVInt(inStream);
    byte[] buffer;
    // For BytesWritable and Text, use the specified length to set the length
    // this causes the "obvious" translations to work. So that if you emit
    // a string "abc" from C++, it shows up as "abc".
    if (obj instanceof BytesWritable) {
        buffer = new byte[numBytes];
        inStream.readFully(buffer);//from   w  w  w  .  ja  v a  2 s.  c  o m
        ((BytesWritable) obj).set(buffer, 0, numBytes);
    } else if (obj instanceof Text) {
        buffer = new byte[numBytes];
        inStream.readFully(buffer);
        ((Text) obj).set(buffer);
    } else {
        obj.readFields(inStream);
    }
}

---

From source file:it.crs4.pydoop.mapreduce.pipes.CommonStub.java

License:Apache License

protected void initSoket() throws Exception {
    int port = Integer.parseInt(System.getenv("mapreduce.pipes.command.port"));

    System.err.println("port:" + port);

    java.net.InetAddress address = java.net.InetAddress.getLocalHost();

    socket = new Socket(address.getHostName(), port);
    InputStream input = socket.getInputStream();
    OutputStream output = socket.getOutputStream();

    System.err.println("ready to read");
    // try to read
    dataInput = new DataInputStream(input);

    WritableUtils.readVInt(dataInput);

    String str = Text.readString(dataInput);

    Text.readString(dataInput);//from  www . ja v  a2  s  .c o  m

    dataOut = new DataOutputStream(output);
    WritableUtils.writeVInt(dataOut, 57);
    String s = createDigest("password".getBytes(), str);

    Text.writeString(dataOut, s);

    // start
    WritableUtils.readVInt(dataInput);
    int cuttentAnswer = WritableUtils.readVInt(dataInput);
    System.out.println("CURRENT_PROTOCOL_VERSION:" + cuttentAnswer);

    // get configuration
    // should be MessageType.SET_JOB_CONF.code
    WritableUtils.readVInt(dataInput);

    // array length

    int j = WritableUtils.readVInt(dataInput);
    for (int i = 0; i < j; i++) {
        Text.readString(dataInput);
        i++;
        Text.readString(dataInput);
    }
}

---

From source file:it.crs4.pydoop.mapreduce.pipes.PipeApplicationRunnableStub.java

License:Apache License

public void binaryProtocolStub() {
    try {/*from   ww w .  java  2s . c  o  m*/

        initSoket();
        System.out.println("start OK");

        // RUN_MAP.code
        // should be 3

        int answer = WritableUtils.readVInt(dataInput);
        System.out.println("RunMap:" + answer);
        FileSplit split = new FileSplit();
        readObject(split, dataInput);

        WritableUtils.readVInt(dataInput);
        WritableUtils.readVInt(dataInput);
        // end runMap
        // get InputTypes
        WritableUtils.readVInt(dataInput);
        String inText = Text.readString(dataInput);
        System.out.println("Key class:" + inText);
        inText = Text.readString(dataInput);
        System.out.println("Value class:" + inText);

        @SuppressWarnings("unused")
        int inCode = 0;

        // read all data from sender and write to output
        while ((inCode = WritableUtils.readVInt(dataInput)) == 4) {
            FloatWritable key = new FloatWritable();
            NullWritable value = NullWritable.get();
            readObject(key, dataInput);
            System.out.println("value:" + key.get());
            readObject(value, dataInput);
        }

        WritableUtils.writeVInt(dataOut, 54);

        dataOut.flush();
        dataOut.close();

    } catch (Exception x) {
        x.printStackTrace();
    } finally {
        closeSoket();
    }

}

---

From source file:it.crs4.pydoop.mapreduce.pipes.PipeApplicationStub.java

License:Apache License

public void binaryProtocolStub() {
    try {// ww w .  java 2s  . c om

        System.err.println("binaryProtocolStub()");
        initSoket();
        System.err.println("done with socket()");

        // output code
        WritableUtils.writeVInt(dataOut, 50);
        IntWritable wt = new IntWritable();
        wt.set(123);
        writeObject(wt, dataOut);
        writeObject(new Text("value"), dataOut);

        //  PARTITIONED_OUTPUT
        WritableUtils.writeVInt(dataOut, 51);
        WritableUtils.writeVInt(dataOut, 0);
        writeObject(wt, dataOut);
        writeObject(new Text("value"), dataOut);

        // STATUS
        WritableUtils.writeVInt(dataOut, 52);
        Text.writeString(dataOut, "PROGRESS");
        dataOut.flush();

        // progress
        WritableUtils.writeVInt(dataOut, 53);
        dataOut.writeFloat(0.55f);
        // register counter
        WritableUtils.writeVInt(dataOut, 55);
        // id
        WritableUtils.writeVInt(dataOut, 0);
        Text.writeString(dataOut, "group");
        Text.writeString(dataOut, "name");
        // increment counter
        WritableUtils.writeVInt(dataOut, 56);
        WritableUtils.writeVInt(dataOut, 0);

        WritableUtils.writeVLong(dataOut, 2);

        // map item
        int intValue = WritableUtils.readVInt(dataInput);
        System.out.println("intValue:" + intValue);
        IntWritable iw = new IntWritable();
        readObject(iw, dataInput);
        System.out.println("key:" + iw.get());
        Text txt = new Text();
        readObject(txt, dataInput);
        System.out.println("value:" + txt.toString());

        // done
        // end of session
        WritableUtils.writeVInt(dataOut, 54);

        System.out.println("finish");
        dataOut.flush();
        dataOut.close();

    } catch (Exception x) {
        x.printStackTrace();
    } finally {
        closeSoket();
    }
}

---

From source file:it.crs4.pydoop.mapreduce.pipes.PipeReducerStub.java

License:Apache License

public void binaryProtocolStub() {
    try {/*from   w  w  w . j a  va 2 s  .  co m*/

        initSoket();

        //should be 5
        //RUN_REDUCE boolean 
        WritableUtils.readVInt(dataInput);
        WritableUtils.readVInt(dataInput);
        int intValue = WritableUtils.readVInt(dataInput);
        System.out.println("getIsJavaRecordWriter:" + intValue);

        // reduce key
        WritableUtils.readVInt(dataInput);
        // value of reduce key
        BooleanWritable value = new BooleanWritable();
        readObject(value, dataInput);
        System.out.println("reducer key :" + value);
        // reduce value code:

        // reduce values
        while ((intValue = WritableUtils.readVInt(dataInput)) == 7) {
            Text txt = new Text();
            // value
            readObject(txt, dataInput);
            System.out.println("reduce value  :" + txt);
        }

        // done
        WritableUtils.writeVInt(dataOut, 54);

        dataOut.flush();
        dataOut.close();

    } catch (Exception x) {
        x.printStackTrace();
    } finally {
        closeSoket();

    }
}

---

From source file:ivory.core.data.document.LazyIntDocVector.java

License:Apache License

@Override
public void readFields(DataInput in) throws IOException {
    numTerms = WritableUtils.readVInt(in);
    if (numTerms == 0) {
        bytes = null;//w  w w .  j  av a  2s.c o  m
        return;
    }
    bytes = new byte[WritableUtils.readVInt(in)];
    in.readFully(bytes);
}

---

From source file:ivory.core.data.document.LazyTermDocVector.java

License:Apache License

@Override
public void readFields(DataInput in) throws IOException {
    read = true;//  ww w. ja v a  2s . c  o  m
    numTerms = WritableUtils.readVInt(in);
    if (numTerms == 0) {
        rawBytes = null;
        terms = null;
        return;
    }
    terms = new String[numTerms];
    for (int i = 0; i < numTerms; i++) {
        terms[i] = in.readUTF();
    }
    rawBytes = new byte[WritableUtils.readVInt(in)];
    in.readFully(rawBytes);
}

---

From source file:ivory.core.data.index.PostingsAccumulator.java

License:Apache License

public void readFields(DataInput in) throws IOException {
    docnos.clear();/*from  w w  w .  j  a  va  2  s .  c  o  m*/
    positions.clear();
    int n = in.readInt();

    for (int i = 0; i < n; i++)
        docnos.add(in.readInt());

    int[] p;
    short m;
    for (int i = 0; i < n; i++) {
        m = in.readShort();
        p = new int[m];
        for (int j = 0; j < m; j++)
            p[j] = WritableUtils.readVInt(in);
        positions.add(p);
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• »