Example usage for org.apache.hadoop.io WritableUtils writeVInt

Introduction

In this page you can find the example usage for org.apache.hadoop.io WritableUtils writeVInt.

Prototype

public static void writeVInt(DataOutput stream, int i) throws IOException 

Source Link

Document

Serializes an integer to a binary stream with zero-compressed encoding.

Usage

From source file:org.apache.hama.pipes.protocol.UplinkReader.java

License:Apache License

public void sync() throws IOException, SyncException, InterruptedException {
    LOG.debug("Got MessageType.SYNC");

    peer.sync(); // this call blocks

    WritableUtils.writeVInt(this.outStream, MessageType.SYNC.code);
    binProtocol.flush();//from   w  w  w .j  a va2 s.  com
    LOG.debug("Responded MessageType.SYNC");
}

---

From source file:org.apache.hama.pipes.protocol.UplinkReader.java

License:Apache License

public void getMessage() throws IOException {
    LOG.debug("Got MessageType.GET_MSG");
    Writable message = peer.getCurrentMessage();
    if (message != null) {
        WritableUtils.writeVInt(this.outStream, MessageType.GET_MSG.code);
        binProtocol.writeObject(message);
        LOG.debug("Responded MessageType.GET_MSG - Message: "
                + ((message.toString().length() < 10) ? message.toString()
                        : message.toString().substring(0, 9) + "..."));
    } else {//from  ww w . j av  a  2 s .c  om
        WritableUtils.writeVInt(this.outStream, MessageType.END_OF_DATA.code);
        LOG.debug("Responded MessageType.END_OF_DATA");
    }
    binProtocol.flush();
}

---

From source file:org.apache.hama.pipes.protocol.UplinkReader.java

License:Apache License

public void getMessageCount() throws IOException {
    WritableUtils.writeVInt(this.outStream, MessageType.GET_MSG_COUNT.code);
    WritableUtils.writeVInt(this.outStream, peer.getNumCurrentMessages());
    binProtocol.flush();//from  w w  w  .j  ava  2  s .co  m
    LOG.debug("Responded MessageType.GET_MSG_COUNT - Count: " + peer.getNumCurrentMessages());
}

---

From source file:org.apache.hama.pipes.protocol.UplinkReader.java

License:Apache License

public void incrementCounter() throws IOException {
    LOG.debug("Got MessageType.INCREMENT_COUNTER");

    String group = Text.readString(this.inStream);
    String name = Text.readString(this.inStream);
    long amount = WritableUtils.readVLong(this.inStream);

    LOG.debug("Got MessageType.INCREMENT_COUNTER group: " + group + " name: " + name + " amount: " + amount);

    peer.incrementCounter(group, name, amount);

    WritableUtils.writeVInt(this.outStream, MessageType.INCREMENT_COUNTER.code);
    binProtocol.flush();/*from   w w  w.  j a  v  a2s .  c o m*/
    LOG.debug("Responded MessageType.INCREMENT_COUNTER");
}

---

From source file:org.apache.hama.pipes.protocol.UplinkReader.java

License:Apache License

@SuppressWarnings("unchecked")
public void sendMessage() throws IOException, InstantiationException, IllegalAccessException {
    String peerName = Text.readString(this.inStream);

    M message = (M) ReflectionUtils.newInstance(
            (Class<? extends M>) conf.getClass(Constants.MESSAGE_CLASS, BytesWritable.class), conf);

    LOG.debug("Got MessageType.SEND_MSG peerName: " + peerName + " messageClass: "
            + message.getClass().getName());

    readObject(message);/*from  www. j a  va  2s  .  c o m*/

    peer.send(peerName, message);

    WritableUtils.writeVInt(this.outStream, MessageType.SEND_MSG.code);
    binProtocol.flush();
    LOG.debug("Responded MessageType.SEND_MSG");

    LOG.debug("Sent message to peerName: " + peerName + " messageClass: " + message.getClass().getName()
            + " Message: " + ((message.toString().length() < 10) ? message.toString()
                    : message.toString().substring(0, 9) + "..."));
}

---

From source file:org.apache.hama.pipes.protocol.UplinkReader.java

License:Apache License

public void readKeyValue() throws IOException {

    boolean nullinput = peer.getConfiguration().get(Constants.INPUT_FORMAT_CLASS) == null || peer
            .getConfiguration().get(Constants.INPUT_FORMAT_CLASS).equals("org.apache.hama.bsp.NullInputFormat");

    if (!nullinput) {

        KeyValuePair<KEYIN, VALUEIN> pair = peer.readNext();

        if (pair != null) {
            WritableUtils.writeVInt(this.outStream, MessageType.READ_KEYVALUE.code);
            binProtocol.writeObject((Writable) pair.getKey());
            binProtocol.writeObject((Writable) pair.getValue());

            LOG.debug("Responded MessageType.READ_KEYVALUE -" + " Key: "
                    + ((pair.getKey().toString().length() < 10) ? pair.getKey().toString()
                            : pair.getKey().toString().substring(0, 9) + "...")
                    + " Value: " + ((pair.getValue().toString().length() < 10) ? pair.getValue().toString()
                            : pair.getValue().toString().substring(0, 9) + "..."));

        } else {// ww w  .ja  va 2  s.  c om
            WritableUtils.writeVInt(this.outStream, MessageType.END_OF_DATA.code);
            LOG.debug("Responded MessageType.READ_KEYVALUE - END_OF_DATA");
        }
        binProtocol.flush();

    } else {
        WritableUtils.writeVInt(this.outStream, MessageType.END_OF_DATA.code);
        binProtocol.flush();
        LOG.debug("Responded MessageType.READ_KEYVALUE - END_OF_DATA");
    }
}

---

From source file:org.apache.hama.pipes.protocol.UplinkReader.java

License:Apache License

@SuppressWarnings("unchecked")
public void writeKeyValue() throws IOException {

    KEYOUT keyOut = (KEYOUT) ReflectionUtils
            .newInstance((Class<? extends KEYOUT>) conf.getClass("bsp.output.key.class", Object.class), conf);

    VALUEOUT valueOut = (VALUEOUT) ReflectionUtils.newInstance(
            (Class<? extends VALUEOUT>) conf.getClass("bsp.output.value.class", Object.class), conf);

    LOG.debug("Got MessageType.WRITE_KEYVALUE keyOutClass: " + keyOut.getClass().getName() + " valueOutClass: "
            + valueOut.getClass().getName());

    readObject((Writable) keyOut);/*  w w w  . j a  v  a  2s .c  o m*/
    readObject((Writable) valueOut);

    peer.write(keyOut, valueOut);

    WritableUtils.writeVInt(this.outStream, MessageType.WRITE_KEYVALUE.code);
    binProtocol.flush();
    LOG.debug("Responded MessageType.WRITE_KEYVALUE");

    LOG.debug("Done MessageType.WRITE_KEYVALUE -" + " Key: "
            + ((keyOut.toString().length() < 10) ? keyOut.toString()
                    : keyOut.toString().substring(0, 9) + "...")
            + " Value: " + ((valueOut.toString().length() < 10) ? valueOut.toString()
                    : valueOut.toString().substring(0, 9) + "..."));
}

---

From source file:org.apache.hama.pipes.protocol.UplinkReader.java

License:Apache License

public void seqFileOpen() throws IOException {
    String path = Text.readString(this.inStream);
    // option - read = "r" or write = "w"
    String option = Text.readString(this.inStream);
    // key and value class stored in the SequenceFile
    String keyClass = Text.readString(this.inStream);
    String valueClass = Text.readString(this.inStream);
    LOG.debug("GOT MessageType.SEQFILE_OPEN - Path: " + path);
    LOG.debug("GOT MessageType.SEQFILE_OPEN - Option: " + option);
    LOG.debug("GOT MessageType.SEQFILE_OPEN - KeyClass: " + keyClass);
    LOG.debug("GOT MessageType.SEQFILE_OPEN - ValueClass: " + valueClass);

    int fileID = -1;

    if (option.equals("r")) {
        SequenceFile.Reader reader;
        try {//from  w  w  w .j  a va  2  s  . co  m
            reader = new SequenceFile.Reader(fs, new Path(path), conf);

            if (reader.getKeyClassName().equals(keyClass) && reader.getValueClassName().equals(valueClass)) {
                // try to load key and value class
                Class<?> sequenceKeyClass = conf.getClassLoader().loadClass(keyClass);
                Class<?> sequenceValueClass = conf.getClassLoader().loadClass(valueClass);

                // try to instantiate key and value class
                Writable sequenceKeyWritable = (Writable) ReflectionUtils.newInstance(sequenceKeyClass, conf);
                Writable sequenceValueWritable = (Writable) ReflectionUtils.newInstance(sequenceValueClass,
                        conf);

                // put new fileID and key and value Writable instances into HashMap
                fileID = reader.hashCode();
                this.sequenceFileReaders.put(fileID,
                        new AbstractMap.SimpleEntry<SequenceFile.Reader, Entry<Writable, Writable>>(reader,
                                new AbstractMap.SimpleEntry<Writable, Writable>(sequenceKeyWritable,
                                        sequenceValueWritable)));

            } else { // keyClass or valueClass is wrong
                fileID = -1;
                if (!reader.getKeyClassName().equals(keyClass)) {
                    LOG.error("SEQFILE_OPEN - Wrong KeyClass: " + keyClass + " File KeyClass: "
                            + reader.getKeyClassName());
                } else {
                    LOG.error("SEQFILE_OPEN - Wrong ValueClass: " + valueClass + " File ValueClass: "
                            + reader.getValueClassName());
                }
            }

        } catch (IOException e) {
            LOG.error("SEQFILE_OPEN - " + e.getMessage());
            fileID = -1;
        } catch (ClassNotFoundException e) {
            LOG.error("SEQFILE_OPEN - " + e.getMessage());
            fileID = -1;
        }

    } else if (option.equals("w")) {
        SequenceFile.Writer writer;
        try {
            // SequenceFile.Writer has an exclusive lease for a file
            // No other client can write to this file until other Writer has
            // completed
            if (!this.sequenceFileWriterPaths.contains(path)) {

                // try to load key and value class
                Class<?> sequenceKeyClass = conf.getClassLoader().loadClass(keyClass);
                Class<?> sequenceValueClass = conf.getClassLoader().loadClass(valueClass);

                // try to instantiate key and value class
                Writable sequenceKeyWritable = (Writable) ReflectionUtils.newInstance(sequenceKeyClass, conf);
                Writable sequenceValueWritable = (Writable) ReflectionUtils.newInstance(sequenceValueClass,
                        conf);

                writer = new SequenceFile.Writer(fs, conf, new Path(path), sequenceKeyClass,
                        sequenceValueClass);

                // put new fileID and key and value Writable instances into HashMap
                fileID = writer.hashCode();
                this.sequenceFileWriters.put(fileID,
                        new AbstractMap.SimpleEntry<SequenceFile.Writer, Entry<Writable, Writable>>(writer,
                                new AbstractMap.SimpleEntry<Writable, Writable>(sequenceKeyWritable,
                                        sequenceValueWritable)));

                // add path to set (exclusive access)
                this.sequenceFileWriterPaths.add(path);

            } else { // Path was already opened by another SequenceFile.Writer
                fileID = -1;
                LOG.error("SEQFILE_OPEN - Path: " + path + " is already used by another Writer!");
            }

        } catch (IOException e) {
            LOG.error("SEQFILE_OPEN - " + e.getMessage());
            fileID = -1;
        } catch (ClassNotFoundException e) {
            LOG.error("SEQFILE_OPEN - " + e.getMessage());
            fileID = -1;
        }
    } else { // wrong option
        LOG.error("SEQFILE_OPEN - Wrong option: '" + option + "'");
    }

    WritableUtils.writeVInt(this.outStream, MessageType.SEQFILE_OPEN.code);
    WritableUtils.writeVInt(this.outStream, fileID);
    binProtocol.flush();
    LOG.debug("Responded MessageType.SEQFILE_OPEN - FileID: " + fileID);
}

---

From source file:org.apache.hama.pipes.protocol.UplinkReader.java

License:Apache License

public void seqFileReadNext() throws IOException {
    int fileID = WritableUtils.readVInt(this.inStream);
    LOG.debug("GOT MessageType.SEQFILE_READNEXT - FileID: " + fileID);

    // check if fileID is available in sequenceFileReader
    if (this.sequenceFileReaders.containsKey(fileID)) {

        Writable sequenceKeyWritable = sequenceFileReaders.get(fileID).getValue().getKey();
        Writable sequenceValueWritable = sequenceFileReaders.get(fileID).getValue().getValue();

        // try to read next key/value pair from SequenceFile.Reader
        if (this.sequenceFileReaders.get(fileID).getKey().next(sequenceKeyWritable, sequenceValueWritable)) {

            WritableUtils.writeVInt(this.outStream, MessageType.SEQFILE_READNEXT.code);
            binProtocol.writeObject(sequenceKeyWritable);
            binProtocol.writeObject(sequenceValueWritable);

            LOG.debug("Responded MessageType.SEQFILE_READNEXT -" + " Key: "
                    + ((sequenceKeyWritable.toString().length() < 10) ? sequenceKeyWritable.toString()
                            : sequenceKeyWritable.toString().substring(0, 9) + "...")
                    + " Value: "
                    + ((sequenceValueWritable.toString().length() < 10) ? sequenceValueWritable.toString()
                            : sequenceValueWritable.toString().substring(0, 9) + "..."));

        } else { // false when at end of file
            WritableUtils.writeVInt(this.outStream, MessageType.END_OF_DATA.code);
            LOG.debug("Responded MessageType.SEQFILE_READNEXT - END_OF_DATA");
        }/*w  w  w  .j ava2s  .c o m*/
        binProtocol.flush();

    } else { // no fileID stored
        LOG.error("MessageType.SEQFILE_READNEXT: FileID " + fileID + " not found!");
        WritableUtils.writeVInt(this.outStream, MessageType.END_OF_DATA.code);
        LOG.debug("Responded MessageType.SEQFILE_READNEXT - END_OF_DATA");
        binProtocol.flush();
    }
}

---

From source file:org.apache.hama.pipes.protocol.UplinkReader.java

License:Apache License

public void seqFileAppend() throws IOException {
    int fileID = WritableUtils.readVInt(this.inStream);
    LOG.debug("GOT MessageType.SEQFILE_APPEND - FileID: " + fileID);

    boolean result = false;

    // check if fileID is available in sequenceFileWriter
    if (this.sequenceFileWriters.containsKey(fileID)) {

        Writable sequenceKeyWritable = sequenceFileWriters.get(fileID).getValue().getKey();
        Writable sequenceValueWritable = sequenceFileWriters.get(fileID).getValue().getValue();

        // try to read key and value
        readObject(sequenceKeyWritable);
        readObject(sequenceValueWritable);

        if ((sequenceKeyWritable != null) && (sequenceValueWritable != null)) {

            // append to sequenceFile
            this.sequenceFileWriters.get(fileID).getKey().append(sequenceKeyWritable, sequenceValueWritable);

            LOG.debug("Stored data: Key: "
                    + ((sequenceKeyWritable.toString().length() < 10) ? sequenceKeyWritable.toString()
                            : sequenceKeyWritable.toString().substring(0, 9) + "...")
                    + " Value: "
                    + ((sequenceValueWritable.toString().length() < 10) ? sequenceValueWritable.toString()
                            : sequenceValueWritable.toString().substring(0, 9) + "..."));

            result = true;/*from   w  ww .  java  2 s. c o  m*/
        }
    } else { // no fileID stored

        // Skip written data from InputStream
        int availableBytes = this.inStream.available();
        this.inStream.skip(availableBytes);
        LOG.debug("MessageType.SEQFILE_APPEND: skip " + availableBytes + " bytes");
        LOG.error("MessageType.SEQFILE_APPEND: FileID " + fileID + " not found!");
    }

    // RESPOND
    WritableUtils.writeVInt(this.outStream, MessageType.SEQFILE_APPEND.code);
    WritableUtils.writeVInt(this.outStream, result ? 1 : 0);
    binProtocol.flush();
    LOG.debug("Responded MessageType.SEQFILE_APPEND - Result: " + result);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• »