Example usage for org.apache.hadoop.io WritableUtils readVInt

Introduction

In this page you can find the example usage for org.apache.hadoop.io WritableUtils readVInt.

Prototype

public static int readVInt(DataInput stream) throws IOException 

Source Link

Document

Reads a zero-compressed encoded integer from input stream and returns it.

Usage

From source file:com.blm.orc.OrcNewSplit.java

License:Apache License

@Override
public void readFields(DataInput in) throws IOException {
    //deserialize path, offset, length using FileSplit
    super.readFields(in);

    byte flags = in.readByte();
    hasFooter = (OrcSplit.FOOTER_FLAG & flags) != 0;
    isOriginal = (OrcSplit.ORIGINAL_FLAG & flags) != 0;
    hasBase = (OrcSplit.BASE_FLAG & flags) != 0;

    deltas.clear();/*from   w  ww  .ja  v  a 2s  . c  om*/
    int numDeltas = in.readInt();
    for (int i = 0; i < numDeltas; i++) {
        deltas.add(in.readLong());
    }
    if (hasFooter) {
        // deserialize FileMetaInfo fields
        String compressionType = Text.readString(in);
        int bufferSize = WritableUtils.readVInt(in);
        int metadataSize = WritableUtils.readVInt(in);

        // deserialize FileMetaInfo field footer
        int footerBuffSize = WritableUtils.readVInt(in);
        ByteBuffer footerBuff = ByteBuffer.allocate(footerBuffSize);
        in.readFully(footerBuff.array(), 0, footerBuffSize);
        OrcFile.WriterVersion writerVersion = ReaderImpl.getWriterVersion(WritableUtils.readVInt(in));

        fileMetaInfo = new ReaderImpl.FileMetaInfo(compressionType, bufferSize, metadataSize, footerBuff,
                writerVersion);
    }
}

---

From source file:com.blm.orc.OrcSplit.java

License:Apache License

@Override
public void readFields(DataInput in) throws IOException {
    //deserialize path, offset, length using FileSplit
    super.readFields(in);

    byte flags = in.readByte();
    hasFooter = (FOOTER_FLAG & flags) != 0;
    isOriginal = (ORIGINAL_FLAG & flags) != 0;
    hasBase = (BASE_FLAG & flags) != 0;

    deltas.clear();// w w w  .ja va2s .c o m
    int numDeltas = in.readInt();
    for (int i = 0; i < numDeltas; i++) {
        deltas.add(in.readLong());
    }
    if (hasFooter) {
        // deserialize FileMetaInfo fields
        String compressionType = Text.readString(in);
        int bufferSize = WritableUtils.readVInt(in);
        int metadataSize = WritableUtils.readVInt(in);

        // deserialize FileMetaInfo field footer
        int footerBuffSize = WritableUtils.readVInt(in);
        ByteBuffer footerBuff = ByteBuffer.allocate(footerBuffSize);
        in.readFully(footerBuff.array(), 0, footerBuffSize);
        OrcFile.WriterVersion writerVersion = ReaderImpl.getWriterVersion(WritableUtils.readVInt(in));

        fileMetaInfo = new ReaderImpl.FileMetaInfo(compressionType, bufferSize, metadataSize, footerBuff,
                writerVersion);
    }
}

---

From source file:com.chinamobile.bcbsp.action.Directive.java

License:Apache License

@Override
public void readFields(DataInput in) throws IOException {
    this.faultSSStep = in.readInt();
    this.timestamp = in.readLong();
    int t = in.readInt();
    if (Directive.Type.Request.value() == t) {
        this.type = Directive.Type.Request;
        int length = WritableUtils.readVInt(in);
        if (length > 0) {
            this.actionList = new ArrayList<WorkerManagerAction>();
            for (int i = 0; i < length; ++i) {
                WorkerManagerAction.ActionType actionType = WritableUtils.readEnum(in,
                        WorkerManagerAction.ActionType.class);
                WorkerManagerAction action = WorkerManagerAction.createAction(actionType);
                action.readFields(in);//from w  w  w  .j av a  2s  .  c  om
                this.actionList.add(action);
            }
        } else {
            this.actionList = null;
        }

        this.workerManagersName = WritableUtils.readCompressedStringArray(in);
    } else if (Directive.Type.Response.value() == t) {
        this.type = Directive.Type.Response;
        this.status = new WorkerManagerStatus();
        this.status.readFields(in);
    } else {
        throw new IllegalStateException("Wrong directive type:" + t);
    }

    /* Zhicheng Liu added */
    this.migrateSSStep = in.readInt();

}

---

From source file:com.chinamobile.bcbsp.client.BSPJobClient.java

License:Apache License

/**
 * Read a splits file into a list of raw splits.
 * @param in// ww w. j  av  a  2s.c o m
 *        the stream to read from
 * @return the complete list of splits
 * @throws IOException
 *         NEU change in version-0.2.3 add new function
 */
public static RawSplit[] readSplitFile(DataInput in) throws IOException {
    byte[] header = new byte[SPLIT_FILE_HEADER.length];
    in.readFully(header);
    if (!Arrays.equals(SPLIT_FILE_HEADER, header)) {
        throw new IOException("Invalid header on split file");
    }
    int vers = WritableUtils.readVInt(in);
    if (vers != CURRENT_SPLIT_FILE_VERSION) {
        throw new IOException("Unsupported split version " + vers);
    }
    int len = WritableUtils.readVInt(in);
    RawSplit[] result = new RawSplit[len];
    for (int i = 0; i < len; ++i) {
        result[i] = new RawSplit();
        result[i].readFields(in);
    }
    return result;
}

---

From source file:com.chinamobile.bcbsp.sync.SuperStepCommand.java

License:Apache License

@Override
public void readFields(DataInput in) throws IOException {
    this.commandType = in.readInt();
    this.initWritePath = Text.readString(in);
    this.initReadPath = Text.readString(in);
    this.ableCheckPoint = in.readInt();
    this.nextSuperStepNum = in.readInt();
    this.oldCheckPoint = in.readInt();
    int count = in.readInt();
    this.aggValues = new String[count];
    for (int i = 0; i < count; i++) {
        this.aggValues[i] = Text.readString(in);
    }//from   ww  w. j  av a2 s  .c om
    int size = WritableUtils.readVInt(in);
    if (size > 0) {
        String[] partitionToWMName = WritableUtils.readCompressedStringArray(in);
        this.partitionToWorkerManagerNameAndPort = new HashMap<Integer, String>();
        for (int j = 0; j < size; j++) {
            this.partitionToWorkerManagerNameAndPort.put(j, partitionToWMName[j]);
        }
    }
    this.migrateStaffIDs = in.readUTF();
    this.migrateVertexCommand.readFields(in);

}

---

From source file:com.cloudera.branchreduce.onezero.PartialSolution.java

License:Open Source License

@Override
public void readFields(DataInput in) throws IOException {
    this.numVariables = WritableUtils.readVInt(in);
    this.fixLimit = WritableUtils.readVInt(in);
    this.values.clear();
    for (int i = 0; i < Math.max(numVariables, fixLimit); i++) {
        values.set(i, in.readBoolean());
    }//from w w w.  j  a va 2 s .co  m
}

---

From source file:com.cloudera.crunch.type.writable.TextMapWritable.java

License:Open Source License

@Override
public void readFields(DataInput in) throws IOException {
    instance.clear();/*from  www  .j  av a  2s.  c o  m*/
    try {
        this.valueClazz = (Class<T>) Class.forName(Text.readString(in));
    } catch (ClassNotFoundException e) {
        throw (IOException) new IOException("Failed map init").initCause(e);
    }
    int entries = WritableUtils.readVInt(in);
    try {
        for (int i = 0; i < entries; i++) {
            Text txt = new Text();
            txt.readFields(in);
            T value = valueClazz.newInstance();
            value.readFields(in);
            instance.put(txt, value);
        }
    } catch (IllegalAccessException e) {
        throw (IOException) new IOException("Failed map init").initCause(e);
    } catch (InstantiationException e) {
        throw (IOException) new IOException("Failed map init").initCause(e);
    }
}

---

From source file:com.cloudera.crunch.type.writable.TupleWritable.java

License:Apache License

/**
 * {@inheritDoc}/*w w w .j a  v  a  2s .c o m*/
 */
@SuppressWarnings("unchecked")
// No static typeinfo on Tuples
public void readFields(DataInput in) throws IOException {
    int card = WritableUtils.readVInt(in);
    values = new Writable[card];
    written = WritableUtils.readVLong(in);
    Class<? extends Writable>[] cls = new Class[card];
    try {
        for (int i = 0; i < card; ++i) {
            if (has(i)) {
                cls[i] = Class.forName(Text.readString(in)).asSubclass(Writable.class);
            }
        }
        for (int i = 0; i < card; ++i) {
            if (has(i)) {
                values[i] = cls[i].newInstance();
                values[i].readFields(in);
            }
        }
    } catch (ClassNotFoundException e) {
        throw (IOException) new IOException("Failed tuple init").initCause(e);
    } catch (IllegalAccessException e) {
        throw (IOException) new IOException("Failed tuple init").initCause(e);
    } catch (InstantiationException e) {
        throw (IOException) new IOException("Failed tuple init").initCause(e);
    }
}

---

From source file:com.cloudera.science.matching.graph.VertexState.java

License:Open Source License

@Override
public void readFields(DataInput in) throws IOException {
    bidder = in.readBoolean();/*from   www . j  a  va  2 s  . c  o m*/
    if (!bidder) {
        price = new BigDecimal(in.readUTF());
    }
    matchId.readFields(in);
    this.priceIndex = Maps.newHashMap();
    int sz = WritableUtils.readVInt(in);
    for (int i = 0; i < sz; i++) {
        Text vertexId = new Text();
        vertexId.readFields(in);
        String price = in.readUTF();
        priceIndex.put(vertexId, new BigDecimal(price));
    }
}

---

From source file:com.datasalt.pangool.io.BytesWritable.java

License:Apache License

public void readFields(DataInput in) throws IOException {
    setSize(0); // clear the old data
    setSize(WritableUtils.readVInt(in));
    in.readFully(bytes, 0, size);//from w  ww  .  j a v  a2 s.com
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• »