Example usage for org.apache.hadoop.io WritableUtils readVInt

Introduction

In this page you can find the example usage for org.apache.hadoop.io WritableUtils readVInt.

Prototype

public static int readVInt(DataInput stream) throws IOException 

Source Link

Document

Reads a zero-compressed encoded integer from input stream and returns it.

Usage

From source file:com.asakusafw.runtime.io.util.WritableRawComparableUnion.java

License:Apache License

@Override
public void readFields(DataInput in) throws IOException {
    this.position = WritableUtils.readVInt(in);
    objects[position].readFields(in);//w ww  .  j  a v  a  2s . c  o m
}

---

From source file:com.asakusafw.runtime.stage.collector.SortableSlot.java

License:Apache License

@Override
public void readFields(DataInput in) throws IOException {
    this.slotNumber = WritableUtils.readVInt(in);
    int length = WritableUtils.readVInt(in);
    buffer.reset(0, 0);/*ww  w  . java 2 s. c o  m*/
    buffer.write(in, length);
}

---

From source file:com.asakusafw.runtime.stage.collector.WritableSlot.java

License:Apache License

@Override
public void readFields(DataInput in) throws IOException {
    buffer.reset(0, 0);/*  www. j  av a  2 s .com*/
    int length = WritableUtils.readVInt(in);
    buffer.write(in, length);
}

---

From source file:com.asakusafw.runtime.stage.input.StageInputDriver.java

License:Apache License

@SuppressWarnings("rawtypes")
private static List<StageInput> decode(Configuration conf, String encoded)
        throws IOException, ClassNotFoundException {
    assert conf != null;
    assert encoded != null;
    ByteArrayInputStream source = new ByteArrayInputStream(encoded.getBytes(ASCII));
    DataInputStream input = new DataInputStream(new GZIPInputStream(new Base64InputStream(source)));
    long version = WritableUtils.readVLong(input);
    if (version != SERIAL_VERSION) {
        throw new IOException(MessageFormat.format("Invalid StageInput version: framework={0}, saw={1}",
                SERIAL_VERSION, version));
    }/*from w w  w  .  j a va  2 s  . c o  m*/
    String[] dictionary = WritableUtils.readStringArray(input);
    int inputListSize = WritableUtils.readVInt(input);
    List<StageInput> results = new ArrayList<>();
    for (int inputListIndex = 0; inputListIndex < inputListSize; inputListIndex++) {
        String pathString = readEncoded(input, dictionary);
        String formatName = readEncoded(input, dictionary);
        String mapperName = readEncoded(input, dictionary);
        int attributeCount = WritableUtils.readVInt(input);
        Map<String, String> attributes = new HashMap<>();
        for (int attributeIndex = 0; attributeIndex < attributeCount; attributeIndex++) {
            String keyString = readEncoded(input, dictionary);
            String valueString = readEncoded(input, dictionary);
            attributes.put(keyString, valueString);
        }
        Class<? extends InputFormat> formatClass = conf.getClassByName(formatName)
                .asSubclass(InputFormat.class);
        Class<? extends Mapper> mapperClass = conf.getClassByName(mapperName).asSubclass(Mapper.class);
        results.add(new StageInput(pathString, formatClass, mapperClass, attributes));
    }
    return results;
}

---

From source file:com.asakusafw.runtime.stage.input.StageInputDriver.java

License:Apache License

private static String readEncoded(DataInput input, String[] dictionary) throws IOException {
    assert input != null;
    assert dictionary != null;
    int index = WritableUtils.readVInt(input);
    if (index < 0 || index >= dictionary.length) {
        throw new IOException(MessageFormat.format("Invalid encoded value: index={0}, dict={1}", index,
                Arrays.toString(dictionary)));
    }/*from  w w w . j a  v a  2  s.c om*/
    return dictionary[index];
}

---

From source file:com.asakusafw.runtime.stage.input.StageInputSplit.java

License:Apache License

@SuppressWarnings("unchecked")
@Override//www  .  j a  va2 s .  co m
public void readFields(DataInput in) throws IOException {
    this.mapperClass = (Class<? extends Mapper<?, ?, ?, ?>>) readClassByName(Mapper.class, in);
    int sourceCount = WritableUtils.readVInt(in);
    List<Source> newSources = new ArrayList<>();
    for (int i = 0; i < sourceCount; i++) {
        Class<? extends InputFormat<?, ?>> formatClass = (Class<? extends InputFormat<?, ?>>) readClassByName(
                InputFormat.class, in);
        Class<? extends InputSplit> splitClass = readClassByName(InputSplit.class, in);
        InputSplit inputSplit = ReflectionUtils.newInstance(splitClass, getConf());
        ((Writable) inputSplit).readFields(in);
        newSources.add(new Source(inputSplit, formatClass));
    }
    this.sources = newSources;
    int locationCount = WritableUtils.readVInt(in);
    if (locationCount < 0) {
        this.locations = null;
    } else {
        String[] array = new String[locationCount];
        for (int i = 0; i < array.length; i++) {
            array[i] = WritableUtils.readString(in);
        }
        this.locations = array;
    }
}

---

From source file:com.asakusafw.runtime.stage.output.BridgeOutputFormat.java

License:Apache License

private static List<OutputSpec> getSpecs(JobContext context) {
    assert context != null;
    String encoded = context.getConfiguration().getRaw(KEY);
    if (encoded == null) {
        return Collections.emptyList();
    }/*  w w  w  .  java 2s.c  om*/
    VariableTable table = getVariableTable(context);
    try {
        ByteArrayInputStream source = new ByteArrayInputStream(encoded.getBytes(ASCII));
        DataInputStream input = new DataInputStream(new GZIPInputStream(new Base64InputStream(source)));
        long version = WritableUtils.readVLong(input);
        if (version != SERIAL_VERSION) {
            throw new IOException(MessageFormat.format("Invalid StageOutput version: framework={0}, saw={1}",
                    SERIAL_VERSION, version));
        }
        List<OutputSpec> results = new ArrayList<>();
        int specCount = WritableUtils.readVInt(input);
        for (int specIndex = 0; specIndex < specCount; specIndex++) {
            String basePath = WritableUtils.readString(input);
            try {
                basePath = table.parse(basePath);
            } catch (IllegalArgumentException e) {
                throw new IllegalStateException(MessageFormat.format("Invalid basePath: {0}", basePath), e);
            }
            int patternCount = WritableUtils.readVInt(input);
            List<String> patterns = new ArrayList<>();
            for (int patternIndex = 0; patternIndex < patternCount; patternIndex++) {
                String pattern = WritableUtils.readString(input);
                try {
                    pattern = table.parse(pattern);
                } catch (IllegalArgumentException e) {
                    throw new IllegalStateException(
                            MessageFormat.format("Invalid delete pattern: {0}", pattern), e);
                }
                patterns.add(pattern);
            }
            results.add(new OutputSpec(basePath, patterns, true));
        }
        return results;
    } catch (IOException e) {
        throw new IllegalStateException(e);
    }
}

---

From source file:com.asakusafw.runtime.value.DecimalOption.java

License:Apache License

@SuppressWarnings("deprecation")
@Override/*from  www .j  a v  a 2 s . c o m*/
public void readFields(DataInput in) throws IOException {
    int head = in.readByte() & 0xff;
    if ((head & MASK_PRESENT) == 0) {
        setNull();
        return;
    }
    boolean plus = (head & MASK_PLUS) != 0;
    int scale = WritableUtils.readVInt(in);
    int length = WritableUtils.readVInt(in);

    DecimalBuffer buffer = BUFFER_MAIN.get();
    byte[] target = buffer.setMeta(plus, scale, length);
    in.readFully(target, target.length - length, length);
    modify(buffer.toBigDecimal());
}

---

From source file:com.bah.culvert.data.CKeyValue.java

License:Apache License

@Override
public void readFields(DataInput in) throws IOException {
    // Get the byte length
    int rowLength = WritableUtils.readVInt(in);
    int familyLength = WritableUtils.readVInt(in);
    int qualifierLength = WritableUtils.readVInt(in);
    int valueLength = WritableUtils.readVInt(in);

    this.rowId = new byte[rowLength];
    this.family = new byte[familyLength];
    this.qualifier = new byte[qualifierLength];
    this.value = new byte[valueLength];

    // read the data
    in.readFully(this.rowId);
    in.readFully(this.family);
    in.readFully(this.qualifier);
    in.readFully(this.value);
    this.timestamp = WritableUtils.readVLong(in);
}

---

From source file:com.bah.culvert.data.CRange.java

License:Apache License

@Override
public void readFields(DataInput in) throws IOException {
    // Get the byte length
    int startLength = WritableUtils.readVInt(in);
    int endLength = WritableUtils.readVInt(in);

    start = new byte[startLength];
    end = new byte[endLength];

    // read the data
    in.readFully(start);//from www . j a  va 2s  .  c o  m
    in.readFully(end);

    startInclusive = in.readBoolean();
    endInclusive = in.readBoolean();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• »