Example usage for org.apache.hadoop.io ShortWritable get

Introduction

In this page you can find the example usage for org.apache.hadoop.io ShortWritable get.

Prototype

public short get() 

Source Link

Document

Return the value of this ShortWritable.

Usage

From source file:com.axiomine.largecollections.kryo.serializers.ShortWritableSerializer.java

License:Apache License

public void write(Kryo kryo, Output output, ShortWritable object) {
    output.writeShort(object.get());
}

---

From source file:com.github.ygf.pagerank.PageRankTopNMapper.java

License:Apache License

@Override
public void map(ShortWritable inKey, FloatArrayWritable inValue, Context context)
        throws IOException, InterruptedException {

    Configuration conf = context.getConfiguration();
    short blockSize = Short.parseShort(conf.get("pagerank.block_size"));
    int topResults = Integer.parseInt(conf.get("pagerank.top_results"));

    Writable[] vStripe = inValue.get();//from w  ww . j  a va2 s  .  co m
    for (int i = 0; i < vStripe.length; i++) {
        int page = 1 + (inKey.get() - 1) * blockSize + i;
        float pageRank = ((FloatWritable) vStripe[i]).get();

        // The elements in the queue are sorted (in non-decreasing order) by
        // PageRank. The queue is filled up until it contains topResults
        // elements. Then, a new element will be added only if its PageRank
        // is greater than the lowest PageRank in the queue. If the queue is
        // full and a new element is added, the one with the lowest PageRank
        // is removed from the queue.
        if (topN.size() < topResults || pageRank >= topN.peek().getKey()) {
            topN.add(new AbstractMap.SimpleEntry<Float, Integer>(pageRank, page));
            if (topN.size() > topResults) {
                topN.poll();
            }
        }
    }
}

---

From source file:com.kylinolap.job.hadoop.cube.FactDistinctColumnsReducer.java

License:Apache License

@Override
public void reduce(ShortWritable key, Iterable<Text> values, Context context)
        throws IOException, InterruptedException {
    TblColRef col = columnList.get(key.get());

    HashSet<ByteArray> set = new HashSet<ByteArray>();
    for (Text textValue : values) {
        ByteArray value = new ByteArray(Bytes.copy(textValue.getBytes(), 0, textValue.getLength()));
        set.add(value);//  w ww .  j  a  v a 2s  .  c  o m
    }

    Configuration conf = context.getConfiguration();
    FileSystem fs = FileSystem.get(conf);
    String outputPath = conf.get(BatchConstants.OUTPUT_PATH);
    FSDataOutputStream out = fs.create(new Path(outputPath, col.getName()));

    try {
        for (ByteArray value : set) {
            out.write(value.data);
            out.write('\n');
        }
    } finally {
        out.close();
    }

}

---

From source file:com.kylinolap.job.hadoop.invertedindex.IIDistinctColumnsReducer.java

License:Apache License

@Override
public void reduce(ShortWritable key, Iterable<Text> values, Context context)
        throws IOException, InterruptedException {
    String columnName = columns[key.get()];

    HashSet<ByteArray> set = new HashSet<ByteArray>();
    for (Text textValue : values) {
        ByteArray value = new ByteArray(Bytes.copy(textValue.getBytes(), 0, textValue.getLength()));
        set.add(value);//from  w  w w.jav a  2s. c o  m
    }

    Configuration conf = context.getConfiguration();
    FileSystem fs = FileSystem.get(conf);
    String outputPath = conf.get(BatchConstants.OUTPUT_PATH);
    FSDataOutputStream out = fs.create(new Path(outputPath, columnName));

    try {
        for (ByteArray value : set) {
            out.write(value.data);
            out.write('\n');
        }
    } finally {
        out.close();
    }

}

---

From source file:org.shaf.core.util.IOUtilsTest.java

License:Apache License

/**
 * Test writing of {@code short} value.//from w  w w  . j  a v  a 2  s.c o m
 */
@Test
public void testWriteShort() {
    byte[] buf = null;

    short value = 123;
    try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
            DataOutputStream out = new DataOutputStream(baos);) {
        IOUtils.writeObject(value, out);
        buf = baos.toByteArray();
    } catch (IOException exc) {
        fail(exc.getMessage());
    }

    try (ByteArrayInputStream bais = new ByteArrayInputStream(buf);
            DataInputStream in = new DataInputStream(bais);) {
        ShortWritable probe = new ShortWritable();
        probe.readFields(in);
        assertEquals(value, probe.get());
    } catch (IOException exc) {
        fail(exc.getMessage());
    }
}

---