Example usage for org.apache.hadoop.io ShortWritable ShortWritable

Introduction

In this page you can find the example usage for org.apache.hadoop.io ShortWritable ShortWritable.

Prototype

public ShortWritable() 

Source Link

Usage

From source file:com.cloudera.recordservice.mr.RecordServiceRecord.java

License:Apache License

/**
 * Returns the corresponding Writable object for this column type.
 *///  w ww .  j av a2 s. co m
public Writable getWritableInstance(com.cloudera.recordservice.core.Schema.Type type) {
    switch (type) {
    case BOOLEAN:
        return new BooleanWritable();
    case TINYINT:
        return new ByteWritable();
    case SMALLINT:
        return new ShortWritable();
    case INT:
        return new IntWritable();
    case BIGINT:
        return new LongWritable();
    case FLOAT:
        return new FloatWritable();
    case DOUBLE:
        return new DoubleWritable();
    case VARCHAR:
    case CHAR:
    case STRING:
        return new Text();
    case TIMESTAMP_NANOS:
        return new TimestampNanosWritable();
    case DECIMAL:
        return new DecimalWritable();
    default:
        throw new UnsupportedOperationException("Unexpected type: " + toString());
    }
}

---

From source file:com.github.ygf.pagerank.PageRankMatrixMapper.java

License:Apache License

@Override
public void map(LongWritable inKey, Text inValue, Context context) throws IOException, InterruptedException {

    // This task gets a line from links-simple-sorted.txt that contains the
    // out links of a page v. It produces results with keys (i, j) 
    // corresponding to the indexes of the block M_{i,j} in which each
    // link v -> w should be stored. The value is (v, w, degree(v)).

    Configuration conf = context.getConfiguration();
    short blockSize = Short.parseShort(conf.get("pagerank.block_size"));

    String[] lineParts = inValue.toString().split(":\\s+");
    String[] vOutlinks = lineParts[1].split("\\s+");

    ShortWritable[] blockIndexes = new ShortWritable[2];
    blockIndexes[0] = new ShortWritable();
    blockIndexes[1] = new ShortWritable();

    ShortWritable[] blockEntry = new ShortWritable[3];
    blockEntry[0] = new ShortWritable();
    blockEntry[1] = new ShortWritable();
    blockEntry[2] = new ShortWritable();

    int v, w;/*w w w  . j a  va  2  s .  co m*/
    short i, j;

    v = Integer.parseInt(lineParts[0]);
    j = (short) ((v - 1) / blockSize + 1);

    for (int k = 0; k < vOutlinks.length; k++) {
        w = Integer.parseInt(vOutlinks[k]);
        i = (short) ((w - 1) / blockSize + 1);

        // Indexes of the block M_{i,j}.
        blockIndexes[0].set(i);
        blockIndexes[1].set(j);
        // One entry of the block M_{i,j} corresponding to the v -> w link.
        // The sparse block representation also needs information about
        // the degree of the vector v.
        blockEntry[0].set((short) ((v - 1) % blockSize));
        blockEntry[1].set((short) ((w - 1) % blockSize));
        blockEntry[2].set((short) vOutlinks.length);

        context.write(new ShortArrayWritable(blockIndexes), new ShortArrayWritable(blockEntry));
    }
}

---

From source file:com.github.ygf.pagerank.PageRankMatrixReducer.java

License:Apache License

@Override
public void reduce(ShortArrayWritable inKey, Iterable<ShortArrayWritable> inValues, Context context)
        throws IOException, InterruptedException {

    // This task receives all the entries in M_{i,j} and builds the compact
    // representation of the block. See Section 5.2.4 of Mining of Massive
    // Datasets (http://infolab.stanford.edu/~ullman/mmds.html) for details.
    // Only blocks with at least one nonzero entry are generated. 

    Configuration conf = context.getConfiguration();
    short blockSize = Short.parseShort(conf.get("pagerank.block_size"));

    short vIndexInBlock, wIndexInBlock, vDegree;
    List<List<Short>> blockColumns = new ArrayList<List<Short>>(blockSize);
    for (int k = 0; k < blockSize; k++) {
        blockColumns.add(new ArrayList<Short>());
    }//  w  w  w .ja  va  2  s. c o m

    for (ShortArrayWritable inValue : inValues) {
        Writable[] blockEntry = inValue.get();
        vIndexInBlock = ((ShortWritable) blockEntry[0]).get();
        wIndexInBlock = ((ShortWritable) blockEntry[1]).get();
        vDegree = ((ShortWritable) blockEntry[2]).get();

        if (blockColumns.get(vIndexInBlock).isEmpty()) {
            blockColumns.get(vIndexInBlock).add(vDegree);
        }
        blockColumns.get(vIndexInBlock).add(wIndexInBlock);
    }

    ShortWritable[][] blockColumnWritables = new ShortWritable[blockColumns.size()][];
    for (int k = 0; k < blockColumns.size(); k++) {
        List<Short> column = blockColumns.get(k);
        blockColumnWritables[k] = new ShortWritable[column.size()];
        for (int l = 0; l < column.size(); l++) {
            blockColumnWritables[k][l] = new ShortWritable();
            blockColumnWritables[k][l].set(column.get(l).shortValue());
        }
    }

    context.write(inKey, new MatrixBlockWritable(blockColumnWritables));
}

---

From source file:com.yassergonzalez.pagerank.PageRankMatrixMapper.java

License:Apache License

@Override
public void map(LongWritable inKey, Text inValue, Context context) throws IOException, InterruptedException {

    // This task gets a line from links-simple-sorted.txt that contains the
    // out links of a page v. It produces results with keys (i, j)
    // corresponding to the indexes of the block M_{i,j} in which each
    // link v -> w should be stored. The value is (v, w, degree(v)).

    Configuration conf = context.getConfiguration();
    short blockSize = Short.parseShort(conf.get("pagerank.block_size"));

    String[] lineParts = inValue.toString().split(":\\s+");
    String[] vOutlinks = lineParts[1].split("\\s+");

    ShortWritable[] blockIndexes = new ShortWritable[2];
    blockIndexes[0] = new ShortWritable();
    blockIndexes[1] = new ShortWritable();

    ShortWritable[] blockEntry = new ShortWritable[3];
    blockEntry[0] = new ShortWritable();
    blockEntry[1] = new ShortWritable();
    blockEntry[2] = new ShortWritable();

    int v, w;//from  www .  ja v a 2  s . c  o  m
    short i, j;

    v = Integer.parseInt(lineParts[0]);
    j = (short) ((v - 1) / blockSize + 1);

    for (int k = 0; k < vOutlinks.length; k++) {
        w = Integer.parseInt(vOutlinks[k]);
        i = (short) ((w - 1) / blockSize + 1);

        // Indexes of the block M_{i,j}.
        blockIndexes[0].set(i);
        blockIndexes[1].set(j);
        // One entry of the block M_{i,j} corresponding to the v -> w link.
        // The sparse block representation also needs information about
        // the degree of the vector v.
        blockEntry[0].set((short) ((v - 1) % blockSize));
        blockEntry[1].set((short) ((w - 1) % blockSize));
        blockEntry[2].set((short) vOutlinks.length);

        context.write(new ShortArrayWritable(blockIndexes), new ShortArrayWritable(blockEntry));
    }
}

---

From source file:com.yassergonzalez.pagerank.PageRankMatrixReducer.java

License:Apache License

@Override
public void reduce(ShortArrayWritable inKey, Iterable<ShortArrayWritable> inValues, Context context)
        throws IOException, InterruptedException {

    // This task receives all the entries in M_{i,j} and builds the compact
    // representation of the block. See Section 5.2.4 of Mining of Massive
    // Datasets (http://infolab.stanford.edu/~ullman/mmds.html) for details.
    // Only blocks with at least one nonzero entry are generated.

    Configuration conf = context.getConfiguration();
    short blockSize = Short.parseShort(conf.get("pagerank.block_size"));

    short vIndexInBlock, wIndexInBlock, vDegree;
    List<List<Short>> blockColumns = new ArrayList<List<Short>>(blockSize);
    for (int k = 0; k < blockSize; k++) {
        blockColumns.add(new ArrayList<Short>());
    }// ww w.j  av a 2 s  . c  om

    for (ShortArrayWritable inValue : inValues) {
        Writable[] blockEntry = inValue.get();
        vIndexInBlock = ((ShortWritable) blockEntry[0]).get();
        wIndexInBlock = ((ShortWritable) blockEntry[1]).get();
        vDegree = ((ShortWritable) blockEntry[2]).get();

        if (blockColumns.get(vIndexInBlock).isEmpty()) {
            blockColumns.get(vIndexInBlock).add(vDegree);
        }
        blockColumns.get(vIndexInBlock).add(wIndexInBlock);
    }

    ShortWritable[][] blockColumnWritables = new ShortWritable[blockColumns.size()][];
    for (int k = 0; k < blockColumns.size(); k++) {
        List<Short> column = blockColumns.get(k);
        blockColumnWritables[k] = new ShortWritable[column.size()];
        for (int l = 0; l < column.size(); l++) {
            blockColumnWritables[k][l] = new ShortWritable();
            blockColumnWritables[k][l].set(column.get(l).shortValue());
        }
    }

    context.write(inKey, new MatrixBlockWritable(blockColumnWritables));
}

---

From source file:org.apache.orc.mapred.OrcMapredRecordReader.java

License:Apache License

static ShortWritable nextShort(ColumnVector vector, int row, Object previous) {
    if (vector.isRepeating) {
        row = 0;/*  ww  w . j a v a 2s .  c  o  m*/
    }
    if (vector.noNulls || !vector.isNull[row]) {
        ShortWritable result;
        if (previous == null || previous.getClass() != ShortWritable.class) {
            result = new ShortWritable();
        } else {
            result = (ShortWritable) previous;
        }
        result.set((short) ((LongColumnVector) vector).vector[row]);
        return result;
    } else {
        return null;
    }
}

---

From source file:org.apache.orc.mapred.OrcStruct.java

License:Apache License

public static WritableComparable createValue(TypeDescription type) {
    switch (type.getCategory()) {
    case BOOLEAN:
        return new BooleanWritable();
    case BYTE:/*w w w .j a v  a2s .co m*/
        return new ByteWritable();
    case SHORT:
        return new ShortWritable();
    case INT:
        return new IntWritable();
    case LONG:
        return new LongWritable();
    case FLOAT:
        return new FloatWritable();
    case DOUBLE:
        return new DoubleWritable();
    case BINARY:
        return new BytesWritable();
    case CHAR:
    case VARCHAR:
    case STRING:
        return new Text();
    case DATE:
        return new DateWritable();
    case TIMESTAMP:
        return new OrcTimestamp();
    case DECIMAL:
        return new HiveDecimalWritable();
    case STRUCT: {
        OrcStruct result = new OrcStruct(type);
        int c = 0;
        for (TypeDescription child : type.getChildren()) {
            result.setFieldValue(c++, createValue(child));
        }
        return result;
    }
    case UNION:
        return new OrcUnion(type);
    case LIST:
        return new OrcList(type);
    case MAP:
        return new OrcMap(type);
    default:
        throw new IllegalArgumentException("Unknown type " + type);
    }
}

---

From source file:org.shaf.core.util.IOUtils.java

License:Apache License

/**
 * Reads an {@link Object} of the specified type from the {@link DataInput}.
 * /*w ww . j av  a2 s.  c  om*/
 * @param cls
 *            the type of the reading object.
 * @param in
 *            the data input stream.
 * @return the read object.
 * @throws IOException
 *             if I/O error occurs.
 */
public static final Object readObject(Class<?> cls, DataInput in) throws IOException {
    try {
        if (cls == null) {
            throw new IOException("Reading class is not defined: null.");
        } else if (ClassUtils.isBoolean(cls)) {
            BooleanWritable obj = new BooleanWritable();
            obj.readFields(in);
            return obj.get();
        } else if (ClassUtils.isByte(cls)) {
            ByteWritable obj = new ByteWritable();
            obj.readFields(in);
            return obj.get();
        } else if (ClassUtils.isShort(cls)) {
            ShortWritable obj = new ShortWritable();
            obj.readFields(in);
            return obj.get();
        } else if (ClassUtils.isInteger(cls)) {
            IntWritable obj = new IntWritable();
            obj.readFields(in);
            return obj.get();
        } else if (ClassUtils.isLong(cls)) {
            LongWritable obj = new LongWritable();
            obj.readFields(in);
            return obj.get();
        } else if (ClassUtils.isFloat(cls)) {
            FloatWritable obj = new FloatWritable();
            obj.readFields(in);
            return obj.get();
        } else if (ClassUtils.isDouble(cls)) {
            DoubleWritable obj = new DoubleWritable();
            obj.readFields(in);
            return obj.get();
        } else if (ClassUtils.isString(cls)) {
            return Text.readString(in);
        } else if (ClassUtils.isEnum(cls)) {
            IntWritable obj = new IntWritable();
            obj.readFields(in);
            return cls.getEnumConstants()[obj.get()];
        } else if (ClassUtils.isArray(cls)) {
            int length = (int) readObject(int.class, in);
            Object array = Array.newInstance(cls.getComponentType(), length);
            for (int j = 0; j < length; j++) {
                Object a = readObject(cls.getComponentType(), in);
                Array.set(array, j, a);
            }
            return array;
        } else {
            Object obj = cls.newInstance();
            ((Writable) obj).readFields(in);
            return obj;
        }
    } catch (IllegalArgumentException | InstantiationException | IllegalAccessException exc) {
        throw new IOException(exc);
    }
}

---

From source file:org.shaf.core.util.IOUtilsTest.java

License:Apache License

/**
 * Test writing of {@code short} value.// w  w w.  jav a2  s  .  c  o m
 */
@Test
public void testWriteShort() {
    byte[] buf = null;

    short value = 123;
    try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
            DataOutputStream out = new DataOutputStream(baos);) {
        IOUtils.writeObject(value, out);
        buf = baos.toByteArray();
    } catch (IOException exc) {
        fail(exc.getMessage());
    }

    try (ByteArrayInputStream bais = new ByteArrayInputStream(buf);
            DataInputStream in = new DataInputStream(bais);) {
        ShortWritable probe = new ShortWritable();
        probe.readFields(in);
        assertEquals(value, probe.get());
    } catch (IOException exc) {
        fail(exc.getMessage());
    }
}

---