Example usage for org.apache.hadoop.io VLongWritable VLongWritable

Introduction

In this page you can find the example usage for org.apache.hadoop.io VLongWritable VLongWritable.

Prototype

public VLongWritable(long value) 

Source Link

Usage

From source file:edu.uci.ics.pregelix.benchmark.io.TextCCInputFormat.java

License:Apache License

@Override
public TextVertexReader createVertexReader(InputSplit split, TaskAttemptContext context) throws IOException {
    return new TextVertexReaderFromEachLine() {
        StringTokenizer items;//  ww w  .  j  av  a2  s  . com

        @Override
        protected VLongWritable getId(Text line) throws IOException {
            items = new StringTokenizer(line.toString());
            return new VLongWritable(Long.parseLong(items.nextToken()));
        }

        @Override
        protected VLongWritable getValue(Text line) throws IOException {
            return null;
        }

        @Override
        protected Iterable<Edge<VLongWritable, NullWritable>> getEdges(Text line) throws IOException {
            List<Edge<VLongWritable, NullWritable>> edges = new ArrayList<Edge<VLongWritable, NullWritable>>();
            Map<VLongWritable, NullWritable> edgeMap = new HashMap<VLongWritable, NullWritable>();
            while (items.hasMoreTokens()) {
                edgeMap.put(new VLongWritable(Long.parseLong(items.nextToken())), null);
            }
            for (Entry<VLongWritable, NullWritable> entry : edgeMap.entrySet()) {
                MapMutableEdge<VLongWritable, NullWritable> edge = new MapMutableEdge<VLongWritable, NullWritable>();
                edge.setEntry(entry);
                edge.setValue(null);
                edges.add(edge);
            }
            return edges;
        }

    };
}

---

From source file:edu.uci.ics.pregelix.benchmark.io.TextPRInputFormat.java

License:Apache License

@Override
public TextVertexReader createVertexReader(InputSplit split, TaskAttemptContext context) throws IOException {
    return new TextVertexReaderFromEachLine() {
        StringTokenizer items;//  w  ww  .j a  v a 2  s  .  co  m

        @Override
        protected VLongWritable getId(Text line) throws IOException {
            items = new StringTokenizer(line.toString());
            return new VLongWritable(Long.parseLong(items.nextToken()));
        }

        @Override
        protected DoubleWritable getValue(Text line) throws IOException {
            return null;
        }

        @Override
        protected Iterable<Edge<VLongWritable, NullWritable>> getEdges(Text line) throws IOException {
            List<Edge<VLongWritable, NullWritable>> edges = new ArrayList<Edge<VLongWritable, NullWritable>>();
            Map<VLongWritable, NullWritable> edgeMap = new HashMap<VLongWritable, NullWritable>();
            while (items.hasMoreTokens()) {
                edgeMap.put(new VLongWritable(Long.parseLong(items.nextToken())), null);
            }
            for (Entry<VLongWritable, NullWritable> entry : edgeMap.entrySet()) {
                MapMutableEdge<VLongWritable, NullWritable> edge = new MapMutableEdge<VLongWritable, NullWritable>();
                edge.setEntry(entry);
                edge.setValue(null);
                edges.add(edge);
            }
            return edges;
        }

    };
}

---

From source file:edu.uci.ics.pregelix.benchmark.io.TextSPInputFormat.java

License:Apache License

@Override
public TextVertexReader createVertexReader(InputSplit split, TaskAttemptContext context) throws IOException {
    return new TextVertexReaderFromEachLine() {
        StringTokenizer items;//from  w  ww .ja v  a2  s. co  m

        @Override
        protected VLongWritable getId(Text line) throws IOException {
            items = new StringTokenizer(line.toString());
            return new VLongWritable(Long.parseLong(items.nextToken()));
        }

        @Override
        protected DoubleWritable getValue(Text line) throws IOException {
            return null;
        }

        @Override
        protected Iterable<Edge<VLongWritable, DoubleWritable>> getEdges(Text line) throws IOException {
            List<Edge<VLongWritable, DoubleWritable>> edges = new ArrayList<Edge<VLongWritable, DoubleWritable>>();
            Map<VLongWritable, DoubleWritable> edgeMap = new HashMap<VLongWritable, DoubleWritable>();
            while (items.hasMoreTokens()) {
                edgeMap.put(new VLongWritable(Long.parseLong(items.nextToken())), null);
            }
            for (Entry<VLongWritable, DoubleWritable> entry : edgeMap.entrySet()) {
                MapMutableEdge<VLongWritable, DoubleWritable> edge = new MapMutableEdge<VLongWritable, DoubleWritable>();
                edge.setEntry(entry);
                edge.setValue(new DoubleWritable(1.0));
                edges.add(edge);
            }
            return edges;
        }

    };
}

---

From source file:edu.uci.ics.pregelix.benchmark.vertex.ConnectedComponentsVertex.java

License:Apache License

/**
 * Propagates the smallest vertex id to all neighbors. Will always choose to
 * halt and only reactivate if a smaller id has been sent to it.
 * // w  ww  .  ja v  a2  s. c  om
 * @param messages
 *            Iterator of messages from the previous superstep.
 * @throws IOException
 */
@Override
public void compute(Iterable<VLongWritable> messages) throws IOException {
    long currentComponent = getValue().get();

    // First superstep is special, because we can simply look at the neighbors
    if (getSuperstep() == 0) {
        for (Edge<VLongWritable, NullWritable> edge : getEdges()) {
            long neighbor = edge.getTargetVertexId().get();
            if (neighbor < currentComponent) {
                currentComponent = neighbor;
            }
        }
        // Only need to send value if it is not the own id
        if (currentComponent != getValue().get()) {
            setValue(new VLongWritable(currentComponent));
            for (Edge<VLongWritable, NullWritable> edge : getEdges()) {
                VLongWritable neighbor = edge.getTargetVertexId();
                if (neighbor.get() > currentComponent) {
                    sendMessage(neighbor, getValue());
                }
            }
        }

        voteToHalt();
        return;
    }

    boolean changed = false;
    // did we get a smaller id ?
    for (VLongWritable message : messages) {
        long candidateComponent = message.get();
        if (candidateComponent < currentComponent) {
            currentComponent = candidateComponent;
            changed = true;
        }
    }

    // propagate new component id to the neighbors
    if (changed) {
        setValue(new VLongWritable(currentComponent));
        sendMessageToAllEdges(getValue());
    }
    voteToHalt();
}

---

From source file:hivemall.utils.hadoop.WritableUtils.java

License:Open Source License

public static Writable toWritable(Object object) {
    if (object == null) {
        return null; //return NullWritable.get();
    }/*from   ww  w  .  j  av  a 2 s . c o m*/
    if (object instanceof Writable) {
        return (Writable) object;
    }
    if (object instanceof String) {
        return new Text((String) object);
    }
    if (object instanceof Long) {
        return new VLongWritable((Long) object);
    }
    if (object instanceof Integer) {
        return new VIntWritable((Integer) object);
    }
    if (object instanceof Byte) {
        return new ByteWritable((Byte) object);
    }
    if (object instanceof Double) {
        return new DoubleWritable((Double) object);
    }
    if (object instanceof Float) {
        return new FloatWritable((Float) object);
    }
    if (object instanceof Boolean) {
        return new BooleanWritable((Boolean) object);
    }
    if (object instanceof byte[]) {
        return new BytesWritable((byte[]) object);
    }
    return new BytesWritable(object.toString().getBytes());
}

---

From source file:org.apache.hcatalog.data.ReaderWriter.java

License:Apache License

public static void writeDatum(DataOutput out, Object val) throws IOException {
    // write the data type
    byte type = DataType.findType(val);
    switch (type) {
    case DataType.LIST:
        out.writeByte(DataType.LIST);/*from   w  ww. java 2 s  .  c o  m*/
        List<?> list = (List<?>) val;
        int sz = list.size();
        out.writeInt(sz);
        for (int i = 0; i < sz; i++) {
            writeDatum(out, list.get(i));
        }
        return;

    case DataType.MAP:
        out.writeByte(DataType.MAP);
        Map<?, ?> m = (Map<?, ?>) val;
        out.writeInt(m.size());
        Iterator<?> i = m.entrySet().iterator();
        while (i.hasNext()) {
            Entry<?, ?> entry = (Entry<?, ?>) i.next();
            writeDatum(out, entry.getKey());
            writeDatum(out, entry.getValue());
        }
        return;

    case DataType.INTEGER:
        out.writeByte(DataType.INTEGER);
        new VIntWritable((Integer) val).write(out);
        return;

    case DataType.LONG:
        out.writeByte(DataType.LONG);
        new VLongWritable((Long) val).write(out);
        return;

    case DataType.FLOAT:
        out.writeByte(DataType.FLOAT);
        out.writeFloat((Float) val);
        return;

    case DataType.DOUBLE:
        out.writeByte(DataType.DOUBLE);
        out.writeDouble((Double) val);
        return;

    case DataType.BOOLEAN:
        out.writeByte(DataType.BOOLEAN);
        out.writeBoolean((Boolean) val);
        return;

    case DataType.BYTE:
        out.writeByte(DataType.BYTE);
        out.writeByte((Byte) val);
        return;

    case DataType.SHORT:
        out.writeByte(DataType.SHORT);
        out.writeShort((Short) val);
        return;

    case DataType.STRING:
        String s = (String) val;
        byte[] utfBytes = s.getBytes(ReaderWriter.UTF8);
        out.writeByte(DataType.STRING);
        out.writeInt(utfBytes.length);
        out.write(utfBytes);
        return;

    case DataType.BINARY:
        byte[] ba = (byte[]) val;
        out.writeByte(DataType.BINARY);
        out.writeInt(ba.length);
        out.write(ba);
        return;

    case DataType.NULL:
        out.writeByte(DataType.NULL);
        return;

    default:
        throw new IOException("Unexpected data type " + type + " found in stream.");
    }
}

---

From source file:org.apache.hive.hcatalog.data.ReaderWriter.java

License:Apache License

public static void writeDatum(DataOutput out, Object val) throws IOException {
    // write the data type
    byte type = DataType.findType(val);
    out.write(type);/*from ww  w  . j  a va2 s  .  c  o  m*/
    switch (type) {
    case DataType.LIST:
        List<?> list = (List<?>) val;
        int sz = list.size();
        out.writeInt(sz);
        for (int i = 0; i < sz; i++) {
            writeDatum(out, list.get(i));
        }
        return;

    case DataType.MAP:
        Map<?, ?> m = (Map<?, ?>) val;
        out.writeInt(m.size());
        Iterator<?> i = m.entrySet().iterator();
        while (i.hasNext()) {
            Entry<?, ?> entry = (Entry<?, ?>) i.next();
            writeDatum(out, entry.getKey());
            writeDatum(out, entry.getValue());
        }
        return;

    case DataType.INTEGER:
        new VIntWritable((Integer) val).write(out);
        return;

    case DataType.LONG:
        new VLongWritable((Long) val).write(out);
        return;

    case DataType.FLOAT:
        out.writeFloat((Float) val);
        return;

    case DataType.DOUBLE:
        out.writeDouble((Double) val);
        return;

    case DataType.BOOLEAN:
        out.writeBoolean((Boolean) val);
        return;

    case DataType.BYTE:
        out.writeByte((Byte) val);
        return;

    case DataType.SHORT:
        out.writeShort((Short) val);
        return;

    case DataType.STRING:
        String s = (String) val;
        byte[] utfBytes = s.getBytes(ReaderWriter.UTF8);
        out.writeInt(utfBytes.length);
        out.write(utfBytes);
        return;

    case DataType.BINARY:
        byte[] ba = (byte[]) val;
        out.writeInt(ba.length);
        out.write(ba);
        return;

    case DataType.NULL:
        //for NULL we just write out the type
        return;
    case DataType.CHAR:
        new HiveCharWritable((HiveChar) val).write(out);
        return;
    case DataType.VARCHAR:
        new HiveVarcharWritable((HiveVarchar) val).write(out);
        return;
    case DataType.DECIMAL:
        new HiveDecimalWritable((HiveDecimal) val).write(out);
        return;
    case DataType.DATE:
        new DateWritable((Date) val).write(out);
        return;
    case DataType.TIMESTAMP:
        new TimestampWritable((java.sql.Timestamp) val).write(out);
        return;
    default:
        throw new IOException("Unexpected data type " + type + " found in stream.");
    }
}

---

From source file:org.elasticsearch.hadoop.serialization.WritableTypeToJsonTest.java

License:Apache License

@Test
public void testVLong() {
    writableTypeToJson(new VLongWritable(Long.MAX_VALUE));
}

---

From source file:org.elasticsearch.hadoop.util.WritableUtils.java

License:Apache License

@SuppressWarnings({ "unchecked", "rawtypes" })
public static Writable toWritable(Object object) {
    if (object instanceof Writable) {
        return (Writable) object;
    }/*from  w w  w . jav  a2 s . com*/
    if (object == null) {
        return NullWritable.get();
    }
    if (object instanceof String) {
        return new Text((String) object);
    }
    if (object instanceof Long) {
        return new VLongWritable((Long) object);
    }
    if (object instanceof Integer) {
        return new VIntWritable((Integer) object);
    }
    if (object instanceof Byte) {
        return new ByteWritable((Byte) object);
    }
    if (object instanceof Short) {
        return WritableCompatUtil.availableShortWritable((Short) object);
    }
    if (object instanceof Double) {
        return new DoubleWritable((Double) object);
    }
    if (object instanceof Float) {
        return new FloatWritable((Float) object);
    }
    if (object instanceof Boolean) {
        return new BooleanWritable((Boolean) object);
    }
    if (object instanceof byte[]) {
        return new BytesWritable((byte[]) object);
    }
    if (object instanceof List) {
        List<Object> list = (List<Object>) object;
        if (!list.isEmpty()) {
            Object first = list.get(0);
            Writable[] content = new Writable[list.size()];
            for (int i = 0; i < content.length; i++) {
                content[i] = toWritable(list.get(i));
            }
            return new ArrayWritable(toWritable(first).getClass(), content);
        }
        return new ArrayWritable(NullWritable.class, new Writable[0]);
    }
    if (object instanceof SortedSet) {
        SortedMapWritable smap = new SortedMapWritable();
        SortedSet<Object> set = (SortedSet) object;
        for (Object obj : set) {
            smap.put((WritableComparable) toWritable(obj), NullWritable.get());
        }
        return smap;
    }
    if (object instanceof Set) {
        MapWritable map = new MapWritable();
        Set<Object> set = (Set) object;
        for (Object obj : set) {
            map.put(toWritable(obj), NullWritable.get());
        }
        return map;
    }
    if (object instanceof SortedMap) {
        SortedMapWritable smap = new SortedMapWritable();
        Map<Object, Object> map = (Map) object;
        for (Map.Entry<?, ?> entry : map.entrySet()) {
            smap.put((WritableComparable) toWritable(entry.getKey()), toWritable(entry.getValue()));
        }
        return smap;
    }
    if (object instanceof Map) {
        MapWritable result = new MapWritable();
        Map<Object, Object> map = (Map) object;
        for (Map.Entry<?, ?> entry : map.entrySet()) {
            result.put(toWritable(entry.getKey()), toWritable(entry.getValue()));
        }
        return result;
    }
    // fall-back to bytearray
    return new BytesWritable(object.toString().getBytes(StringUtils.UTF_8));
}

---

• «
• 1
• 2
• »