Example usage for org.apache.hadoop.io MapWritable MapWritable

Introduction

In this page you can find the example usage for org.apache.hadoop.io MapWritable MapWritable.

Prototype

public MapWritable()

Source Link

Document

Default constructor.

Usage

From source file:org.apache.hama.graph.GraphJobMessage.java

License:Apache License

@Override
public void readFields(DataInput in) throws IOException {
    flag = in.readByte();/*from   w  ww  . j  a  va  2 s  .com*/
    if (isVertexMessage()) {
        vertexId = GraphJobRunner.createVertexIDObject();
        vertexId.readFields(in);

        this.numOfValues = in.readInt();
        int bytesLength = in.readInt();
        byte[] temp = new byte[bytesLength];
        in.readFully(temp);
        byteBuffer.write(temp);
    } else if (isMapMessage()) {
        map = new MapWritable();
        map.readFields(in);
    } else if (isVerticesSizeMessage()) {
        integerMessage = new IntWritable();
        integerMessage.readFields(in);
    } else if (isPartitioningMessage()) {
        this.numOfValues = in.readInt();
        int bytesLength = in.readInt();
        byte[] temp = new byte[bytesLength];
        in.readFully(temp);
        byteBuffer.write(temp);
    } else {
        vertexId = ReflectionUtils.newInstance(GraphJobRunner.VERTEX_ID_CLASS, null);
        vertexId.readFields(in);
    }
}

From source file:org.apache.hama.graph.GraphJobRunner.java

License:Apache License

/**
 * The master task is going to check the number of updated vertices and do
 * master aggregation. In case of no aggregators defined, we save a sync by
 * reading multiple typed messages./*  w w  w .  java  2 s . c o m*/
 */
private void doAggregationUpdates(BSPPeer<Writable, Writable, Writable, Writable, GraphJobMessage> peer)
        throws IOException, SyncException, InterruptedException {

    // this is only done in every second iteration
    if (isMasterTask(peer)) {
        MapWritable updatedCnt = new MapWritable();
        // send total number of vertices.
        updatedCnt.put(FLAG_VERTEX_TOTAL_VERTICES,
                new LongWritable((peer.getCounter(GraphJobCounter.INPUT_VERTICES).getCounter())));
        // exit if there's no update made
        if (globalUpdateCounts == 0) {
            updatedCnt.put(FLAG_MESSAGE_COUNTS, new IntWritable(Integer.MIN_VALUE));
        } else {
            getAggregationRunner().doMasterAggregation(updatedCnt);
        }
        // send the updates from the master tasks back to the slaves
        for (String peerName : peer.getAllPeerNames()) {
            peer.send(peerName, new GraphJobMessage(updatedCnt));
        }
    }

    if (getAggregationRunner().isEnabled()) {
        peer.sync();
        // now the map message must be read that might be send from the master
        updated = getAggregationRunner().receiveAggregatedValues(peer.getCurrentMessage().getMap(), iteration);
    }
}

From source file:org.apache.hama.graph.TestGraphJobMessage.java

License:Apache License

public List<GraphJobMessage> getMessages() throws IOException {
    GraphJobMessage mapMsg = new GraphJobMessage(new MapWritable());
    GraphJobMessage vertexMsg1 = new GraphJobMessage(new Text("1"), WritableUtils.serialize(new IntWritable()));
    GraphJobMessage vertexMsg2 = new GraphJobMessage(new Text("2"), WritableUtils.serialize(new IntWritable()));
    GraphJobMessage vertexMsg3 = new GraphJobMessage(new Text("3"), WritableUtils.serialize(new IntWritable()));
    return Lists.newArrayList(mapMsg, vertexMsg1, vertexMsg2, vertexMsg3);
}

From source file:org.apache.hama.graph.Vertex.java

License:Apache License

@Override
public void addVertex(V vertexID, List<Edge<V, E>> edges, M value) throws IOException {
    MapWritable msg = new MapWritable();
    // Create the new vertex.
    Vertex<V, E, M> vertex = GraphJobRunner.<V, E, M>newVertexInstance(GraphJobRunner.VERTEX_CLASS);
    vertex.setEdges(edges);//from w w  w. j  a v a 2  s. c om
    vertex.setValue(value);
    vertex.setVertexID(vertexID);

    msg.put(GraphJobRunner.FLAG_VERTEX_INCREASE, vertex);
    runner.getPeer().send(runner.getHostName(vertexID), new GraphJobMessage(msg));

    alterVertexCounter(1);
}

From source file:org.apache.hama.graph.Vertex.java

License:Apache License

@Override
public void remove() throws IOException {
    MapWritable msg = new MapWritable();
    msg.put(GraphJobRunner.FLAG_VERTEX_DECREASE, this.vertexID);

    // Get master task peer.
    String destPeer = GraphJobRunner.getMasterTask(this.getPeer());
    runner.getPeer().send(destPeer, new GraphJobMessage(msg));

    alterVertexCounter(-1);/*w w  w . ja  v  a  2  s. c  om*/
}

From source file:org.apache.hama.ml.recommendation.cf.OnlineTrainBSP.java

License:Apache License

private DoubleMatrix normalizeMatrix(BSPPeer<Text, VectorWritable, Text, VectorWritable, MapWritable> peer,
        DoubleMatrix featureMatrix, IntWritable msgFeatureMatrix, boolean broadcast)
        throws IOException, SyncException, InterruptedException {
    // send to master peer
    MapWritable msg = new MapWritable();
    MatrixWritable mtx = new MatrixWritable(featureMatrix);
    msg.put(msgFeatureMatrix, mtx);//  w w  w .j av  a2  s .  c om
    String master = peer.getPeerName(peer.getNumPeers() / 2);
    peer.send(master, msg);
    peer.sync();

    // normalize
    DoubleMatrix res = null;
    if (peer.getPeerName().equals(master)) {
        res = new DenseDoubleMatrix(featureMatrix.getRowCount(), featureMatrix.getColumnCount(), 0);
        int incomingMsgCount = 0;
        while ((msg = peer.getCurrentMessage()) != null) {
            MatrixWritable tmp = (MatrixWritable) msg.get(msgFeatureMatrix);
            res.add(tmp.getMatrix());
            incomingMsgCount++;
        }
        res.divide(incomingMsgCount);
    }

    if (broadcast) {
        if (peer.getPeerName().equals(master)) {
            // broadcast to all
            msg = new MapWritable();
            msg.put(msgFeatureMatrix, new MatrixWritable(res));
            // send to all
            for (String peerName : peer.getAllPeerNames()) {
                peer.send(peerName, msg);
            }
        }
        peer.sync();
        // receive normalized value from master
        msg = peer.getCurrentMessage();
        featureMatrix = ((MatrixWritable) msg.get(msgFeatureMatrix)).getMatrix();
    }
    return res;
}

From source file:org.apache.hama.ml.recommendation.cf.OnlineTrainBSP.java

License:Apache License

private void sendTo(BSPPeer<Text, VectorWritable, Text, VectorWritable, MapWritable> peer,
        HashMap<Text, LinkedList<IntWritable>> senderList, HashMap<Text, DoubleVector> normalizedValues)
        throws IOException {

    for (Map.Entry<Text, DoubleVector> e : normalizedValues.entrySet()) {
        MapWritable msgTmp = new MapWritable();
        // send to interested peers
        msgTmp.put(OnlineCF.Settings.MSG_ITEM_MATRIX, e.getKey());
        msgTmp.put(OnlineCF.Settings.MSG_VALUE, new VectorWritable(e.getValue()));
        Iterator<IntWritable> iter = senderList.get(e.getKey()).iterator();
        while (iter.hasNext()) {
            peer.send(peer.getPeerName(iter.next().get()), msgTmp);
        }//from www  . ja va 2 s  . c  o m
    }
}

From source file:org.apache.hama.ml.recommendation.cf.OnlineTrainBSP.java

License:Apache License

private void getNormalizedItemFactorizedValues(
        BSPPeer<Text, VectorWritable, Text, VectorWritable, MapWritable> peer,
        HashMap<Text, DoubleVector> normalizedValues, HashMap<Text, LinkedList<IntWritable>> senderList)
        throws IOException {

    HashMap<Text, Integer> normalizedValueCount = new HashMap<Text, Integer>();
    Text itemId = null;//from  www  . j a  v a2s  .  c o  m
    VectorWritable value = null;
    IntWritable senderId = null;
    MapWritable msg = new MapWritable();

    while ((msg = peer.getCurrentMessage()) != null) {
        itemId = (Text) msg.get(OnlineCF.Settings.MSG_ITEM_MATRIX);
        value = (VectorWritable) msg.get(OnlineCF.Settings.MSG_VALUE);
        senderId = (IntWritable) msg.get(OnlineCF.Settings.MSG_SENDER_ID);

        if (normalizedValues.containsKey(itemId) == false) {
            DenseDoubleVector tmp = new DenseDoubleVector(MATRIX_RANK, 0.0);
            normalizedValues.put(itemId, tmp);
            normalizedValueCount.put(itemId, 0);
            senderList.put(itemId, new LinkedList<IntWritable>());
        }

        normalizedValues.put(itemId, normalizedValues.get(itemId).add(value.getVector()));
        normalizedValueCount.put(itemId, normalizedValueCount.get(itemId) + 1);
        senderList.get(itemId).add(senderId);
    }

    // normalize
    for (Map.Entry<Text, DoubleVector> e : normalizedValues.entrySet()) {
        double count = normalizedValueCount.get(e.getKey());
        e.setValue(e.getValue().multiply(1.0 / count));
    }
}

From source file:org.apache.hama.ml.recommendation.cf.OnlineTrainBSP.java

License:Apache License

private void receiveSyncedItemFactorizedValues(
        BSPPeer<Text, VectorWritable, Text, VectorWritable, MapWritable> peer) throws IOException {

    MapWritable msg = new MapWritable();
    Text itemId = null;//from   www .j av  a  2 s.  com
    // messages are arriving take them
    while ((msg = peer.getCurrentMessage()) != null) {
        itemId = (Text) msg.get(OnlineCF.Settings.MSG_ITEM_MATRIX);
        itemsMatrix.put(itemId.toString(), (VectorWritable) msg.get(OnlineCF.Settings.MSG_VALUE));
    }
}

From source file:org.apache.hama.ml.recommendation.cf.OnlineTrainBSP.java

License:Apache License

private void sendItemFactorizedValues(BSPPeer<Text, VectorWritable, Text, VectorWritable, MapWritable> peer)
        throws IOException, SyncException, InterruptedException {
    int peerCount = peer.getNumPeers();
    // item factorized values should be normalized
    IntWritable peerId = new IntWritable(peer.getPeerIndex());

    for (Map.Entry<String, VectorWritable> item : itemsMatrix.entrySet()) {
        MapWritable msg = new MapWritable();
        msg.put(OnlineCF.Settings.MSG_ITEM_MATRIX, new Text(item.getKey()));
        msg.put(OnlineCF.Settings.MSG_VALUE, item.getValue());
        msg.put(OnlineCF.Settings.MSG_SENDER_ID, peerId);
        peer.send(peer.getPeerName(item.getKey().hashCode() % peerCount), msg);
    }//  ww  w.j  ava  2s  . c om
}