List of usage examples for org.apache.hadoop.io MapWritable put
@Override
public Writable put(Writable key, Writable value)
From source file:org.apache.hama.bsp.PartitioningRunner.java
License:Apache License
@Override
@SuppressWarnings({ "rawtypes" })
public void bsp(BSPPeer<Writable, Writable, Writable, Writable, MapWritable> peer)
throws IOException, SyncException, InterruptedException {
Partitioner partitioner = getPartitioner();
KeyValuePair<Writable, Writable> rawRecord = null;
KeyValuePair<Writable, Writable> convertedRecord = null;
Class rawKeyClass = null;/*from ww w.jav a 2s .com*/
Class rawValueClass = null;
MapWritable raw = null;
while ((rawRecord = peer.readNext()) != null) {
if (rawKeyClass == null && rawValueClass == null) {
rawKeyClass = rawRecord.getKey().getClass();
rawValueClass = rawRecord.getValue().getClass();
}
convertedRecord = converter.convertRecord(rawRecord, conf);
if (convertedRecord == null) {
throw new IOException("The converted record can't be null.");
}
int index = converter.getPartitionId(convertedRecord, partitioner, conf, peer, peer.getNumPeers());
raw = new MapWritable();
raw.put(rawRecord.getKey(), rawRecord.getValue());
peer.send(peer.getPeerName(index), raw);
}
peer.sync();
MapWritable record;
while ((record = peer.getCurrentMessage()) != null) {
for (Map.Entry<Writable, Writable> e : record.entrySet()) {
peer.write(e.getKey(), e.getValue());
}
}
}
From source file:org.apache.hama.examples.ClassSerializePrinting.java
License:Apache License
@Override public void bsp(BSPPeer<NullWritable, NullWritable, IntWritable, Text, MapWritable> bspPeer) throws IOException, SyncException, InterruptedException { for (int i = 0; i < NUM_SUPERSTEPS; i++) { for (String otherPeer : bspPeer.getAllPeerNames()) { MapWritable map = new MapWritable(); map.put(new Text(bspPeer.getPeerName()), new IntWritable(i)); bspPeer.send(otherPeer, map); }// w ww . j ava 2 s . c o m // Test superstep counter if (i != bspPeer.getSuperstepCount()) { throw new IOException(); } bspPeer.sync(); MapWritable msg = null; while ((msg = bspPeer.getCurrentMessage()) != null) { for (Entry<Writable, Writable> e : msg.entrySet()) { bspPeer.write((IntWritable) e.getValue(), (Text) e.getKey()); } } } }
From source file:org.apache.hama.graph.AggregationRunner.java
License:Apache License
/** * Runs the aggregators by sending their values to the master task. * //ww w . j ava 2s.c o m * @param changedVertexCnt */ public void sendAggregatorValues(BSPPeer<Writable, Writable, Writable, Writable, GraphJobMessage> peer, int activeVertices, int changedVertexCnt) throws IOException { // send msgCounts to the master task MapWritable updatedCnt = new MapWritable(); updatedCnt.put(GraphJobRunner.FLAG_MESSAGE_COUNTS, new IntWritable(activeVertices)); // send total number of vertices changes updatedCnt.put(GraphJobRunner.FLAG_VERTEX_ALTER_COUNTER, new LongWritable(changedVertexCnt)); // also send aggregated values to the master if (aggregators != null) { for (int i = 0; i < this.aggregators.length; i++) { updatedCnt.put(aggregatorValueFlag[i], aggregators[i].getValue()); if (isAbstractAggregator[i]) { updatedCnt.put(aggregatorIncrementFlag[i], ((AbstractAggregator<M>) aggregators[i]).getTimesAggregated()); } } for (int i = 0; i < aggregators.length; i++) { // now create new aggregators for the next iteration aggregators[i] = getNewAggregator(aggregatorClassNames[i]); if (GraphJobRunner.isMasterTask(peer)) { masterAggregator[i] = getNewAggregator(aggregatorClassNames[i]); } } } peer.send(GraphJobRunner.getMasterTask(peer), new GraphJobMessage(updatedCnt)); }
From source file:org.apache.hama.graph.AggregationRunner.java
License:Apache License
/** * The method the master task does, it globally aggregates the values of each * peer and updates the given map accordingly. *//* www . j ava 2 s .com*/ public void doMasterAggregation(MapWritable updatedCnt) { if (isEnabled()) { // work through the master aggregators for (int i = 0; i < masterAggregator.length; i++) { Writable lastAggregatedValue = masterAggregator[i].getValue(); if (isAbstractAggregator[i]) { final AbstractAggregator<M> intern = ((AbstractAggregator<M>) masterAggregator[i]); final Writable finalizeAggregation = intern.finalizeAggregation(); if (intern.finalizeAggregation() != null) { lastAggregatedValue = finalizeAggregation; } // this count is usually the times of active // vertices in the graph updatedCnt.put(aggregatorIncrementFlag[i], intern.getTimesAggregated()); } updatedCnt.put(aggregatorValueFlag[i], lastAggregatedValue); } } }
From source file:org.apache.hama.graph.GraphJobRunner.java
License:Apache License
/** * The master task is going to check the number of updated vertices and do * master aggregation. In case of no aggregators defined, we save a sync by * reading multiple typed messages./*from ww w. jav a2s. c om*/ */ private void doAggregationUpdates(BSPPeer<Writable, Writable, Writable, Writable, GraphJobMessage> peer) throws IOException, SyncException, InterruptedException { // this is only done in every second iteration if (isMasterTask(peer)) { MapWritable updatedCnt = new MapWritable(); // send total number of vertices. updatedCnt.put(FLAG_VERTEX_TOTAL_VERTICES, new LongWritable((peer.getCounter(GraphJobCounter.INPUT_VERTICES).getCounter()))); // exit if there's no update made if (globalUpdateCounts == 0) { updatedCnt.put(FLAG_MESSAGE_COUNTS, new IntWritable(Integer.MIN_VALUE)); } else { getAggregationRunner().doMasterAggregation(updatedCnt); } // send the updates from the master tasks back to the slaves for (String peerName : peer.getAllPeerNames()) { peer.send(peerName, new GraphJobMessage(updatedCnt)); } } if (getAggregationRunner().isEnabled()) { peer.sync(); // now the map message must be read that might be send from the master updated = getAggregationRunner().receiveAggregatedValues(peer.getCurrentMessage().getMap(), iteration); } }
From source file:org.apache.hama.graph.Vertex.java
License:Apache License
@Override public void addVertex(V vertexID, List<Edge<V, E>> edges, M value) throws IOException { MapWritable msg = new MapWritable(); // Create the new vertex. Vertex<V, E, M> vertex = GraphJobRunner.<V, E, M>newVertexInstance(GraphJobRunner.VERTEX_CLASS); vertex.setEdges(edges);//from w w w .ja v a 2 s.c o m vertex.setValue(value); vertex.setVertexID(vertexID); msg.put(GraphJobRunner.FLAG_VERTEX_INCREASE, vertex); runner.getPeer().send(runner.getHostName(vertexID), new GraphJobMessage(msg)); alterVertexCounter(1); }
From source file:org.apache.hama.graph.Vertex.java
License:Apache License
@Override public void remove() throws IOException { MapWritable msg = new MapWritable(); msg.put(GraphJobRunner.FLAG_VERTEX_DECREASE, this.vertexID); // Get master task peer. String destPeer = GraphJobRunner.getMasterTask(this.getPeer()); runner.getPeer().send(destPeer, new GraphJobMessage(msg)); alterVertexCounter(-1);/*from w w w . j a v a 2 s .c om*/ }
From source file:org.apache.hama.ml.recommendation.cf.OnlineTrainBSP.java
License:Apache License
private DoubleMatrix normalizeMatrix(BSPPeer<Text, VectorWritable, Text, VectorWritable, MapWritable> peer, DoubleMatrix featureMatrix, IntWritable msgFeatureMatrix, boolean broadcast) throws IOException, SyncException, InterruptedException { // send to master peer MapWritable msg = new MapWritable(); MatrixWritable mtx = new MatrixWritable(featureMatrix); msg.put(msgFeatureMatrix, mtx); String master = peer.getPeerName(peer.getNumPeers() / 2); peer.send(master, msg);/* www . jav a 2s .c om*/ peer.sync(); // normalize DoubleMatrix res = null; if (peer.getPeerName().equals(master)) { res = new DenseDoubleMatrix(featureMatrix.getRowCount(), featureMatrix.getColumnCount(), 0); int incomingMsgCount = 0; while ((msg = peer.getCurrentMessage()) != null) { MatrixWritable tmp = (MatrixWritable) msg.get(msgFeatureMatrix); res.add(tmp.getMatrix()); incomingMsgCount++; } res.divide(incomingMsgCount); } if (broadcast) { if (peer.getPeerName().equals(master)) { // broadcast to all msg = new MapWritable(); msg.put(msgFeatureMatrix, new MatrixWritable(res)); // send to all for (String peerName : peer.getAllPeerNames()) { peer.send(peerName, msg); } } peer.sync(); // receive normalized value from master msg = peer.getCurrentMessage(); featureMatrix = ((MatrixWritable) msg.get(msgFeatureMatrix)).getMatrix(); } return res; }
From source file:org.apache.hama.ml.recommendation.cf.OnlineTrainBSP.java
License:Apache License
private void sendTo(BSPPeer<Text, VectorWritable, Text, VectorWritable, MapWritable> peer, HashMap<Text, LinkedList<IntWritable>> senderList, HashMap<Text, DoubleVector> normalizedValues) throws IOException { for (Map.Entry<Text, DoubleVector> e : normalizedValues.entrySet()) { MapWritable msgTmp = new MapWritable(); // send to interested peers msgTmp.put(OnlineCF.Settings.MSG_ITEM_MATRIX, e.getKey()); msgTmp.put(OnlineCF.Settings.MSG_VALUE, new VectorWritable(e.getValue())); Iterator<IntWritable> iter = senderList.get(e.getKey()).iterator(); while (iter.hasNext()) { peer.send(peer.getPeerName(iter.next().get()), msgTmp); }// ww w . j a v a2s. c om } }
From source file:org.apache.hama.ml.recommendation.cf.OnlineTrainBSP.java
License:Apache License
private void sendItemFactorizedValues(BSPPeer<Text, VectorWritable, Text, VectorWritable, MapWritable> peer) throws IOException, SyncException, InterruptedException { int peerCount = peer.getNumPeers(); // item factorized values should be normalized IntWritable peerId = new IntWritable(peer.getPeerIndex()); for (Map.Entry<String, VectorWritable> item : itemsMatrix.entrySet()) { MapWritable msg = new MapWritable(); msg.put(OnlineCF.Settings.MSG_ITEM_MATRIX, new Text(item.getKey())); msg.put(OnlineCF.Settings.MSG_VALUE, item.getValue()); msg.put(OnlineCF.Settings.MSG_SENDER_ID, peerId); peer.send(peer.getPeerName(item.getKey().hashCode() % peerCount), msg); }/* ww w . j a va 2 s . c om*/ }