List of usage examples for org.apache.hadoop.io MapWritable MapWritable
public MapWritable()
From source file:org.apache.flume.channel.file.FlumeEvent.java
License:Apache License
@Override public void readFields(DataInput in) throws IOException { MapWritable map = new MapWritable(); map.readFields(in);//w w w .ja v a 2 s .c o m setHeaders(fromMapWritable(map)); byte[] body = null; int bodyLength = in.readInt(); if (bodyLength != -1) { body = new byte[bodyLength]; in.readFully(body); } setBody(body); }
From source file:org.apache.flume.channel.file.FlumeEvent.java
License:Apache License
private MapWritable toMapWritable(Map<String, String> map) { MapWritable result = new MapWritable(); if (map != null) { for (Map.Entry<String, String> entry : map.entrySet()) { result.put(new Text(entry.getKey()), new Text(entry.getValue())); }/*from ww w. jav a 2s. c o m*/ } return result; }
From source file:org.apache.flume.channel.recoverable.memory.RecoverableMemoryChannelEvent.java
License:Apache License
@Override public void readFields(DataInput in) throws IOException { sequenceId = in.readLong();/* w ww. j a va 2 s .c o m*/ MapWritable map = new MapWritable(); map.readFields(in); setHeaders(fromMapWritable(map)); byte[] body = null; int bodyLength = in.readInt(); if (bodyLength != -1) { body = new byte[bodyLength]; in.readFully(body); } setBody(body); }
From source file:org.apache.giraph.types.ops.MapTypeOps.java
License:Apache License
@Override public MapWritable create() { return new MapWritable(); }
From source file:org.apache.gora.util.WritableUtils.java
License:Apache License
public static final void writeProperties(DataOutput out, Properties props) throws IOException { MapWritable propsWritable = new MapWritable(); for (Entry<Object, Object> prop : props.entrySet()) { Writable key = new Text(prop.getKey().toString()); Writable value = new Text(prop.getValue().toString()); propsWritable.put(key, value);/*from w w w . j ava 2 s . c o m*/ } propsWritable.write(out); }
From source file:org.apache.gora.util.WritableUtils.java
License:Apache License
public static final Properties readProperties(DataInput in) throws IOException { Properties props = new Properties(); MapWritable propsWritable = new MapWritable(); propsWritable.readFields(in);/*from w w w .jav a2s . co m*/ for (Entry<Writable, Writable> prop : propsWritable.entrySet()) { String key = prop.getKey().toString(); String value = prop.getValue().toString(); props.put(key, value); } return props; }
From source file:org.apache.hama.bsp.PartitioningRunner.java
License:Apache License
@Override
@SuppressWarnings({ "rawtypes" })
public void bsp(BSPPeer<Writable, Writable, Writable, Writable, MapWritable> peer)
throws IOException, SyncException, InterruptedException {
Partitioner partitioner = getPartitioner();
KeyValuePair<Writable, Writable> rawRecord = null;
KeyValuePair<Writable, Writable> convertedRecord = null;
Class rawKeyClass = null;/* w w w . j a v a2 s.co m*/
Class rawValueClass = null;
MapWritable raw = null;
while ((rawRecord = peer.readNext()) != null) {
if (rawKeyClass == null && rawValueClass == null) {
rawKeyClass = rawRecord.getKey().getClass();
rawValueClass = rawRecord.getValue().getClass();
}
convertedRecord = converter.convertRecord(rawRecord, conf);
if (convertedRecord == null) {
throw new IOException("The converted record can't be null.");
}
int index = converter.getPartitionId(convertedRecord, partitioner, conf, peer, peer.getNumPeers());
raw = new MapWritable();
raw.put(rawRecord.getKey(), rawRecord.getValue());
peer.send(peer.getPeerName(index), raw);
}
peer.sync();
MapWritable record;
while ((record = peer.getCurrentMessage()) != null) {
for (Map.Entry<Writable, Writable> e : record.entrySet()) {
peer.write(e.getKey(), e.getValue());
}
}
}
From source file:org.apache.hama.examples.ClassSerializePrinting.java
License:Apache License
@Override public void bsp(BSPPeer<NullWritable, NullWritable, IntWritable, Text, MapWritable> bspPeer) throws IOException, SyncException, InterruptedException { for (int i = 0; i < NUM_SUPERSTEPS; i++) { for (String otherPeer : bspPeer.getAllPeerNames()) { MapWritable map = new MapWritable(); map.put(new Text(bspPeer.getPeerName()), new IntWritable(i)); bspPeer.send(otherPeer, map); }//from w w w. ja v a 2s.c o m // Test superstep counter if (i != bspPeer.getSuperstepCount()) { throw new IOException(); } bspPeer.sync(); MapWritable msg = null; while ((msg = bspPeer.getCurrentMessage()) != null) { for (Entry<Writable, Writable> e : msg.entrySet()) { bspPeer.write((IntWritable) e.getValue(), (Text) e.getKey()); } } } }
From source file:org.apache.hama.graph.AggregationRunner.java
License:Apache License
/** * Runs the aggregators by sending their values to the master task. * //from w ww .j a v a 2s .com * @param changedVertexCnt */ public void sendAggregatorValues(BSPPeer<Writable, Writable, Writable, Writable, GraphJobMessage> peer, int activeVertices, int changedVertexCnt) throws IOException { // send msgCounts to the master task MapWritable updatedCnt = new MapWritable(); updatedCnt.put(GraphJobRunner.FLAG_MESSAGE_COUNTS, new IntWritable(activeVertices)); // send total number of vertices changes updatedCnt.put(GraphJobRunner.FLAG_VERTEX_ALTER_COUNTER, new LongWritable(changedVertexCnt)); // also send aggregated values to the master if (aggregators != null) { for (int i = 0; i < this.aggregators.length; i++) { updatedCnt.put(aggregatorValueFlag[i], aggregators[i].getValue()); if (isAbstractAggregator[i]) { updatedCnt.put(aggregatorIncrementFlag[i], ((AbstractAggregator<M>) aggregators[i]).getTimesAggregated()); } } for (int i = 0; i < aggregators.length; i++) { // now create new aggregators for the next iteration aggregators[i] = getNewAggregator(aggregatorClassNames[i]); if (GraphJobRunner.isMasterTask(peer)) { masterAggregator[i] = getNewAggregator(aggregatorClassNames[i]); } } } peer.send(GraphJobRunner.getMasterTask(peer), new GraphJobMessage(updatedCnt)); }
From source file:org.apache.hama.graph.GraphJobMessage.java
License:Apache License
public void fastReadFields(DataInput in) throws IOException { flag = in.readByte();// w w w . j a v a 2s . c o m if (isVertexMessage()) { vertexId = GraphJobRunner.createVertexIDObject(); vertexId.readFields(in); /* * vertexValue = GraphJobRunner.createVertexValue(); * vertexValue.readFields(in); */ } else if (isMapMessage()) { map = new MapWritable(); map.readFields(in); } else if (isVerticesSizeMessage()) { integerMessage = new IntWritable(); integerMessage.readFields(in); } else { vertexId = ReflectionUtils.newInstance(GraphJobRunner.VERTEX_ID_CLASS, null); vertexId.readFields(in); } }