List of usage examples for org.apache.hadoop.io LongWritable get
public long get()
From source file:gr.ntua.ece.cslab.modissense.queries.clients.mr.GeneralHotIntQueryReducer.java
@Override protected void reduce(LongWritable key, Iterable<HotnessInterestWritable> values, Context context) throws IOException, InterruptedException { Iterator<HotnessInterestWritable> it = values.iterator(); Integer hotness = 0;/*from w ww .j a v a 2 s. c om*/ Double interest = 0d; while (it.hasNext()) { HotnessInterestWritable c = it.next(); hotness += c.getHotness(); interest += c.getInterest(); } Put put = new Put(Bytes.toBytes(key.get())); put.add("cf".getBytes(), "hotness".getBytes(), Bytes.toBytes(hotness)); put.add("cf".getBytes(), "interest".getBytes(), Bytes.toBytes(interest)); context.write(null, put); }
From source file:graphvis.engine.FruchtermanReingoldGraphVis.java
License:MIT License
/** * Apply the FruchtermanReingold algorithm on every vertex. Divided in to 6 supersteps. * @param vertex the vertex to calculate on * @param messages messages received from previous superstep * @throws IOException/* www .ja v a 2 s . c o m*/ */ @Override public void compute(Vertex<LongWritable, VertexValueWritable, EdgeValueWritable> vertex, Iterable<MessageWritable> messages) throws IOException { // Super Step 0 if (getSuperstep() % SUPERSTEPS == 0) { // Everybody is awake //Get default aggregator values. double T = ((DoubleWritable) aggregator.getAggregatedValue("T")).get(); double k = ((DoubleWritable) aggregator.getAggregatedValue("k")).get(); // Very first superstep: init in random position if (getSuperstep() == 0) { Random random = new Random(); VectorWritable pos = new VectorWritable((random.nextDouble() - 0.5) * 100.0, (random.nextDouble() - 0.5) * 100.0); VectorWritable disp = new VectorWritable(0.0, 0.0); VertexValueWritable vertexValue = new VertexValueWritable(pos, disp); vertex.setValue(vertexValue); if (vertex.getId().get() == 1) { //Initialize aggregator values. aggregator.aggregate("k", new DoubleWritable(-k + Math.sqrt(AREA / getTotalNumVertices()))); aggregator.aggregate("T", new DoubleWritable(-T + W / 10)); T = W / 10; } } else { // If it's not the very first superstep, let's chill! if (vertex.getId().get() == 1) { //cool aggregator.aggregate("T", new DoubleWritable(-SPEED)); T = T - SPEED; } } // If we're not frozen yet, wake everyone up and send own position to everyone for next superstep if (T > 0 || (T < MIN_DIST && T > -MIN_DIST && vertex.getId().get() == 1)) { LongWritable ownId = vertex.getId(); long intOwnId = ownId.get(); VectorWritable ownPos = new VectorWritable(vertex.getValue().getPos().getX(), vertex.getValue().getPos().getY()); long totalVertices = getTotalNumVertices(); // We assume that vertices are numbered 1..n where n is the number // of vertices for (long i = 1; i <= totalVertices; i++) { // Send position messages to everyone except self if (i != intOwnId) { sendMessage(new LongWritable(i), new MessageWritable(ownId, ownPos)); } } } } else if (getSuperstep() % SUPERSTEPS == 1) { // Everybody is awake // calculate repulsive forces between everyone VertexValueWritable vertexValue = vertex.getValue(); VectorWritable pos = vertexValue.getPos(); // We start with zero displacement VectorWritable disp = new VectorWritable(0.0, 0.0); for (MessageWritable messageWritable : messages) { VectorWritable otherPos = messageWritable.getPos(); VectorWritable delta = pos.subtract(otherPos); double deltaLength = delta.length(); // if dots are in the same place, let's try to separate them if (deltaLength < MIN_DIST) { delta = makeUpDelta(); deltaLength = delta.length(); } // Update displacement disp = disp.add(delta.multiply(fr(deltaLength) / deltaLength)); } // set new disp vertex.setValue(new VertexValueWritable(pos, disp)); // Send position to neighbors VectorWritable ownPos = new VectorWritable(pos.getX(), pos.getY()); LongWritable ownId = vertex.getId(); for (Edge<LongWritable, EdgeValueWritable> edge : vertex.getEdges()) { sendMessage(edge.getTargetVertexId(), new MessageWritable(ownId, ownPos)); } } else if (getSuperstep() % SUPERSTEPS == 2) { // Vertices with in-edges are awake // anyone who received a message, calculate displacement, then // reply and wait, because in the next step they might get more messages // from vertices which are connected by out-edges // param3 true: send a message back with position applyAttractiveForces(vertex, messages, true); // if there are no out-edges, move, otherwise wait until next step and move then if (vertex.getNumEdges() == 0) { move(vertex); } } else if (getSuperstep() % SUPERSTEPS == 3) { // Vertices with out-edges are awake // param3 true: no need to send another message back applyAttractiveForces(vertex, messages, false); // move, those who don't have out-edges have already moved move(vertex); // Wake up vertices with in-edges for (Edge<LongWritable, EdgeValueWritable> edge : vertex.getEdges()) { sendMessage(edge.getTargetVertexId(), new MessageWritable(vertex.getId(), new VectorWritable())); } } else if (getSuperstep() % SUPERSTEPS == 4) { // Vertices with in-edges are awake LongWritable ownId = new LongWritable(vertex.getId().get()); VectorWritable ownPos = new VectorWritable(vertex.getValue().getPos().getX(), vertex.getValue().getPos().getY()); // Send new position back to everyone from whom a msg was rcvd for (MessageWritable messageWritable : messages) { sendMessage(messageWritable.getSrcId(), new MessageWritable(ownId, ownPos)); } } else if (getSuperstep() % SUPERSTEPS == 5) { // Vertices with out-edges are awake // Set neighbor's position in edge value for (MessageWritable messageWritable : messages) { long srcId = messageWritable.getSrcId().get(); VectorWritable pos = new VectorWritable(messageWritable.getPos().getX(), messageWritable.getPos().getY()); for (Edge<LongWritable, EdgeValueWritable> edge : vertex.getEdges()) { long targetId = edge.getTargetVertexId().get(); if (targetId == srcId) { // Keep weight LongWritable weight = new LongWritable(edge.getValue().getWeight().get()); vertex.setEdgeValue(new LongWritable(targetId), new EdgeValueWritable(weight, pos)); } } } // Wake everyone up for the next superstep, including self long totalVertices = getTotalNumVertices(); for (int i = 1; i <= totalVertices; i++) { sendMessage(new LongWritable(i), new MessageWritable()); } } vertex.voteToHalt(); }
From source file:hivemall.mix.MixMessageEncoder.java
License:Open Source License
private static void encodeObject(final Object obj, final ByteBuf buf) throws IOException { assert (obj != null); if (obj instanceof Integer) { Integer i = (Integer) obj; buf.writeByte(INTEGER_TYPE);//from ww w. j a v a 2s .co m buf.writeInt(i.intValue()); } else if (obj instanceof Text) { Text t = (Text) obj; byte[] b = t.getBytes(); int length = t.getLength(); buf.writeByte(TEXT_TYPE); buf.writeInt(length); buf.writeBytes(b, 0, length); } else if (obj instanceof String) { String s = (String) obj; buf.writeByte(STRING_TYPE); writeString(s, buf); } else if (obj instanceof IntWritable) { IntWritable i = (IntWritable) obj; buf.writeByte(INT_WRITABLE_TYPE); buf.writeInt(i.get()); } else if (obj instanceof LongWritable) { LongWritable l = (LongWritable) obj; buf.writeByte(LONG_WRITABLE_TYPE); buf.writeLong(l.get()); } else { throw new IllegalStateException("Unexpected type: " + obj.getClass().getName()); } }
From source file:hr.fer.tel.rovkp.homework02.task01.TripTimesMapper.java
@Override protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { DebsRecordParser parser = new DebsRecordParser(); // Skip headers if (key.get() == 0) return;//from w w w . j a v a2 s . c om // Parse and emit String record = value.toString(); try { parser.parse(record); int secs = parser.getTripTimeInSecs(); String medallion = parser.getMedallion(); context.write(new Text(medallion), new TripTimesTuple(secs, secs, secs)); } catch (Exception ex) { System.err.println(ex); } }
From source file:hr.fer.tel.rovkp.homework02.task02.LocationsMapper.java
@Override protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { String record = value.toString(); // Skip headers if (key.get() == 0) return;//from w w w. j a v a 2 s.co m DebsRecordParser parser = new DebsRecordParser(); try { parser.parse(record); context.write(new IntWritable(parser.getPassengerCategory()), new Text(parser.getInput())); } catch (ParseException ex) { System.err.println(ex); } }
From source file:in.dream_lab.goffish.giraph.examples.SubgraphTriangleCount.java
License:Apache License
@Override public void compute(Iterable<IMessage<LongWritable, BytesWritable>> subgraphMessages) throws IOException { // Convert adjacency list to adjacency set ISubgraph<TriangleCountSubgraphValue, LongWritable, NullWritable, LongWritable, LongWritable, LongWritable> subgraph = getSubgraph(); if (getSuperstep() == 0) { TriangleCountSubgraphValue triangleCountSubgraphValue = new TriangleCountSubgraphValue(); triangleCountSubgraphValue.adjSet = new HashMap<Long, Set<Long>>(); subgraph.setSubgraphValue(triangleCountSubgraphValue); for (IVertex<LongWritable, NullWritable, LongWritable, LongWritable> vertex : subgraph .getLocalVertices()) {// w ww .ja va2 s. c o m Set<Long> adjVertices = new HashSet<Long>(); for (IEdge<NullWritable, LongWritable, LongWritable> edge : vertex.getOutEdges()) { adjVertices.add(edge.getSinkVertexId().get()); } triangleCountSubgraphValue.adjSet.put(vertex.getVertexId().get(), adjVertices); } return; } else if (getSuperstep() == 1) { TriangleCountSubgraphValue triangleCountSubgraphValue = subgraph.getSubgraphValue(); long triangleCount = triangleCountSubgraphValue.triangleCount; Map<LongWritable, ExtendedByteArrayDataOutput> msg = new HashMap<>(); for (IVertex<LongWritable, NullWritable, LongWritable, LongWritable> vertex : subgraph .getLocalVertices()) { for (IEdge<NullWritable, LongWritable, LongWritable> edge : vertex.getOutEdges()) { IVertex<LongWritable, NullWritable, LongWritable, LongWritable> adjVertex = subgraph .getVertexById(edge.getSinkVertexId()); // Preparing messages to be sent to remote adjacent vertices. if (adjVertex.isRemote() && adjVertex.getVertexId().get() > vertex.getVertexId().get()) { LongWritable remoteSubgraphId = ((IRemoteVertex<LongWritable, NullWritable, LongWritable, LongWritable, LongWritable>) adjVertex) .getSubgraphId(); ExtendedByteArrayDataOutput vertexIds = msg.get(remoteSubgraphId); if (vertexIds == null) { vertexIds = new ExtendedByteArrayDataOutput(); msg.put(remoteSubgraphId, vertexIds); } vertexIds.writeLong(adjVertex.getVertexId().get()); vertexIds.writeLong(vertex.getVertexId().get()); vertexIds.writeLong(vertex.getVertexId().get()); } else if (adjVertex.isRemote() || vertex.getVertexId().get() > adjVertex.getVertexId().get()) continue; if (adjVertex.isRemote()) { continue; //as it has no outedges } // Counting triangles which have at least two vertices in the same // subgraph. for (IEdge<NullWritable, LongWritable, LongWritable> edgeAdjVertex : adjVertex.getOutEdges()) { IVertex<LongWritable, NullWritable, LongWritable, LongWritable> adjAdjVertex = subgraph .getVertexById(edgeAdjVertex.getSinkVertexId()); if (adjAdjVertex.isRemote() || adjAdjVertex.getVertexId().get() > adjVertex.getVertexId().get()) { if (triangleCountSubgraphValue.adjSet.get(vertex.getVertexId().get()) .contains(adjAdjVertex.getVertexId().get())) { triangleCount++; //trianglesList.append(vertex.getVertexID().get() + " " + adjVertex.getVertexID().get() // + " " + adjAdjVertex.getVertexID().get() + "\n"); } } } } } triangleCountSubgraphValue.triangleCount = triangleCount; sendPackedMessages(msg); } else if (getSuperstep() == 2) { Map<Long, List<Pair<Long, Long>>> ids = new HashMap<Long, List<Pair<Long, Long>>>(); unpackMessages(subgraphMessages, ids); Map<LongWritable, ExtendedByteArrayDataOutput> msg = new HashMap<>(); for (Map.Entry<Long, List<Pair<Long, Long>>> entry : ids.entrySet()) { IVertex<LongWritable, NullWritable, LongWritable, LongWritable> vertex = subgraph .getVertexById(new LongWritable(entry.getKey())); List<Pair<Long, Long>> idPairs = entry.getValue(); for (IEdge<NullWritable, LongWritable, LongWritable> edge : vertex.getOutEdges()) { IVertex<LongWritable, NullWritable, LongWritable, LongWritable> adjVertex = subgraph .getVertexById(edge.getSinkVertexId()); if (adjVertex.isRemote() && adjVertex.getVertexId().get() > vertex.getVertexId().get()) { LongWritable remoteSubgraphId = ((IRemoteVertex<LongWritable, NullWritable, LongWritable, LongWritable, LongWritable>) adjVertex) .getSubgraphId(); ExtendedByteArrayDataOutput vertexIds = msg.get(remoteSubgraphId); if (vertexIds == null) { vertexIds = new ExtendedByteArrayDataOutput(); msg.put(remoteSubgraphId, vertexIds); } for (Pair<Long, Long> id : idPairs) { LongWritable firstId = new LongWritable(id.first); IRemoteVertex<LongWritable, NullWritable, LongWritable, LongWritable, LongWritable> sinkSubgraphID = (IRemoteVertex<LongWritable, NullWritable, LongWritable, LongWritable, LongWritable>) subgraph .getVertexById(firstId); if (sinkSubgraphID.getSubgraphId() != remoteSubgraphId) { vertexIds.writeLong(adjVertex.getVertexId().get()); vertexIds.writeLong(firstId.get()); vertexIds.writeLong(vertex.getVertexId().get()); } } } } } sendPackedMessages(msg); } else { long triangleCount = subgraph.getSubgraphValue().triangleCount; Map<Long, List<Pair<Long, Long>>> ids = new HashMap<Long, List<Pair<Long, Long>>>(); unpackMessages(subgraphMessages, ids); for (Map.Entry<Long, List<Pair<Long, Long>>> entry : ids.entrySet()) { IVertex<LongWritable, NullWritable, LongWritable, LongWritable> vertex = subgraph .getVertexById(new LongWritable(entry.getKey())); for (Pair<Long, Long> p : entry.getValue()) { for (IEdge<NullWritable, LongWritable, LongWritable> edge : vertex.getOutEdges()) { if (edge.getSinkVertexId().get() == p.first) { triangleCount++; } } } } subgraph.getSubgraphValue().triangleCount = triangleCount; } voteToHalt(); }
From source file:in.dream_lab.goffish.hama.DenseGraphLongTextAdjacencyListReader.java
License:Apache License
@Override public List<ISubgraph<S, V, E, LongWritable, LongWritable, LongWritable>> getSubgraphs() throws IOException, SyncException, InterruptedException { /* Used for logging */ Runtime runtime = Runtime.getRuntime(); int mb = 1024 * 1024; LOG.info("Free Memory in Reader: " + runtime.freeMemory() / mb + " Total Memory: " + runtime.totalMemory() / mb); LOG.info("Free Memory after Reaching reader " + Runtime.getRuntime().freeMemory()); KeyValuePair<Writable, Writable> pair; long edgeCount = 0; vertexMap = Maps.newHashMap();/*from w w w . j a va 2 s. co m*/ remoteVertexMap = Maps.newHashMap(); LOG.info("SETUP Starting Free Memory: " + runtime.freeMemory() / mb + " Total Memory: " + runtime.totalMemory() / mb); LOG.info("SETUP Starting " + peer.getPeerIndex() + " Memory: " + Runtime.getRuntime().freeMemory()); int count = -1; while ((pair = peer.readNext()) != null) { count++; // NOTE: Confirm that data starts from value and not from key. String stringInput = pair.getValue().toString(); String vertexValue[] = stringInput.split("\\s+"); LongWritable vertexID = new LongWritable(Long.parseLong(vertexValue[0])); List<IEdge<E, LongWritable, LongWritable>> _adjList = new ArrayList<IEdge<E, LongWritable, LongWritable>>(); for (int j = 1; j < vertexValue.length; j++) { LongWritable sinkID = new LongWritable(Long.parseLong(vertexValue[j])); LongWritable edgeID = new LongWritable(edgeCount++ | (((long) peer.getPeerIndex()) << 32)); Edge<E, LongWritable, LongWritable> e = new Edge<E, LongWritable, LongWritable>(edgeID, sinkID); _adjList.add(e); } vertexMap.put(vertexID.get(), createVertexInstance(vertexID, _adjList)); } LOG.info("Number of Vertices: " + vertexMap.size() + " Edges: " + edgeCount); LOG.info("Free Memory: " + runtime.freeMemory() / mb + " Total Memory: " + runtime.totalMemory() / mb); System.gc(); peer.sync(); LOG.info("Creating Remote Vertex Objects"); /* Create remote vertex objects. */ for (IVertex<V, E, LongWritable, LongWritable> vertex : vertexMap.values()) { for (IEdge<E, LongWritable, LongWritable> e : vertex.getOutEdges()) { LongWritable sinkID = e.getSinkVertexId(); if (!vertexMap.containsKey(sinkID.get())) { IRemoteVertex<V, E, LongWritable, LongWritable, LongWritable> sink = new RemoteVertex<>(sinkID); remoteVertexMap.put(sinkID.get(), sink); } } } peer.sync(); Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition = new Partition<>( peer.getPeerIndex()); LOG.info("Calling formSubgraph()"); formSubgraphs(partition); //clearing used memory vertexMap = null; LOG.info("Done with formSubgraph()"); /* * Tell other partitions our Vertex Ids and their subgraphIDs */ Message<LongWritable, LongWritable> question = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); question.setControlInfo(controlInfo); /* * Message format being sent: subgraphID1 count1 vertex1 vertex2 ... subgraphID2 count2 vertex1 vertex2 ... */ for (ISubgraph<S, V, E, LongWritable, LongWritable, LongWritable> subgraphs : partition.getSubgraphs()) { controlInfo.addextraInfo(Longs.toByteArray(subgraphs.getSubgraphId().get())); controlInfo.addextraInfo(Longs.toByteArray(subgraphs.getLocalVertexCount())); for (IVertex<V, E, LongWritable, LongWritable> v : subgraphs.getLocalVertices()) { byte vertexIDbytes[] = Longs.toByteArray(v.getVertexId().get()); controlInfo.addextraInfo(vertexIDbytes); } } sendToAllPartitions(question); LOG.info("Completed first superstep in reader"); System.out.println("Before 2nd Superstep " + (runtime.totalMemory() - runtime.freeMemory()) / mb); peer.sync(); LOG.info("Started superstep 2 in reader"); Message<LongWritable, LongWritable> msg; Map<Integer, List<Message<LongWritable, LongWritable>>> replyMessages = new HashMap<Integer, List<Message<LongWritable, LongWritable>>>(); // Receiving 1 message per partition while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { /* * Subgraph Partition mapping broadcast Format of received message: * partitionID subgraphID1 subgraphID2 ... */ if (msg.getMessageType() == Message.MessageType.SUBGRAPH) { Iterable<BytesWritable> subgraphList = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Integer partitionID = Ints.fromByteArray(subgraphList.iterator().next().getBytes()); for (BytesWritable subgraphListElement : Iterables.skip(subgraphList, 1)) { LongWritable subgraphID = new LongWritable(Longs.fromByteArray(subgraphListElement.getBytes())); subgraphPartitionMap.put((K) subgraphID, partitionID); } continue; } /* * receiving vertices to set Remote Vertex subgraph id. */ Iterator<BytesWritable> remoteVertexQuery = ((ControlMessage) msg.getControlInfo()).getExtraInfo() .iterator(); while (remoteVertexQuery.hasNext()) { Long subgraphID = Longs.fromByteArray(remoteVertexQuery.next().getBytes()); Long vertexCount = Longs.fromByteArray(remoteVertexQuery.next().getBytes()); for (long i = 0; i < vertexCount; i++) { Long remoteVertexID = Longs.fromByteArray(remoteVertexQuery.next().getBytes()); if (remoteVertexMap.containsKey(remoteVertexID)) { RemoteVertex<V, E, LongWritable, LongWritable, LongWritable> sink = (RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>) remoteVertexMap .get(remoteVertexID); sink.setSubgraphID(new LongWritable(subgraphID)); } } } } LOG.info("Completed 2nd superstep in reader"); LOG.info("Reader finished"); return partition.getSubgraphs(); }
From source file:in.dream_lab.goffish.hama.DenseGraphLongTextAdjacencyListReader.java
License:Apache License
void formSubgraphs(Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition) throws IOException, SyncException, InterruptedException { long subgraphCount = 0; Message<LongWritable, LongWritable> subgraphLocationBroadcast = new Message<LongWritable, LongWritable>(); subgraphLocationBroadcast.setMessageType(Message.MessageType.SUBGRAPH); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); subgraphLocationBroadcast.setControlInfo(controlInfo); byte partitionBytes[] = Ints.toByteArray(peer.getPeerIndex()); controlInfo.addextraInfo(partitionBytes); // initialize disjoint set DisjointSets<IVertex<V, E, LongWritable, LongWritable>> ds = new DisjointSets<IVertex<V, E, LongWritable, LongWritable>>( vertexMap.size() + remoteVertexMap.size()); for (IVertex<V, E, LongWritable, LongWritable> vertex : vertexMap.values()) { ds.addSet(vertex);/*from w w w. j a v a 2 s . c o m*/ } for (IVertex<V, E, LongWritable, LongWritable> vertex : remoteVertexMap.values()) { ds.addSet(vertex); } // union edge pairs for (IVertex<V, E, LongWritable, LongWritable> vertex : vertexMap.values()) { for (IEdge<E, LongWritable, LongWritable> edge : vertex.getOutEdges()) { IVertex<V, E, LongWritable, LongWritable> sink = vertexMap.get(edge.getSinkVertexId().get()); if (sink == null) { sink = remoteVertexMap.get(edge.getSinkVertexId().get()); } ds.union(vertex, sink); } } Collection<? extends Collection<IVertex<V, E, LongWritable, LongWritable>>> components = ds.retrieveSets(); for (Collection<IVertex<V, E, LongWritable, LongWritable>> component : components) { LongWritable subgraphID = new LongWritable( subgraphCount++ | (((long) partition.getPartitionId()) << 32)); Subgraph<S, V, E, LongWritable, LongWritable, LongWritable> subgraph = new Subgraph<S, V, E, LongWritable, LongWritable, LongWritable>( peer.getPeerIndex(), subgraphID); for (IVertex<V, E, LongWritable, LongWritable> vertex : component) { subgraph.addVertex(vertex); // Dont add remote vertices to the VertexSubgraphMap as remote vertex // subgraphID is unknown if (!vertex.isRemote()) { vertexSubgraphMap.put(vertex.getVertexId(), subgraph.getSubgraphId()); } } partition.addSubgraph(subgraph); byte subgraphIDbytes[] = Longs.toByteArray(subgraphID.get()); controlInfo.addextraInfo(subgraphIDbytes); } sendToAllPartitions(subgraphLocationBroadcast); }
From source file:in.dream_lab.goffish.hama.LongTextAdjacencyListReader.java
License:Apache License
@Override public List<ISubgraph<S, V, E, LongWritable, LongWritable, LongWritable>> getSubgraphs() throws IOException, SyncException, InterruptedException { KeyValuePair<Writable, Writable> pair; long edgeCount = 0; vertexMap = Maps.newHashMap();//from ww w . jav a2 s .c om remoteVertexMap = Maps.newHashMap(); while ((pair = peer.readNext()) != null) { String stringInput = pair.getValue().toString(); String vertexValue[] = stringInput.split("\\s+"); LongWritable vertexID = new LongWritable(Long.parseLong(vertexValue[0])); Vertex<V, E, LongWritable, LongWritable> vertex = new Vertex<V, E, LongWritable, LongWritable>( vertexID); for (int j = 1; j < vertexValue.length; j++) { LongWritable sinkID = new LongWritable(Long.parseLong(vertexValue[j])); LongWritable edgeID = new LongWritable(edgeCount++ | (((long) peer.getPeerIndex()) << 32)); Edge<E, LongWritable, LongWritable> e = new Edge<E, LongWritable, LongWritable>(edgeID, sinkID); vertex.addEdge(e); } vertexMap.put(vertexID.get(), vertex); } /* Create remote vertex objects. */ for (IVertex<V, E, LongWritable, LongWritable> vertex : vertexMap.values()) { for (IEdge<E, LongWritable, LongWritable> e : vertex.getOutEdges()) { LongWritable sinkID = e.getSinkVertexId(); if (!vertexMap.containsKey(sinkID.get())) { IRemoteVertex<V, E, LongWritable, LongWritable, LongWritable> sink = new RemoteVertex<>(sinkID); remoteVertexMap.put(sinkID.get(), sink); } } } Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition = new Partition<>( peer.getPeerIndex()); formSubgraphs(partition); /* * Ask Remote vertices to send their subgraph IDs. Requires 2 supersteps * because the graph is directed */ Message<LongWritable, LongWritable> question = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); question.setControlInfo(controlInfo); /* * Message format being sent: partitionID remotevertex1 remotevertex2 ... */ byte partitionIDbytes[] = Ints.toByteArray(peer.getPeerIndex()); controlInfo.addextraInfo(partitionIDbytes); for (IVertex<V, E, LongWritable, LongWritable> v : remoteVertexMap.values()) { byte vertexIDbytes[] = Longs.toByteArray(v.getVertexId().get()); controlInfo.addextraInfo(vertexIDbytes); } sendToAllPartitions(question); peer.sync(); Message<LongWritable, LongWritable> msg; Map<Integer, List<Message<LongWritable, LongWritable>>> replyMessages = new HashMap<Integer, List<Message<LongWritable, LongWritable>>>(); // Receiving 1 message per partition while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { /* * Subgraph Partition mapping broadcast Format of received message: * partitionID subgraphID1 subgraphID2 ... */ if (msg.getMessageType() == Message.MessageType.SUBGRAPH) { Iterable<BytesWritable> subgraphList = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Integer partitionID = Ints.fromByteArray(subgraphList.iterator().next().getBytes()); for (BytesWritable subgraphListElement : Iterables.skip(subgraphList, 1)) { LongWritable subgraphID = new LongWritable(Longs.fromByteArray(subgraphListElement.getBytes())); subgraphPartitionMap.put((K) subgraphID, partitionID); } continue; } /* * receiving query to find subgraph id Remote Vertex */ Iterable<BytesWritable> RemoteVertexQuery = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); /* * Reply format : sinkID1 subgraphID1 sinkID2 subgraphID2 ... */ Message<LongWritable, LongWritable> subgraphIDReply = new Message<LongWritable, LongWritable>(); controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.NORMAL); subgraphIDReply.setControlInfo(controlInfo); Integer sinkPartition = Ints.fromByteArray(RemoteVertexQuery.iterator().next().getBytes()); boolean hasAVertex = false; for (BytesWritable remoteVertex : Iterables.skip(RemoteVertexQuery, 1)) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(remoteVertex.getBytes())); LongWritable sinkSubgraphID = vertexSubgraphMap.get(sinkID); // In case this partition does not have the vertex /* * Case 1 : If vertex does not exist Case 2 : If vertex exists but is * remote, then its subgraphID is null */ if (sinkSubgraphID == null) { continue; } hasAVertex = true; byte sinkIDbytes[] = Longs.toByteArray(sinkID.get()); controlInfo.addextraInfo(sinkIDbytes); byte subgraphIDbytes[] = Longs.toByteArray(sinkSubgraphID.get()); controlInfo.addextraInfo(subgraphIDbytes); } if (hasAVertex) { peer.send(peer.getPeerName(sinkPartition.intValue()), (Message<K, M>) subgraphIDReply); } } peer.sync(); while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { Iterable<BytesWritable> remoteVertexReply = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Iterator<BytesWritable> queryResponse = remoteVertexReply.iterator(); while (queryResponse.hasNext()) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(queryResponse.next().getBytes())); LongWritable remoteSubgraphID = new LongWritable( Longs.fromByteArray(queryResponse.next().getBytes())); RemoteVertex<V, E, LongWritable, LongWritable, LongWritable> sink = (RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>) remoteVertexMap .get(sinkID.get()); assert (sink != null); sink.setSubgraphID(remoteSubgraphID); } } return partition.getSubgraphs(); }
From source file:in.dream_lab.goffish.hama.LongTextAdjacencyListReader.java
License:Apache License
void formSubgraphs(Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition) throws IOException { long subgraphCount = 0; Message<LongWritable, LongWritable> subgraphLocationBroadcast = new Message<LongWritable, LongWritable>(); subgraphLocationBroadcast.setMessageType(IMessage.MessageType.SUBGRAPH); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); subgraphLocationBroadcast.setControlInfo(controlInfo); byte partitionBytes[] = Ints.toByteArray(peer.getPeerIndex()); controlInfo.addextraInfo(partitionBytes); // initialize disjoint set DisjointSets<IVertex<V, E, LongWritable, LongWritable>> ds = new DisjointSets<IVertex<V, E, LongWritable, LongWritable>>( vertexMap.size() + remoteVertexMap.size()); for (IVertex<V, E, LongWritable, LongWritable> vertex : vertexMap.values()) { ds.addSet(vertex);/* www . ja va 2 s . com*/ } for (IVertex<V, E, LongWritable, LongWritable> vertex : remoteVertexMap.values()) { ds.addSet(vertex); } // union edge pairs for (IVertex<V, E, LongWritable, LongWritable> vertex : vertexMap.values()) { for (IEdge<E, LongWritable, LongWritable> edge : vertex.getOutEdges()) { IVertex<V, E, LongWritable, LongWritable> sink = vertexMap.get(edge.getSinkVertexId().get()); if (sink == null) { sink = remoteVertexMap.get(edge.getSinkVertexId().get()); } ds.union(vertex, sink); } } Collection<? extends Collection<IVertex<V, E, LongWritable, LongWritable>>> components = ds.retrieveSets(); for (Collection<IVertex<V, E, LongWritable, LongWritable>> component : components) { LongWritable subgraphID = new LongWritable( subgraphCount++ | (((long) partition.getPartitionId()) << 32)); Subgraph<S, V, E, LongWritable, LongWritable, LongWritable> subgraph = new Subgraph<S, V, E, LongWritable, LongWritable, LongWritable>( peer.getPeerIndex(), subgraphID); for (IVertex<V, E, LongWritable, LongWritable> vertex : component) { subgraph.addVertex(vertex); // Dont add remote vertices to the VertexSubgraphMap as remote vertex // subgraphID is unknown if (!vertex.isRemote()) { vertexSubgraphMap.put(vertex.getVertexId(), subgraph.getSubgraphId()); } } partition.addSubgraph(subgraph); byte subgraphIDbytes[] = Longs.toByteArray(subgraphID.get()); controlInfo.addextraInfo(subgraphIDbytes); } sendToAllPartitions(subgraphLocationBroadcast); }