List of usage examples for org.apache.hadoop.io LongWritable get
public long get()
From source file:in.dream_lab.goffish.hama.LongTextJSONReader.java
License:Apache License
@Override public List<ISubgraph<S, V, E, LongWritable, LongWritable, LongWritable>> getSubgraphs() throws IOException, SyncException, InterruptedException { // Map of partitionID,vertex that do not belong to this partition Map<Integer, List<String>> partitionMap = new HashMap<Integer, List<String>>(); vertexMap = new HashMap<LongWritable, IVertex<V, E, LongWritable, LongWritable>>(); // List of edges.Used to create RemoteVertices List<IEdge<E, LongWritable, LongWritable>> _edges = new ArrayList<IEdge<E, LongWritable, LongWritable>>(); KeyValuePair<Writable, Writable> pair; while ((pair = peer.readNext()) != null) { String StringJSONInput = pair.getValue().toString(); JSONArray JSONInput = (JSONArray) JSONValue.parse(StringJSONInput); int partitionID = Integer.parseInt(JSONInput.get(1).toString()); // Vertex does not belong to this partition if (partitionID != peer.getPeerIndex()) { List<String> partitionVertices = partitionMap.get(partitionID); if (partitionVertices == null) { partitionVertices = new ArrayList<String>(); partitionMap.put(partitionID, partitionVertices); }//from ww w . ja v a2s. c o m partitionVertices.add(StringJSONInput); } else { Vertex<V, E, LongWritable, LongWritable> vertex = createVertex(StringJSONInput); vertexMap.put(vertex.getVertexId(), vertex); _edges.addAll(vertex.getOutEdges()); } } // Send vertices to their respective partitions for (Map.Entry<Integer, List<String>> entry : partitionMap.entrySet()) { int partitionID = entry.getKey().intValue(); List<String> vertices = entry.getValue(); for (String vertex : vertices) { Message<LongWritable, LongWritable> vertexMsg = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.VERTEX); controlInfo.setVertexValues(vertex); vertexMsg.setControlInfo(controlInfo); peer.send(peer.getPeerName(partitionID), (Message<K, M>) vertexMsg); } } //End of first SuperStep peer.sync(); Message<LongWritable, LongWritable> msg; while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { String JSONVertex = msg.getControlInfo().toString(); Vertex<V, E, LongWritable, LongWritable> vertex = createVertex(JSONVertex); vertexMap.put(vertex.getVertexId(), vertex); _edges.addAll(vertex.getOutEdges()); } /* Create remote vertex objects. */ for (IEdge<E, LongWritable, LongWritable> e : _edges) { LongWritable sinkID = e.getSinkVertexId(); IVertex<V, E, LongWritable, LongWritable> sink = vertexMap.get(sinkID); if (sink == null) { sink = new RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>(sinkID); vertexMap.put(sinkID, sink); } } //Direct Copy paste from here Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition = new Partition<S, V, E, LongWritable, LongWritable, LongWritable>( peer.getPeerIndex()); formSubgraphs(partition, vertexMap.values()); /* * Ask Remote vertices to send their subgraph IDs. Requires 2 supersteps * because the graph is directed */ Message<LongWritable, LongWritable> question = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); question.setControlInfo(controlInfo); /* * Message format being sent: * partitionID remotevertex1 remotevertex2 ... */ byte partitionIDbytes[] = Ints.toByteArray(peer.getPeerIndex()); controlInfo.addextraInfo(partitionIDbytes); for (IVertex<V, E, LongWritable, LongWritable> v : vertexMap.values()) { if (v instanceof RemoteVertex) { byte vertexIDbytes[] = Longs.toByteArray(v.getVertexId().get()); controlInfo.addextraInfo(vertexIDbytes); } } sendToAllPartitions(question); peer.sync(); Map<Integer, List<Message<LongWritable, LongWritable>>> replyMessages = new HashMap<Integer, List<Message<LongWritable, LongWritable>>>(); //Receiving 1 message per partition while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { /* * Subgraph Partition mapping broadcast * Format of received message: * partitionID subgraphID1 subgraphID2 ... */ if (msg.getMessageType() == Message.MessageType.SUBGRAPH) { Iterable<BytesWritable> subgraphList = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Integer partitionID = Ints.fromByteArray(subgraphList.iterator().next().getBytes()); for (BytesWritable subgraphListElement : Iterables.skip(subgraphList, 1)) { LongWritable subgraphID = new LongWritable(Longs.fromByteArray(subgraphListElement.getBytes())); subgraphPartitionMap.put((K) subgraphID, partitionID); } continue; } /* * receiving query to find subgraph id Remote Vertex */ Iterable<BytesWritable> RemoteVertexQuery = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); /* * Reply format : * sinkID1 subgraphID1 sinkID2 subgraphID2 ... */ Message<LongWritable, LongWritable> subgraphIDReply = new Message<LongWritable, LongWritable>(); controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.NORMAL); subgraphIDReply.setControlInfo(controlInfo); Integer sinkPartition = Ints.fromByteArray(RemoteVertexQuery.iterator().next().getBytes()); boolean hasAVertex = false; for (BytesWritable remoteVertex : Iterables.skip(RemoteVertexQuery, 1)) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(remoteVertex.getBytes())); LongWritable sinkSubgraphID = vertexSubgraphMap.get(sinkID); //In case this partition does not have the vertex /* Case 1 : If vertex does not exist * Case 2 : If vertex exists but is remote, then its subgraphID is null */ if (sinkSubgraphID == null) { continue; } hasAVertex = true; byte sinkIDbytes[] = Longs.toByteArray(sinkID.get()); controlInfo.addextraInfo(sinkIDbytes); byte subgraphIDbytes[] = Longs.toByteArray(sinkSubgraphID.get()); controlInfo.addextraInfo(subgraphIDbytes); } if (hasAVertex) { peer.send(peer.getPeerName(sinkPartition.intValue()), (Message<K, M>) subgraphIDReply); } } peer.sync(); while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { Iterable<BytesWritable> remoteVertexReply = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Iterator<BytesWritable> queryResponse = remoteVertexReply.iterator(); while (queryResponse.hasNext()) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(queryResponse.next().getBytes())); LongWritable remoteSubgraphID = new LongWritable( Longs.fromByteArray(queryResponse.next().getBytes())); RemoteVertex<V, E, LongWritable, LongWritable, LongWritable> sink = (RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>) vertexMap .get(sinkID); sink.setSubgraphID(remoteSubgraphID); } } return partition.getSubgraphs(); }
From source file:in.dream_lab.goffish.hama.LongTextJSONReader.java
License:Apache License
void formSubgraphs(Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition, Collection<IVertex<V, E, LongWritable, LongWritable>> vertices) throws IOException { long subgraphCount = 0; Set<LongWritable> visited = new HashSet<LongWritable>(); Message<LongWritable, LongWritable> subgraphLocationBroadcast = new Message<LongWritable, LongWritable>(); subgraphLocationBroadcast.setMessageType(IMessage.MessageType.SUBGRAPH); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); subgraphLocationBroadcast.setControlInfo(controlInfo); byte partitionBytes[] = Ints.toByteArray(peer.getPeerIndex()); controlInfo.addextraInfo(partitionBytes); // initialize disjoint set DisjointSets<IVertex<V, E, LongWritable, LongWritable>> ds = new DisjointSets<IVertex<V, E, LongWritable, LongWritable>>( vertices.size());/*from w ww . j a va 2s .c om*/ for (IVertex<V, E, LongWritable, LongWritable> vertex : vertices) { ds.addSet(vertex); } // union edge pairs for (IVertex<V, E, LongWritable, LongWritable> vertex : vertices) { if (!vertex.isRemote()) { for (IEdge<E, LongWritable, LongWritable> edge : vertex.getOutEdges()) { IVertex<V, E, LongWritable, LongWritable> sink = vertexMap.get(edge.getSinkVertexId()); ds.union(vertex, sink); } } } Collection<? extends Collection<IVertex<V, E, LongWritable, LongWritable>>> components = ds.retrieveSets(); for (Collection<IVertex<V, E, LongWritable, LongWritable>> component : components) { LongWritable subgraphID = new LongWritable( subgraphCount++ | (((long) partition.getPartitionId()) << 32)); Subgraph<S, V, E, LongWritable, LongWritable, LongWritable> subgraph = new Subgraph<S, V, E, LongWritable, LongWritable, LongWritable>( peer.getPeerIndex(), subgraphID); for (IVertex<V, E, LongWritable, LongWritable> vertex : component) { subgraph.addVertex(vertex); // Dont add remote vertices to the VertexSubgraphMap as remote vertex subgraphID is unknown if (!vertex.isRemote()) { vertexSubgraphMap.put(vertex.getVertexId(), subgraph.getSubgraphId()); } } partition.addSubgraph(subgraph); byte subgraphIDbytes[] = Longs.toByteArray(subgraphID.get()); controlInfo.addextraInfo(subgraphIDbytes); } sendToAllPartitions(subgraphLocationBroadcast); }
From source file:in.dream_lab.goffish.hama.PartitionsLongTextAdjacencyListReader.java
License:Apache License
@Override public List<ISubgraph<S, V, E, LongWritable, LongWritable, LongWritable>> getSubgraphs() throws IOException, SyncException, InterruptedException { Map<Integer, List<String>> partitionMap = new HashMap<Integer, List<String>>(); vertexMap = new HashMap<LongWritable, IVertex<V, E, LongWritable, LongWritable>>(); // List of edges.Used to create RemoteVertices List<IEdge<E, LongWritable, LongWritable>> _edges = new ArrayList<IEdge<E, LongWritable, LongWritable>>(); long edgeCount = 0; LOG.debug("SETUP Starting " + peer.getPeerIndex() + " Memory: " + Runtime.getRuntime().freeMemory()); KeyValuePair<Writable, Writable> pair; while ((pair = peer.readNext()) != null) { //NOTE: Confirm that data starts from value and not from key. String stringInput = pair.getValue().toString(); String vertexValue[] = stringInput.split("\\s+"); //LongWritable sourceID = new LongWritable(Long.parseLong(value[0])); int partitionID = Integer.parseInt(vertexValue[1]); // Vertex does not belong to this partition if (partitionID != peer.getPeerIndex()) { List<String> partitionVertices = partitionMap.get(partitionID); if (partitionVertices == null) { partitionVertices = new ArrayList<String>(); partitionMap.put(partitionID, partitionVertices); }//from w ww .j a v a 2 s . c o m partitionVertices.add(stringInput); } else { LongWritable vertexID = new LongWritable(Long.parseLong(vertexValue[0])); Vertex<V, E, LongWritable, LongWritable> vertex; vertex = new Vertex<V, E, LongWritable, LongWritable>(vertexID); for (int j = 2; j < vertexValue.length; j++) { LongWritable sinkID = new LongWritable(Long.parseLong(vertexValue[j])); LongWritable edgeID = new LongWritable(edgeCount++ | (((long) peer.getPeerIndex()) << 32)); Edge<E, LongWritable, LongWritable> e = new Edge<E, LongWritable, LongWritable>(edgeID, sinkID); vertex.addEdge(e); _edges.add(e); } vertexMap.put(vertex.getVertexId(), vertex); _edges.addAll(vertex.getOutEdges()); } } // Send vertices to their respective partitions for (Map.Entry<Integer, List<String>> entry : partitionMap.entrySet()) { int partitionID = entry.getKey().intValue(); List<String> vertices = entry.getValue(); for (String vertex : vertices) { Message<LongWritable, LongWritable> vertexMsg = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.VERTEX); controlInfo.setVertexValues(vertex); vertexMsg.setControlInfo(controlInfo); peer.send(peer.getPeerName(partitionID), (Message<K, M>) vertexMsg); } } // End of first superstep. peer.sync(); LOG.debug("Second Superstep in Reader " + peer.getPeerIndex() + " Memory: " + Runtime.getRuntime().freeMemory()); Message<LongWritable, LongWritable> msg; while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { ControlMessage ctrlMessage = (ControlMessage) msg.getControlInfo(); String msgString = ctrlMessage.getVertexValues(); // String msgStringArr[] = msgString.split(","); // for (int i = 0; i < msgStringArr.length; i++) { String vertexInfo[] = msgString.split("\\s+"); LongWritable vertexID = new LongWritable(Long.parseLong(vertexInfo[0])); Vertex<V, E, LongWritable, LongWritable> source = (Vertex<V, E, LongWritable, LongWritable>) vertexMap .get(vertexID); if (source == null) { source = new Vertex<V, E, LongWritable, LongWritable>(vertexID); vertexMap.put(source.getVertexId(), source); } for (int j = 2; j < vertexInfo.length; j++) { LongWritable sinkID = new LongWritable(Long.parseLong(vertexInfo[j])); LongWritable edgeID = new LongWritable(edgeCount++ | (((long) peer.getPeerIndex()) << 32)); Edge<E, LongWritable, LongWritable> e = new Edge<E, LongWritable, LongWritable>(edgeID, sinkID); source.addEdge(e); _edges.add(e); } } //} LOG.debug("Creating Remote Vertex Objects"); /* Create remote vertex objects. */ for (IEdge<E, LongWritable, LongWritable> e : _edges) { LongWritable sinkID = e.getSinkVertexId(); IVertex<V, E, LongWritable, LongWritable> sink = vertexMap.get(sinkID); if (sink == null) { sink = new RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>(sinkID); vertexMap.put(sinkID, sink); } } Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition = new Partition<S, V, E, LongWritable, LongWritable, LongWritable>( peer.getPeerIndex()); LOG.debug("Calling formSubgraph()"); formSubgraphs(partition, vertexMap.values()); LOG.debug("Done with formSubgraph()"); /* * Ask Remote vertices to send their subgraph IDs. Requires 2 supersteps * because the graph is directed */ Message<LongWritable, LongWritable> question = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); question.setControlInfo(controlInfo); /* * Message format being sent: partitionID remotevertex1 remotevertex2 ... */ byte partitionIDbytes[] = Ints.toByteArray(peer.getPeerIndex()); controlInfo.addextraInfo(partitionIDbytes); for (IVertex<V, E, LongWritable, LongWritable> v : vertexMap.values()) { if (v instanceof RemoteVertex) { byte vertexIDbytes[] = Longs.toByteArray(v.getVertexId().get()); controlInfo.addextraInfo(vertexIDbytes); } } sendToAllPartitions(question); peer.sync(); Map<Integer, List<Message<LongWritable, LongWritable>>> replyMessages = new HashMap<Integer, List<Message<LongWritable, LongWritable>>>(); // Receiving 1 message per partition while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { /* * Subgraph Partition mapping broadcast Format of received message: * partitionID subgraphID1 subgraphID2 ... */ if (msg.getMessageType() == Message.MessageType.SUBGRAPH) { Iterable<BytesWritable> subgraphList = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Integer partitionID = Ints.fromByteArray(subgraphList.iterator().next().getBytes()); for (BytesWritable subgraphListElement : Iterables.skip(subgraphList, 1)) { LongWritable subgraphID = new LongWritable(Longs.fromByteArray(subgraphListElement.getBytes())); subgraphPartitionMap.put((K) subgraphID, partitionID); } continue; } /* * receiving query to find subgraph id Remote Vertex */ Iterable<BytesWritable> RemoteVertexQuery = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); /* * Reply format : sinkID1 subgraphID1 sinkID2 subgraphID2 ... */ Message<LongWritable, LongWritable> subgraphIDReply = new Message<LongWritable, LongWritable>(); controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.NORMAL); subgraphIDReply.setControlInfo(controlInfo); Integer sinkPartition = Ints.fromByteArray(RemoteVertexQuery.iterator().next().getBytes()); boolean hasAVertex = false; for (BytesWritable remoteVertex : Iterables.skip(RemoteVertexQuery, 1)) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(remoteVertex.getBytes())); LongWritable sinkSubgraphID = vertexSubgraphMap.get(sinkID); // In case this partition does not have the vertex /* * Case 1 : If vertex does not exist Case 2 : If vertex exists but is * remote, then its subgraphID is null */ if (sinkSubgraphID == null) { continue; } hasAVertex = true; byte sinkIDbytes[] = Longs.toByteArray(sinkID.get()); controlInfo.addextraInfo(sinkIDbytes); byte subgraphIDbytes[] = Longs.toByteArray(sinkSubgraphID.get()); controlInfo.addextraInfo(subgraphIDbytes); } if (hasAVertex) { peer.send(peer.getPeerName(sinkPartition.intValue()), (Message<K, M>) subgraphIDReply); } } peer.sync(); while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { Iterable<BytesWritable> remoteVertexReply = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Iterator<BytesWritable> queryResponse = remoteVertexReply.iterator(); while (queryResponse.hasNext()) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(queryResponse.next().getBytes())); LongWritable remoteSubgraphID = new LongWritable( Longs.fromByteArray(queryResponse.next().getBytes())); RemoteVertex<V, E, LongWritable, LongWritable, LongWritable> sink = (RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>) vertexMap .get(sinkID); assert (sink != null); sink.setSubgraphID(remoteSubgraphID); } } return partition.getSubgraphs(); }
From source file:in.dream_lab.goffish.LongMapJSONReader.java
License:Apache License
@Override public List<ISubgraph<S, V, E, LongWritable, LongWritable, LongWritable>> getSubgraphs() throws IOException, SyncException, InterruptedException { LOG.info("Creating vertices"); vertexMap = new HashMap<LongWritable, IVertex<V, E, LongWritable, LongWritable>>(); // List of edges.Used to create RemoteVertices List<IEdge<E, LongWritable, LongWritable>> _edges = new ArrayList<IEdge<E, LongWritable, LongWritable>>(); KeyValuePair<Writable, Writable> pair; while ((pair = peer.readNext()) != null) { String StringJSONInput = pair.getValue().toString(); Vertex<V, E, LongWritable, LongWritable> vertex = createVertex(StringJSONInput); vertexMap.put(vertex.getVertexID(), vertex); _edges.addAll(vertex.outEdges()); }/*from w w w . j a v a2s. c om*/ LOG.info("Sending Vertices to respective partitions"); LOG.info("Received all vertices"); /* Create remote vertex objects. */ for (IEdge<E, LongWritable, LongWritable> e : _edges) { LongWritable sinkID = e.getSinkVertexID(); IVertex<V, E, LongWritable, LongWritable> sink = vertexMap.get(sinkID); if (sink == null) { sink = new RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>(sinkID); vertexMap.put(sinkID, sink); } } //Direct Copy paste from here Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition = new Partition<S, V, E, LongWritable, LongWritable, LongWritable>( peer.getPeerIndex()); formSubgraphs(partition, vertexMap.values()); /* * Ask Remote vertices to send their subgraph IDs. Requires 2 supersteps * because the graph is directed */ Message<LongWritable, LongWritable> question = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); question.setControlInfo(controlInfo); /* * Message format being sent: * partitionID remotevertex1 remotevertex2 ... */ byte partitionIDbytes[] = Ints.toByteArray(peer.getPeerIndex()); controlInfo.addextraInfo(partitionIDbytes); for (IVertex<V, E, LongWritable, LongWritable> v : vertexMap.values()) { if (v instanceof RemoteVertex) { byte vertexIDbytes[] = Longs.toByteArray(v.getVertexID().get()); controlInfo.addextraInfo(vertexIDbytes); } } sendToAllPartitions(question); peer.sync(); Message<LongWritable, LongWritable> msg; //Receiving 1 message per partition while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { /* * Subgraph Partition mapping broadcast * Format of received message: * partitionID subgraphID1 subgraphID2 ... */ if (msg.getMessageType() == Message.MessageType.SUBGRAPH) { Iterable<BytesWritable> subgraphList = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Integer partitionID = Ints.fromByteArray(subgraphList.iterator().next().getBytes()); for (BytesWritable subgraphListElement : Iterables.skip(subgraphList, 1)) { LongWritable subgraphID = new LongWritable(Longs.fromByteArray(subgraphListElement.getBytes())); subgraphPartitionMap.put((K) subgraphID, partitionID); } continue; } /* * receiving query to find subgraph id Remote Vertex */ Iterable<BytesWritable> RemoteVertexQuery = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); /* * Reply format : * sinkID1 subgraphID1 sinkID2 subgraphID2 ... */ Message<LongWritable, LongWritable> subgraphIDReply = new Message<LongWritable, LongWritable>(); controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.NORMAL); subgraphIDReply.setControlInfo(controlInfo); Integer sinkPartition = Ints.fromByteArray(RemoteVertexQuery.iterator().next().getBytes()); boolean hasAVertex = false; for (BytesWritable remoteVertex : Iterables.skip(RemoteVertexQuery, 1)) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(remoteVertex.getBytes())); LongWritable sinkSubgraphID = vertexSubgraphMap.get(sinkID); //In case this partition does not have the vertex /* Case 1 : If vertex does not exist * Case 2 : If vertex exists but is remote, then its subgraphID is null */ if (sinkSubgraphID == null) { continue; } hasAVertex = true; byte sinkIDbytes[] = Longs.toByteArray(sinkID.get()); controlInfo.addextraInfo(sinkIDbytes); byte subgraphIDbytes[] = Longs.toByteArray(sinkSubgraphID.get()); controlInfo.addextraInfo(subgraphIDbytes); } if (hasAVertex) { peer.send(peer.getPeerName(sinkPartition.intValue()), (Message<K, M>) subgraphIDReply); } } peer.sync(); while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { Iterable<BytesWritable> remoteVertexReply = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Iterator<BytesWritable> queryResponse = remoteVertexReply.iterator(); while (queryResponse.hasNext()) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(queryResponse.next().getBytes())); LongWritable remoteSubgraphID = new LongWritable( Longs.fromByteArray(queryResponse.next().getBytes())); RemoteVertex<V, E, LongWritable, LongWritable, LongWritable> sink = (RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>) vertexMap .get(sinkID); if (sink == null) { System.out.println("NULL" + sink); } sink.setSubgraphID(remoteSubgraphID); } } return partition.getSubgraphs(); }
From source file:in.dream_lab.goffish.LongMapJSONReader.java
License:Apache License
void formSubgraphs(Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition, Collection<IVertex<V, E, LongWritable, LongWritable>> vertices) throws IOException { long subgraphCount = 0; Message<LongWritable, LongWritable> subgraphLocationBroadcast = new Message<LongWritable, LongWritable>(); subgraphLocationBroadcast.setMessageType(IMessage.MessageType.SUBGRAPH); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); subgraphLocationBroadcast.setControlInfo(controlInfo); byte partitionBytes[] = Ints.toByteArray(peer.getPeerIndex()); controlInfo.addextraInfo(partitionBytes); // initialize disjoint set DisjointSets<IVertex<V, E, LongWritable, LongWritable>> ds = new DisjointSets<IVertex<V, E, LongWritable, LongWritable>>( vertices.size());/*from w ww. j a va2 s.co m*/ for (IVertex<V, E, LongWritable, LongWritable> vertex : vertices) { ds.addSet(vertex); } // union edge pairs for (IVertex<V, E, LongWritable, LongWritable> vertex : vertices) { if (!vertex.isRemote()) { for (IEdge<E, LongWritable, LongWritable> edge : vertex.outEdges()) { IVertex<V, E, LongWritable, LongWritable> sink = vertexMap.get(edge.getSinkVertexID()); ds.union(vertex, sink); } } } Collection<? extends Collection<IVertex<V, E, LongWritable, LongWritable>>> components = ds.retrieveSets(); for (Collection<IVertex<V, E, LongWritable, LongWritable>> component : components) { LongWritable subgraphID = new LongWritable( subgraphCount++ | (((long) partition.getPartitionID()) << 32)); Subgraph<S, V, E, LongWritable, LongWritable, LongWritable> subgraph = new Subgraph<S, V, E, LongWritable, LongWritable, LongWritable>( peer.getPeerIndex(), subgraphID); for (IVertex<V, E, LongWritable, LongWritable> vertex : component) { subgraph.addVertex(vertex); // Dont add remote vertices to the VertexSubgraphMap as remote vertex subgraphID is unknown if (!vertex.isRemote()) { vertexSubgraphMap.put(vertex.getVertexID(), subgraph.getSubgraphID()); } } partition.addSubgraph(subgraph); byte subgraphIDbytes[] = Longs.toByteArray(subgraphID.get()); controlInfo.addextraInfo(subgraphIDbytes); } sendToAllPartitions(subgraphLocationBroadcast); }
From source file:in.dream_lab.goffish.LongTextAdjacencyListReader.java
License:Apache License
@Override public List<ISubgraph<S, V, E, LongWritable, LongWritable, LongWritable>> getSubgraphs() throws IOException, SyncException, InterruptedException { Map<Integer, List<String>> partitionMap = new HashMap<Integer, List<String>>(); vertexMap = new HashMap<LongWritable, IVertex<V, E, LongWritable, LongWritable>>(); // List of edges.Used to create RemoteVertices List<IEdge<E, LongWritable, LongWritable>> _edges = new ArrayList<IEdge<E, LongWritable, LongWritable>>(); long edgeCount = 0; LOG.info("SETUP Starting " + peer.getPeerIndex() + " Memory: " + Runtime.getRuntime().freeMemory()); KeyValuePair<Writable, Writable> pair; while ((pair = peer.readNext()) != null) { //NOTE: Confirm that data starts from value and not from key. String stringInput = pair.getValue().toString(); String vertexValue[] = stringInput.split("\\s+"); //LongWritable sourceID = new LongWritable(Long.parseLong(value[0])); int partitionID = Integer.parseInt(vertexValue[1]); // Vertex does not belong to this partition if (partitionID != peer.getPeerIndex()) { List<String> partitionVertices = partitionMap.get(partitionID); if (partitionVertices == null) { partitionVertices = new ArrayList<String>(); partitionMap.put(partitionID, partitionVertices); }/* w w w . j a v a2 s.com*/ partitionVertices.add(stringInput); } else { LongWritable vertexID = new LongWritable(Long.parseLong(vertexValue[0])); Vertex<V, E, LongWritable, LongWritable> vertex; vertex = new Vertex<V, E, LongWritable, LongWritable>(vertexID); for (int j = 2; j < vertexValue.length; j++) { LongWritable sinkID = new LongWritable(Long.parseLong(vertexValue[j])); LongWritable edgeID = new LongWritable(edgeCount++ | (((long) peer.getPeerIndex()) << 32)); Edge<E, LongWritable, LongWritable> e = new Edge<E, LongWritable, LongWritable>(edgeID, sinkID); vertex.addEdge(e); _edges.add(e); } vertexMap.put(vertex.getVertexID(), vertex); _edges.addAll(vertex.outEdges()); } } // Send vertices to their respective partitions for (Map.Entry<Integer, List<String>> entry : partitionMap.entrySet()) { int partitionID = entry.getKey().intValue(); List<String> vertices = entry.getValue(); System.out.println("To " + partitionID + " " + vertices.size()); for (String vertex : vertices) { Message<LongWritable, LongWritable> vertexMsg = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.VERTEX); controlInfo.setVertexValues(vertex); vertexMsg.setControlInfo(controlInfo); peer.send(peer.getPeerName(partitionID), (Message<K, M>) vertexMsg); } } // End of first superstep. System.gc(); peer.sync(); LOG.info("Second Superstep in Reader " + peer.getPeerIndex() + " Memory: " + Runtime.getRuntime().freeMemory()); Message<LongWritable, LongWritable> msg; while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { ControlMessage ctrlMessage = (ControlMessage) msg.getControlInfo(); String msgString = ctrlMessage.getVertexValues(); // String msgStringArr[] = msgString.split(","); // for (int i = 0; i < msgStringArr.length; i++) { String vertexInfo[] = msgString.split("\\s+"); LongWritable vertexID = new LongWritable(Long.parseLong(vertexInfo[0])); Vertex<V, E, LongWritable, LongWritable> source = (Vertex<V, E, LongWritable, LongWritable>) vertexMap .get(vertexID); if (source == null) { source = new Vertex<V, E, LongWritable, LongWritable>(vertexID); vertexMap.put(source.getVertexID(), source); } for (int j = 2; j < vertexInfo.length; j++) { LongWritable sinkID = new LongWritable(Long.parseLong(vertexInfo[j])); LongWritable edgeID = new LongWritable(edgeCount++ | (((long) peer.getPeerIndex()) << 32)); Edge<E, LongWritable, LongWritable> e = new Edge<E, LongWritable, LongWritable>(edgeID, sinkID); source.addEdge(e); _edges.add(e); } } //} LOG.info("Creating Remote Vertex Objects"); /* Create remote vertex objects. */ for (IEdge<E, LongWritable, LongWritable> e : _edges) { LongWritable sinkID = e.getSinkVertexID(); IVertex<V, E, LongWritable, LongWritable> sink = vertexMap.get(sinkID); if (sink == null) { sink = new RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>(sinkID); vertexMap.put(sinkID, sink); } } Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition = new Partition<S, V, E, LongWritable, LongWritable, LongWritable>( peer.getPeerIndex()); LOG.info("Calling formSubgraph()"); formSubgraphs(partition, vertexMap.values()); LOG.info("Done with formSubgraph()"); /* * Ask Remote vertices to send their subgraph IDs. Requires 2 supersteps * because the graph is directed */ Message<LongWritable, LongWritable> question = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); question.setControlInfo(controlInfo); /* * Message format being sent: * partitionID remotevertex1 remotevertex2 ... */ byte partitionIDbytes[] = Ints.toByteArray(peer.getPeerIndex()); controlInfo.addextraInfo(partitionIDbytes); for (IVertex<V, E, LongWritable, LongWritable> v : vertexMap.values()) { if (v instanceof RemoteVertex) { byte vertexIDbytes[] = Longs.toByteArray(v.getVertexID().get()); controlInfo.addextraInfo(vertexIDbytes); } } sendToAllPartitions(question); peer.sync(); Map<Integer, List<Message<LongWritable, LongWritable>>> replyMessages = new HashMap<Integer, List<Message<LongWritable, LongWritable>>>(); //Receiving 1 message per partition while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { /* * Subgraph Partition mapping broadcast * Format of received message: * partitionID subgraphID1 subgraphID2 ... */ if (msg.getMessageType() == Message.MessageType.SUBGRAPH) { Iterable<BytesWritable> subgraphList = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Integer partitionID = Ints.fromByteArray(subgraphList.iterator().next().getBytes()); for (BytesWritable subgraphListElement : Iterables.skip(subgraphList, 1)) { LongWritable subgraphID = new LongWritable(Longs.fromByteArray(subgraphListElement.getBytes())); subgraphPartitionMap.put((K) subgraphID, partitionID); } continue; } /* * receiving query to find subgraph id Remote Vertex */ Iterable<BytesWritable> RemoteVertexQuery = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); /* * Reply format : * sinkID1 subgraphID1 sinkID2 subgraphID2 ... */ Message<LongWritable, LongWritable> subgraphIDReply = new Message<LongWritable, LongWritable>(); controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.NORMAL); subgraphIDReply.setControlInfo(controlInfo); Integer sinkPartition = Ints.fromByteArray(RemoteVertexQuery.iterator().next().getBytes()); boolean hasAVertex = false; for (BytesWritable remoteVertex : Iterables.skip(RemoteVertexQuery, 1)) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(remoteVertex.getBytes())); LongWritable sinkSubgraphID = vertexSubgraphMap.get(sinkID); //In case this partition does not have the vertex /* Case 1 : If vertex does not exist * Case 2 : If vertex exists but is remote, then its subgraphID is null */ if (sinkSubgraphID == null) { continue; } hasAVertex = true; byte sinkIDbytes[] = Longs.toByteArray(sinkID.get()); controlInfo.addextraInfo(sinkIDbytes); byte subgraphIDbytes[] = Longs.toByteArray(sinkSubgraphID.get()); controlInfo.addextraInfo(subgraphIDbytes); } if (hasAVertex) { peer.send(peer.getPeerName(sinkPartition.intValue()), (Message<K, M>) subgraphIDReply); } } peer.sync(); while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { Iterable<BytesWritable> remoteVertexReply = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Iterator<BytesWritable> queryResponse = remoteVertexReply.iterator(); while (queryResponse.hasNext()) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(queryResponse.next().getBytes())); LongWritable remoteSubgraphID = new LongWritable( Longs.fromByteArray(queryResponse.next().getBytes())); RemoteVertex<V, E, LongWritable, LongWritable, LongWritable> sink = (RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>) vertexMap .get(sinkID); if (sink == null) { System.out.println("NULLLL" + sink); } sink.setSubgraphID(remoteSubgraphID); } } return partition.getSubgraphs(); }
From source file:in.dream_lab.goffish.LongTextAdjacencyListReader.java
License:Apache License
void formSubgraphs(Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition, Collection<IVertex<V, E, LongWritable, LongWritable>> vertices) throws IOException { long subgraphCount = 0; Set<LongWritable> visited = new HashSet<LongWritable>(); Message<LongWritable, LongWritable> subgraphLocationBroadcast = new Message<LongWritable, LongWritable>(); subgraphLocationBroadcast.setMessageType(IMessage.MessageType.SUBGRAPH); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); subgraphLocationBroadcast.setControlInfo(controlInfo); byte partitionBytes[] = Ints.toByteArray(peer.getPeerIndex()); controlInfo.addextraInfo(partitionBytes); // initialize disjoint set DisjointSets<IVertex<V, E, LongWritable, LongWritable>> ds = new DisjointSets<IVertex<V, E, LongWritable, LongWritable>>( vertices.size());// w ww . jav a 2s .c o m for (IVertex<V, E, LongWritable, LongWritable> vertex : vertices) { ds.addSet(vertex); } // union edge pairs for (IVertex<V, E, LongWritable, LongWritable> vertex : vertices) { if (!vertex.isRemote()) { for (IEdge<E, LongWritable, LongWritable> edge : vertex.outEdges()) { IVertex<V, E, LongWritable, LongWritable> sink = vertexMap.get(edge.getSinkVertexID()); ds.union(vertex, sink); } } } Collection<? extends Collection<IVertex<V, E, LongWritable, LongWritable>>> components = ds.retrieveSets(); for (Collection<IVertex<V, E, LongWritable, LongWritable>> component : components) { LongWritable subgraphID = new LongWritable( subgraphCount++ | (((long) partition.getPartitionID()) << 32)); Subgraph<S, V, E, LongWritable, LongWritable, LongWritable> subgraph = new Subgraph<S, V, E, LongWritable, LongWritable, LongWritable>( peer.getPeerIndex(), subgraphID); for (IVertex<V, E, LongWritable, LongWritable> vertex : component) { subgraph.addVertex(vertex); // Dont add remote vertices to the VertexSubgraphMap as remote vertex subgraphID is unknown if (!vertex.isRemote()) { vertexSubgraphMap.put(vertex.getVertexID(), subgraph.getSubgraphID()); } } partition.addSubgraph(subgraph); byte subgraphIDbytes[] = Longs.toByteArray(subgraphID.get()); controlInfo.addextraInfo(subgraphIDbytes); } sendToAllPartitions(subgraphLocationBroadcast); }
From source file:in.dream_lab.goffish.LongTextJSONReader.java
License:Apache License
@Override public List<ISubgraph<S, V, E, LongWritable, LongWritable, LongWritable>> getSubgraphs() throws IOException, SyncException, InterruptedException { // Map of partitionID,vertex that do not belong to this partition Map<Integer, List<String>> partitionMap = new HashMap<Integer, List<String>>(); vertexMap = new HashMap<LongWritable, IVertex<V, E, LongWritable, LongWritable>>(); // List of edges.Used to create RemoteVertices List<IEdge<E, LongWritable, LongWritable>> _edges = new ArrayList<IEdge<E, LongWritable, LongWritable>>(); KeyValuePair<Writable, Writable> pair; while ((pair = peer.readNext()) != null) { String StringJSONInput = pair.getValue().toString(); JSONArray JSONInput = (JSONArray) JSONValue.parse(StringJSONInput); int partitionID = Integer.parseInt(JSONInput.get(1).toString()); // Vertex does not belong to this partition if (partitionID != peer.getPeerIndex()) { List<String> partitionVertices = partitionMap.get(partitionID); if (partitionVertices == null) { partitionVertices = new ArrayList<String>(); partitionMap.put(partitionID, partitionVertices); }/* w ww. j a v a 2s . co m*/ partitionVertices.add(StringJSONInput); } else { Vertex<V, E, LongWritable, LongWritable> vertex = createVertex(StringJSONInput); vertexMap.put(vertex.getVertexID(), vertex); _edges.addAll(vertex.outEdges()); } } // Send vertices to their respective partitions for (Map.Entry<Integer, List<String>> entry : partitionMap.entrySet()) { int partitionID = entry.getKey().intValue(); List<String> vertices = entry.getValue(); for (String vertex : vertices) { Message<LongWritable, LongWritable> vertexMsg = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.VERTEX); controlInfo.setVertexValues(vertex); vertexMsg.setControlInfo(controlInfo); peer.send(peer.getPeerName(partitionID), (Message<K, M>) vertexMsg); } } //End of first SuperStep peer.sync(); Message<LongWritable, LongWritable> msg; while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { String JSONVertex = msg.getControlInfo().toString(); Vertex<V, E, LongWritable, LongWritable> vertex = createVertex(JSONVertex); vertexMap.put(vertex.getVertexID(), vertex); _edges.addAll(vertex.outEdges()); } /* Create remote vertex objects. */ for (IEdge<E, LongWritable, LongWritable> e : _edges) { LongWritable sinkID = e.getSinkVertexID(); IVertex<V, E, LongWritable, LongWritable> sink = vertexMap.get(sinkID); if (sink == null) { sink = new RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>(sinkID); vertexMap.put(sinkID, sink); } } //Direct Copy paste from here Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition = new Partition<S, V, E, LongWritable, LongWritable, LongWritable>( peer.getPeerIndex()); formSubgraphs(partition, vertexMap.values()); /* * Ask Remote vertices to send their subgraph IDs. Requires 2 supersteps * because the graph is directed */ Message<LongWritable, LongWritable> question = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); question.setControlInfo(controlInfo); /* * Message format being sent: * partitionID remotevertex1 remotevertex2 ... */ byte partitionIDbytes[] = Ints.toByteArray(peer.getPeerIndex()); controlInfo.addextraInfo(partitionIDbytes); for (IVertex<V, E, LongWritable, LongWritable> v : vertexMap.values()) { if (v instanceof RemoteVertex) { byte vertexIDbytes[] = Longs.toByteArray(v.getVertexID().get()); controlInfo.addextraInfo(vertexIDbytes); } } sendToAllPartitions(question); peer.sync(); Map<Integer, List<Message<LongWritable, LongWritable>>> replyMessages = new HashMap<Integer, List<Message<LongWritable, LongWritable>>>(); //Receiving 1 message per partition while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { /* * Subgraph Partition mapping broadcast * Format of received message: * partitionID subgraphID1 subgraphID2 ... */ if (msg.getMessageType() == Message.MessageType.SUBGRAPH) { Iterable<BytesWritable> subgraphList = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Integer partitionID = Ints.fromByteArray(subgraphList.iterator().next().getBytes()); for (BytesWritable subgraphListElement : Iterables.skip(subgraphList, 1)) { LongWritable subgraphID = new LongWritable(Longs.fromByteArray(subgraphListElement.getBytes())); subgraphPartitionMap.put((K) subgraphID, partitionID); } continue; } /* * receiving query to find subgraph id Remote Vertex */ Iterable<BytesWritable> RemoteVertexQuery = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); /* * Reply format : * sinkID1 subgraphID1 sinkID2 subgraphID2 ... */ Message<LongWritable, LongWritable> subgraphIDReply = new Message<LongWritable, LongWritable>(); controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.NORMAL); subgraphIDReply.setControlInfo(controlInfo); Integer sinkPartition = Ints.fromByteArray(RemoteVertexQuery.iterator().next().getBytes()); boolean hasAVertex = false; for (BytesWritable remoteVertex : Iterables.skip(RemoteVertexQuery, 1)) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(remoteVertex.getBytes())); LongWritable sinkSubgraphID = vertexSubgraphMap.get(sinkID); //In case this partition does not have the vertex /* Case 1 : If vertex does not exist * Case 2 : If vertex exists but is remote, then its subgraphID is null */ if (sinkSubgraphID == null) { continue; } hasAVertex = true; byte sinkIDbytes[] = Longs.toByteArray(sinkID.get()); controlInfo.addextraInfo(sinkIDbytes); byte subgraphIDbytes[] = Longs.toByteArray(sinkSubgraphID.get()); controlInfo.addextraInfo(subgraphIDbytes); } if (hasAVertex) { peer.send(peer.getPeerName(sinkPartition.intValue()), (Message<K, M>) subgraphIDReply); } } peer.sync(); while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { Iterable<BytesWritable> remoteVertexReply = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Iterator<BytesWritable> queryResponse = remoteVertexReply.iterator(); while (queryResponse.hasNext()) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(queryResponse.next().getBytes())); LongWritable remoteSubgraphID = new LongWritable( Longs.fromByteArray(queryResponse.next().getBytes())); RemoteVertex<V, E, LongWritable, LongWritable, LongWritable> sink = (RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>) vertexMap .get(sinkID); if (sink == null) { System.out.println("NULLLL" + sink); } sink.setSubgraphID(remoteSubgraphID); } } return partition.getSubgraphs(); }
From source file:in.dream_lab.goffish.PartitionsLongTextAdjacencyListReader.java
License:Apache License
@Override public List<ISubgraph<S, V, E, LongWritable, LongWritable, LongWritable>> getSubgraphs() throws IOException, SyncException, InterruptedException { Map<Integer, List<String>> partitionMap = new HashMap<Integer, List<String>>(); vertexMap = new HashMap<LongWritable, IVertex<V, E, LongWritable, LongWritable>>(); // List of edges.Used to create RemoteVertices List<IEdge<E, LongWritable, LongWritable>> _edges = new ArrayList<IEdge<E, LongWritable, LongWritable>>(); long edgeCount = 0; LOG.info("SETUP Starting " + peer.getPeerIndex() + " Memory: " + Runtime.getRuntime().freeMemory()); KeyValuePair<Writable, Writable> pair; while ((pair = peer.readNext()) != null) { // NOTE: Confirm that data starts from value and not from key. String stringInput = pair.getValue().toString(); String vertexValue[] = stringInput.split("\\s+"); // LongWritable sourceID = new LongWritable(Long.parseLong(value[0])); // int partitionID = Integer.parseInt(vertexValue[1]); LongWritable vertexID = new LongWritable(Long.parseLong(vertexValue[0])); Vertex<V, E, LongWritable, LongWritable> vertex; vertex = new Vertex<V, E, LongWritable, LongWritable>(vertexID); for (int j = 1; j < vertexValue.length; j++) { LongWritable sinkID = new LongWritable(Long.parseLong(vertexValue[j])); LongWritable edgeID = new LongWritable(edgeCount++ | (((long) peer.getPeerIndex()) << 32)); Edge<E, LongWritable, LongWritable> e = new Edge<E, LongWritable, LongWritable>(edgeID, sinkID); vertex.addEdge(e);//www .j av a 2s . c om _edges.add(e); } vertexMap.put(vertex.getVertexID(), vertex); _edges.addAll(vertex.outEdges()); } LOG.info("Number of Vertices: " + vertexMap.size() + " Edges: " + _edges.size()); LOG.info("Creating Remote Vertex Objects"); /* Create remote vertex objects. */ for (IEdge<E, LongWritable, LongWritable> e : _edges) { LongWritable sinkID = e.getSinkVertexID(); IVertex<V, E, LongWritable, LongWritable> sink = vertexMap.get(sinkID); if (sink == null) { sink = new RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>(sinkID); vertexMap.put(sinkID, sink); } } Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition = new Partition<S, V, E, LongWritable, LongWritable, LongWritable>( peer.getPeerIndex()); LOG.info("Calling formSubgraph()"); formSubgraphs(partition, vertexMap.values()); LOG.info("Done with formSubgraph()"); /* * Ask Remote vertices to send their subgraph IDs. Requires 2 supersteps * because the graph is directed */ Message<LongWritable, LongWritable> question = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); question.setControlInfo(controlInfo); /* * Message format being sent: partitionID remotevertex1 remotevertex2 ... */ byte partitionIDbytes[] = Ints.toByteArray(peer.getPeerIndex()); controlInfo.addextraInfo(partitionIDbytes); for (IVertex<V, E, LongWritable, LongWritable> v : vertexMap.values()) { if (v instanceof RemoteVertex) { byte vertexIDbytes[] = Longs.toByteArray(v.getVertexID().get()); controlInfo.addextraInfo(vertexIDbytes); } } sendToAllPartitions(question); peer.sync(); Message<LongWritable, LongWritable> msg; Map<Integer, List<Message<LongWritable, LongWritable>>> replyMessages = new HashMap<Integer, List<Message<LongWritable, LongWritable>>>(); // Receiving 1 message per partition while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { /* * Subgraph Partition mapping broadcast Format of received message: * partitionID subgraphID1 subgraphID2 ... */ if (msg.getMessageType() == Message.MessageType.SUBGRAPH) { Iterable<BytesWritable> subgraphList = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Integer partitionID = Ints.fromByteArray(subgraphList.iterator().next().getBytes()); for (BytesWritable subgraphListElement : Iterables.skip(subgraphList, 1)) { LongWritable subgraphID = new LongWritable(Longs.fromByteArray(subgraphListElement.getBytes())); subgraphPartitionMap.put((K) subgraphID, partitionID); } continue; } /* * receiving query to find subgraph id Remote Vertex */ Iterable<BytesWritable> RemoteVertexQuery = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); /* * Reply format : sinkID1 subgraphID1 sinkID2 subgraphID2 ... */ Message<LongWritable, LongWritable> subgraphIDReply = new Message<LongWritable, LongWritable>(); controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.NORMAL); subgraphIDReply.setControlInfo(controlInfo); Integer sinkPartition = Ints.fromByteArray(RemoteVertexQuery.iterator().next().getBytes()); boolean hasAVertex = false; for (BytesWritable remoteVertex : Iterables.skip(RemoteVertexQuery, 1)) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(remoteVertex.getBytes())); LongWritable sinkSubgraphID = vertexSubgraphMap.get(sinkID); // In case this partition does not have the vertex /* * Case 1 : If vertex does not exist Case 2 : If vertex exists but is * remote, then its subgraphID is null */ if (sinkSubgraphID == null) { continue; } hasAVertex = true; byte sinkIDbytes[] = Longs.toByteArray(sinkID.get()); controlInfo.addextraInfo(sinkIDbytes); byte subgraphIDbytes[] = Longs.toByteArray(sinkSubgraphID.get()); controlInfo.addextraInfo(subgraphIDbytes); } if (hasAVertex) { peer.send(peer.getPeerName(sinkPartition.intValue()), (Message<K, M>) subgraphIDReply); } } peer.sync(); while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { Iterable<BytesWritable> remoteVertexReply = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Iterator<BytesWritable> queryResponse = remoteVertexReply.iterator(); while (queryResponse.hasNext()) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(queryResponse.next().getBytes())); LongWritable remoteSubgraphID = new LongWritable( Longs.fromByteArray(queryResponse.next().getBytes())); RemoteVertex<V, E, LongWritable, LongWritable, LongWritable> sink = (RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>) vertexMap .get(sinkID); if (sink == null) { System.out.println("NULLLL" + sink); } sink.setSubgraphID(remoteSubgraphID); } } return partition.getSubgraphs(); }
From source file:in.dream_lab.goffish.PartitionsLongTextAdjacencyListReader.java
License:Apache License
void formSubgraphs(Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition, Collection<IVertex<V, E, LongWritable, LongWritable>> vertices) throws IOException { long subgraphCount = 0; Message<LongWritable, LongWritable> subgraphLocationBroadcast = new Message<LongWritable, LongWritable>(); subgraphLocationBroadcast.setMessageType(IMessage.MessageType.SUBGRAPH); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); subgraphLocationBroadcast.setControlInfo(controlInfo); byte partitionBytes[] = Ints.toByteArray(peer.getPeerIndex()); controlInfo.addextraInfo(partitionBytes); // initialize disjoint set DisjointSets<IVertex<V, E, LongWritable, LongWritable>> ds = new DisjointSets<IVertex<V, E, LongWritable, LongWritable>>( vertices.size());// ww w. jav a 2s.co m for (IVertex<V, E, LongWritable, LongWritable> vertex : vertices) { ds.addSet(vertex); } // union edge pairs for (IVertex<V, E, LongWritable, LongWritable> vertex : vertices) { if (!vertex.isRemote()) { for (IEdge<E, LongWritable, LongWritable> edge : vertex.outEdges()) { IVertex<V, E, LongWritable, LongWritable> sink = vertexMap.get(edge.getSinkVertexID()); ds.union(vertex, sink); } } } Collection<? extends Collection<IVertex<V, E, LongWritable, LongWritable>>> components = ds.retrieveSets(); for (Collection<IVertex<V, E, LongWritable, LongWritable>> component : components) { LongWritable subgraphID = new LongWritable( subgraphCount++ | (((long) partition.getPartitionID()) << 32)); Subgraph<S, V, E, LongWritable, LongWritable, LongWritable> subgraph = new Subgraph<S, V, E, LongWritable, LongWritable, LongWritable>( peer.getPeerIndex(), subgraphID); for (IVertex<V, E, LongWritable, LongWritable> vertex : component) { subgraph.addVertex(vertex); // Dont add remote vertices to the VertexSubgraphMap as remote vertex // subgraphID is unknown if (!vertex.isRemote()) { vertexSubgraphMap.put(vertex.getVertexID(), subgraph.getSubgraphID()); } } partition.addSubgraph(subgraph); byte subgraphIDbytes[] = Longs.toByteArray(subgraphID.get()); controlInfo.addextraInfo(subgraphIDbytes); } sendToAllPartitions(subgraphLocationBroadcast); }